From 176a068cde4f2d9dfa0336168caead0b1edcb8ac Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Mon, 16 Jan 2023 15:38:33 -0600
Subject: [PATCH 001/871] [extractor/nbc] Fix XML parsing

Python 3.7 compat bug in cb73b8460c3ce6d37ab651a4e44bb23b10056154
Authored by: bashonly
---
 yt_dlp/extractor/nbc.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index 82d759f75..b9f65e927 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -3,7 +3,7 @@
 import re
 
 from .common import InfoExtractor
-from .theplatform import ThePlatformIE
+from .theplatform import ThePlatformIE, default_ns
 from .adobepass import AdobePassIE
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
@@ -700,7 +700,7 @@ def _real_extract(self, url):
         webpage = self._download_webpage(url, video_id)
 
         nbc_data = self._search_json(
-            r'<script>var\s*nbc\s*=', webpage, 'NBC JSON data', video_id)
+            r'<script>\s*var\s+nbc\s*=', webpage, 'NBC JSON data', video_id)
         pdk_acct = nbc_data.get('pdkAcct') or 'Yh1nAC'
         fw_ssid = traverse_obj(nbc_data, ('video', 'fwSSID'))
 
@@ -771,8 +771,8 @@ def _real_extract(self, url):
                 f'https://link.theplatform.com/s/{pdk_acct}/{player_id}', video_id,
                 note='Downloading SMIL data', query=query, fatal=is_live)
         if smil:
-            manifest_url = xpath_attr(smil, './/{*}video', 'src', fatal=is_live)
-            subtitles = self._parse_smil_subtitles(smil, '*')
+            manifest_url = xpath_attr(smil, f'.//{{{default_ns}}}video', 'src', fatal=is_live)
+            subtitles = self._parse_smil_subtitles(smil, default_ns)
             fmts, subs = self._extract_m3u8_formats_and_subtitles(
                 manifest_url, video_id, 'mp4', m3u8_id='hls', fatal=is_live,
                 live=is_live, errnote='No HLS formats found')

From 88d8928bf7630801865cf8728ae5c77234324b7b Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 20 Jan 2023 23:34:16 +0530
Subject: [PATCH 002/871] [plugins] Fix zip search paths

Closes #6011
---
 yt_dlp/plugins.py | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/yt_dlp/plugins.py b/yt_dlp/plugins.py
index ff5ab9d5e..6eecdb4d0 100644
--- a/yt_dlp/plugins.py
+++ b/yt_dlp/plugins.py
@@ -34,9 +34,15 @@ def exec_module(self, module):
 
 @functools.cache
 def dirs_in_zip(archive):
-    with ZipFile(archive) as zip:
-        return set(itertools.chain.from_iterable(
-            Path(file).parents for file in zip.namelist()))
+    try:
+        with ZipFile(archive) as zip_:
+            return set(itertools.chain.from_iterable(
+                Path(file).parents for file in zip_.namelist()))
+    except FileNotFoundError:
+        pass
+    except Exception as e:
+        write_string(f'WARNING: Could not read zip file {archive}: {e}\n')
+    return set()
 
 
 class PluginFinder(importlib.abc.MetaPathFinder):
@@ -57,10 +63,8 @@ def search_locations(self, fullname):
 
         def _get_package_paths(*root_paths, containing_folder='plugins'):
             for config_dir in orderedSet(map(Path, root_paths), lazy=True):
-                plugin_dir = config_dir / containing_folder
-                if not plugin_dir.is_dir():
-                    continue
-                yield from plugin_dir.iterdir()
+                with contextlib.suppress(OSError):
+                    yield from (config_dir / containing_folder).iterdir()
 
         # Load from yt-dlp config folders
         candidate_locations.extend(_get_package_paths(
@@ -76,24 +80,23 @@ def _get_package_paths(*root_paths, containing_folder='plugins'):
             containing_folder='yt-dlp-plugins'))
 
         candidate_locations.extend(map(Path, sys.path))  # PYTHONPATH
+        with contextlib.suppress(ValueError):  # Added when running __main__.py directly
+            candidate_locations.remove(Path(__file__).parent)
 
         parts = Path(*fullname.split('.'))
-        locations = set()
-        for path in dict.fromkeys(candidate_locations):
+        for path in orderedSet(candidate_locations, lazy=True):
             candidate = path / parts
             if candidate.is_dir():
-                locations.add(str(candidate))
-            elif path.name and any(path.with_suffix(suffix).is_file() for suffix in {'.zip', '.egg', '.whl'}):
-                with contextlib.suppress(FileNotFoundError):
-                    if parts in dirs_in_zip(path):
-                        locations.add(str(candidate))
-        return locations
+                yield candidate
+            elif path.suffix in ('.zip', '.egg', '.whl'):
+                if parts in dirs_in_zip(path):
+                    yield candidate
 
     def find_spec(self, fullname, path=None, target=None):
         if fullname not in self.packages:
             return None
 
-        search_locations = self.search_locations(fullname)
+        search_locations = list(map(str, self.search_locations(fullname)))
         if not search_locations:
             return None
 

From 59d7de0da545944c48a82fc2937b996d7cd8cc9c Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 24 Jan 2023 03:43:48 +0530
Subject: [PATCH 003/871] Fix `--concat-playlist`

Closes #6080
---
 yt_dlp/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 1fb44e7f9..fd280726f 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1777,7 +1777,7 @@ def _playlist_infodict(ie_result, strict=False, **kwargs):
         return {
             **info,
             'playlist_index': 0,
-            '__last_playlist_index': max(ie_result['requested_entries'] or (0, 0)),
+            '__last_playlist_index': max(ie_result.get('requested_entries') or (0, 0)),
             'extractor': ie_result['extractor'],
             'extractor_key': ie_result['extractor_key'],
         }

From 37e325b92ff9d784715ac0e5d1f7d96bf5f45ad9 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 25 Jan 2023 22:32:07 +0100
Subject: [PATCH 004/871] [utils] Use local kernel32 for file locking on
 Windows

Ref: https://github.com/ytdl-org/youtube-dl/issues/21545

Authored by: Grub4K
---
 yt_dlp/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 15e1f97cb..458239a12 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -2106,7 +2106,7 @@ class OVERLAPPED(ctypes.Structure):
             ('hEvent', ctypes.wintypes.HANDLE),
         ]
 
-    kernel32 = ctypes.windll.kernel32
+    kernel32 = ctypes.WinDLL('kernel32')
     LockFileEx = kernel32.LockFileEx
     LockFileEx.argtypes = [
         ctypes.wintypes.HANDLE,     # hFile

From 8aa0bd5d10627ece3c1815c01d02fb8bf22847a7 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sun, 29 Jan 2023 00:59:37 -0600
Subject: [PATCH 005/871] [extractor/generic] Avoid catastrophic backtracking
 in KVS regex

Authored by: bashonly
---
 yt_dlp/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 04677b23f..9e4df4cea 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -2637,11 +2637,11 @@ def _extract_embeds(self, url, webpage, *, urlh=None, info_dict={}):
 
         # Look for generic KVS player (before json-ld bc of some urls that break otherwise)
         found = self._search_regex((
-            r'<script\b[^>]+?\bsrc\s*=\s*(["\'])https?://(?:\S+?/)+kt_player\.js\?v=(?P<ver>\d+(?:\.\d+)+)\1[^>]*>',
-            r'kt_player\s*\(\s*(["\'])(?:(?!\1)[\w\W])+\1\s*,\s*(["\'])https?://(?:\S+?/)+kt_player\.swf\?v=(?P<ver>\d+(?:\.\d+)+)\2\s*,',
+            r'<script\b[^>]+?\bsrc\s*=\s*(["\'])https?://(?:(?!\1)[^?#])+/kt_player\.js\?v=(?P<ver>\d+(?:\.\d+)+)\1[^>]*>',
+            r'kt_player\s*\(\s*(["\'])(?:(?!\1)[\w\W])+\1\s*,\s*(["\'])https?://(?:(?!\2)[^?#])+/kt_player\.swf\?v=(?P<ver>\d+(?:\.\d+)+)\2\s*,',
         ), webpage, 'KVS player', group='ver', default=False)
         if found:
-            self.report_detected('KWS Player')
+            self.report_detected('KVS Player')
             if found.split('.')[0] not in ('4', '5', '6'):
                 self.report_warning(f'Untested major version ({found}) in player engine - download may fail.')
             return [self._extract_kvs(url, webpage, video_id)]

From 83c4970e52839ce8761ec61bd19d549aed7d7920 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Tue, 31 Jan 2023 22:30:00 +0900
Subject: [PATCH 006/871] [utils] Fix `time_seconds` to use the provided TZ
 (#6118)

Authored by: Lesmiscore, Grub4K

Fixes https://github.com/yt-dlp/yt-dlp/pull/6056
---
 yt_dlp/utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 458239a12..7d51fe472 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5585,8 +5585,10 @@ def get_first(obj, keys, **kwargs):
 
 
 def time_seconds(**kwargs):
-    t = datetime.datetime.now(datetime.timezone(datetime.timedelta(**kwargs)))
-    return t.timestamp()
+    """
+    Returns TZ-aware time in seconds since the epoch (1970-01-01T00:00:00Z)
+    """
+    return time.time() + datetime.timedelta(**kwargs).total_seconds()
 
 
 # create a JSON Web Signature (jws) with HS256 algorithm

From 8b008d62544b82e24a0ba36c30e8e51855d93419 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 1 Feb 2023 09:39:49 +0530
Subject: [PATCH 007/871] [jsinterp] Support `if` statements

Closes #6131
---
 test/test_jsinterp.py          | 32 ++++++++++++++++++++++++++++++++
 test/test_youtube_signature.py |  4 ++++
 yt_dlp/jsinterp.py             | 15 +++++++++++++++
 3 files changed, 51 insertions(+)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 3c4391c4a..e090dc791 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -155,6 +155,38 @@ def test_call(self):
         self.assertEqual(jsi.call_function('z'), 5)
         self.assertEqual(jsi.call_function('y'), 2)
 
+    def test_if(self):
+        jsi = JSInterpreter('''
+        function x() {
+            let a = 9;
+            if (0==0) {a++}
+            return a
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+        jsi = JSInterpreter('''
+        function x() {
+            if (0==0) {return 10}
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+        jsi = JSInterpreter('''
+        function x() {
+            if (0!=0) {return 1}
+            else {return 10}
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+        """  # Unsupported
+        jsi = JSInterpreter('''
+        function x() {
+            if (0!=0) {return 1}
+            else if (1==0) {return 2}
+            else {return 10}
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+        """
+
     def test_for_loop(self):
         jsi = JSInterpreter('''
         function x() { a=0; for (i=0; i-10; i++) {a++} return a }
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 6d753fbf0..3203538bb 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -134,6 +134,10 @@
         'https://www.youtube.com/s/player/7a062b77/player_ias.vflset/en_US/base.js',
         'NRcE3y3mVtm_cV-W', 'VbsCYUATvqlt5w',
     ),
+    (
+        'https://www.youtube.com/s/player/dac945fd/player_ias.vflset/en_US/base.js',
+        'o8BkRxXhuYsBCWi6RplPdP', '3Lx32v_hmzTm6A',
+    ),
 ]
 
 
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index 3f7d659ac..c2d056aa1 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -403,10 +403,25 @@ def dict_item(key, val):
 
         m = re.match(r'''(?x)
                 (?P<try>try)\s*\{|
+                (?P<if>if)\s*\(|
                 (?P<switch>switch)\s*\(|
                 (?P<for>for)\s*\(
                 ''', expr)
         md = m.groupdict() if m else {}
+        if md.get('if'):
+            cndn, expr = self._separate_at_paren(expr[m.end() - 1:])
+            if_expr, expr = self._separate_at_paren(expr.lstrip())
+            # TODO: "else if" is not handled
+            else_expr = None
+            m = re.match(r'else\s*{', expr)
+            if m:
+                else_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+            cndn = _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion))
+            ret, should_abort = self.interpret_statement(
+                if_expr if cndn else else_expr, local_vars, allow_recursion)
+            if should_abort:
+                return ret, True
+
         if md.get('try'):
             try_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
             err = None

From 776995bc109c5cd1aa56b684fada2ce718a386ec Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Thu, 2 Feb 2023 06:40:19 +0100
Subject: [PATCH 008/871] [utils] `traverse_obj`:  Various improvements

- Add `set` key for transformations/filters
- Add `re.Match` group names
- Fix behavior for `expected_type` with `dict` key
- Raise for filter function signature mismatch in debug

Authored by: Grub4K
---
 test/test_utils.py | 40 ++++++++++++++++++++++++++++++++
 yt_dlp/utils.py    | 58 ++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 88 insertions(+), 10 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3d5a6ea6b..ffe1b729f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -105,6 +105,7 @@
     sanitized_Request,
     shell_quote,
     smuggle_url,
+    str_or_none,
     str_to_int,
     strip_jsonp,
     strip_or_none,
@@ -2015,6 +2016,29 @@ def test_traverse_obj(self):
                          msg='function as query key should perform a filter based on (key, value)')
         self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), {'str'},
                               msg='exceptions in the query function should be catched')
+        if __debug__:
+            with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
+                traverse_obj(_TEST_DATA, lambda a: ...)
+            with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
+                traverse_obj(_TEST_DATA, lambda a, b, c: ...)
+
+        # Test set as key (transformation/type, like `expected_type`)
+        self.assertEqual(traverse_obj(_TEST_DATA, (..., {str.upper}, )), ['STR'],
+                         msg='Function in set should be a transformation')
+        self.assertEqual(traverse_obj(_TEST_DATA, (..., {str})), ['str'],
+                         msg='Type in set should be a type filter')
+        self.assertEqual(traverse_obj(_TEST_DATA, {dict}), _TEST_DATA,
+                         msg='A single set should be wrapped into a path')
+        self.assertEqual(traverse_obj(_TEST_DATA, (..., {str.upper})), ['STR'],
+                         msg='Transformation function should not raise')
+        self.assertEqual(traverse_obj(_TEST_DATA, (..., {str_or_none})),
+                         [item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
+                         msg='Function in set should be a transformation')
+        if __debug__:
+            with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
+                traverse_obj(_TEST_DATA, set())
+            with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
+                traverse_obj(_TEST_DATA, {str.upper, str})
 
         # Test alternative paths
         self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str',
@@ -2106,6 +2130,20 @@ def test_traverse_obj(self):
                          msg='wrap expected_type fuction in try_call')
         self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, ..., expected_type=str), ['str'],
                          msg='eliminate items that expected_type fails on')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}, expected_type=int), {0: 100},
+                         msg='type as expected_type should filter dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2, 2: 'None'}, expected_type=str_or_none), {0: '100', 1: '1.2'},
+                         msg='function as expected_type should transform dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, {int_or_none}), expected_type=int), 1,
+                         msg='expected_type should not filter non final dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 100, 1: 'str'}}, expected_type=int), {0: {0: 100}},
+                         msg='expected_type should transform deep dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, [({0: '...'}, {0: '...'})], expected_type=type(...)), [{0: ...}, {0: ...}],
+                         msg='expected_type should transform branched dict values')
+        self.assertEqual(traverse_obj({1: {3: 4}}, [(1, 2), 3], expected_type=int), [4],
+                         msg='expected_type regression for type matching in tuple branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, ['data', ...], expected_type=int), [],
+                         msg='expected_type regression for type matching in dict result')
 
         # Test get_all behavior
         _GET_ALL_DATA = {'key': [0, 1, 2]}
@@ -2189,6 +2227,8 @@ def test_traverse_obj(self):
                          msg='failing str key on a `re.Match` should return `default`')
         self.assertEqual(traverse_obj(mobj, 8), None,
                          msg='failing int key on a `re.Match` should return `default`')
+        self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 'group')), ['0123', '3'],
+                         msg='function on a `re.Match` should give group name as well')
 
 
 if __name__ == '__main__':
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 7d51fe472..55e1c4415 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5424,6 +5424,9 @@ def traverse_obj(
 
     The keys in the path can be one of:
         - `None`:           Return the current object.
+        - `set`:            Requires the only item in the set to be a type or function,
+                            like `{type}`/`{func}`. If a `type`, returns only values
+                            of this type. If a function, returns `func(obj)`.
         - `str`/`int`:      Return `obj[key]`. For `re.Match`, return `obj.group(key)`.
         - `slice`:          Branch out and return all values in `obj[key]`.
         - `Ellipsis`:       Branch out and return a list of all values.
@@ -5432,6 +5435,8 @@ def traverse_obj(
         - `function`:       Branch out and return values filtered by the function.
                             Read as: `[value for key, value in obj if function(key, value)]`.
                             For `Sequence`s, `key` is the index of the value.
+                            For `re.Match`es, `key` is the group number (0 = full match)
+                            as well as additionally any group names, if given.
         - `dict`            Transform the current object and return a matching dict.
                             Read as: `{key: traverse_obj(obj, path) for key, path in dct.items()}`.
 
@@ -5441,6 +5446,8 @@ def traverse_obj(
     @param default          Value to return if the paths do not match.
     @param expected_type    If a `type`, only accept final values of this type.
                             If any other callable, try to call the function on each result.
+                            If the last key in the path is a `dict`, it will apply to each value inside
+                            the dict instead, recursively. This does respect branching paths.
     @param get_all          If `False`, return the first matching result, otherwise all matching ones.
     @param casesense        If `False`, consider string dictionary keys as case insensitive.
 
@@ -5466,16 +5473,25 @@ def traverse_obj(
     else:
         type_test = lambda val: try_call(expected_type or IDENTITY, args=(val,))
 
-    def apply_key(key, obj):
+    def apply_key(key, test_type, obj):
         if obj is None:
             return
 
         elif key is None:
             yield obj
 
+        elif isinstance(key, set):
+            assert len(key) == 1, 'Set should only be used to wrap a single item'
+            item = next(iter(key))
+            if isinstance(item, type):
+                if isinstance(obj, item):
+                    yield obj
+            else:
+                yield try_call(item, args=(obj,))
+
         elif isinstance(key, (list, tuple)):
             for branch in key:
-                _, result = apply_path(obj, branch)
+                _, result = apply_path(obj, branch, test_type)
                 yield from result
 
         elif key is ...:
@@ -5494,7 +5510,9 @@ def apply_key(key, obj):
             elif isinstance(obj, collections.abc.Mapping):
                 iter_obj = obj.items()
             elif isinstance(obj, re.Match):
-                iter_obj = enumerate((obj.group(), *obj.groups()))
+                iter_obj = itertools.chain(
+                    enumerate((obj.group(), *obj.groups())),
+                    obj.groupdict().items())
             elif traverse_string:
                 iter_obj = enumerate(str(obj))
             else:
@@ -5502,7 +5520,7 @@ def apply_key(key, obj):
             yield from (v for k, v in iter_obj if try_call(key, args=(k, v)))
 
         elif isinstance(key, dict):
-            iter_obj = ((k, _traverse_obj(obj, v)) for k, v in key.items())
+            iter_obj = ((k, _traverse_obj(obj, v, test_type=test_type)) for k, v in key.items())
             yield {k: v if v is not None else default for k, v in iter_obj
                    if v is not None or default is not NO_DEFAULT}
 
@@ -5537,11 +5555,24 @@ def apply_key(key, obj):
             with contextlib.suppress(IndexError):
                 yield obj[key]
 
-    def apply_path(start_obj, path):
+    def lazy_last(iterable):
+        iterator = iter(iterable)
+        prev = next(iterator, NO_DEFAULT)
+        if prev is NO_DEFAULT:
+            return
+
+        for item in iterator:
+            yield False, prev
+            prev = item
+
+        yield True, prev
+
+    def apply_path(start_obj, path, test_type=False):
         objs = (start_obj,)
         has_branched = False
 
-        for key in variadic(path):
+        key = None
+        for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
             if is_user_input and key == ':':
                 key = ...
 
@@ -5551,14 +5582,21 @@ def apply_path(start_obj, path):
             if key is ... or isinstance(key, (list, tuple)) or callable(key):
                 has_branched = True
 
-            key_func = functools.partial(apply_key, key)
+            if __debug__ and callable(key):
+                # Verify function signature
+                inspect.signature(key).bind(None, None)
+
+            key_func = functools.partial(apply_key, key, last)
             objs = itertools.chain.from_iterable(map(key_func, objs))
 
+        if test_type and not isinstance(key, (dict, list, tuple)):
+            objs = map(type_test, objs)
+
         return has_branched, objs
 
-    def _traverse_obj(obj, path, use_list=True):
-        has_branched, results = apply_path(obj, path)
-        results = LazyList(x for x in map(type_test, results) if x is not None)
+    def _traverse_obj(obj, path, use_list=True, test_type=True):
+        has_branched, results = apply_path(obj, path, test_type)
+        results = LazyList(x for x in results if x is not None)
 
         if get_all and has_branched:
             return results.exhaust() if results or use_list else None

From acacb57c7e173b93c6e0f0c43e61b9b2912719d8 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Thu, 2 Feb 2023 06:50:42 +0100
Subject: [PATCH 009/871] [extractor/rumble] Fix format sorting

Closes #6119
Authored by: pukkandan
---
 yt_dlp/extractor/rumble.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py
index b7f798ffb..97f81446c 100644
--- a/yt_dlp/extractor/rumble.py
+++ b/yt_dlp/extractor/rumble.py
@@ -186,7 +186,7 @@ def _real_extract(self, url):
                         'filesize': 'size',
                         'width': 'w',
                         'height': 'h',
-                    }, default={})
+                    }, expected_type=lambda x: int(x) or None)
                 })
 
         subtitles = {

From 7543c9c99bcb116b085fdb1f41b84a0ead04c05d Mon Sep 17 00:00:00 2001
From: "lauren n. liberda" <lauren@selfisekai.rocks>
Date: Thu, 2 Feb 2023 14:32:14 +0100
Subject: [PATCH 010/871] [extractor/twitter] Fix graphql extraction on some
 tweets (#6075)

Authored by: selfisekai
---
 yt_dlp/extractor/twitter.py | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index a4e280c82..d3e52f392 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -769,6 +769,29 @@ class TwitterIE(TwitterBaseIE):
             'age_limit': 0,
         },
         'params': {'noplaylist': True},
+    }, {
+        # id pointing to TweetWithVisibilityResults type entity which wraps the actual Tweet over
+        # note the id different between extraction and url
+        'url': 'https://twitter.com/s2FAKER/status/1621117700482416640',
+        'info_dict': {
+            'id': '1621117577354424321',
+            'display_id': '1621117700482416640',
+            'ext': 'mp4',
+            'title': '뽀 - 아 최우제 이동속도 봐',
+            'description': '아 최우제 이동속도 봐 https://t.co/dxu2U5vXXB',
+            'duration': 24.598,
+            'uploader': '뽀',
+            'uploader_id': 's2FAKER',
+            'uploader_url': 'https://twitter.com/s2FAKER',
+            'upload_date': '20230202',
+            'timestamp': 1675339553.0,
+            'thumbnail': r're:https?://pbs\.twimg\.com/.+',
+            'age_limit': 18,
+            'tags': [],
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+        },
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -811,9 +834,12 @@ def _graphql_to_legacy(self, data, twid):
         result = traverse_obj(data, (
             'threaded_conversation_with_injections_v2', 'instructions', 0, 'entries',
             lambda _, v: v['entryId'] == f'tweet-{twid}', 'content', 'itemContent',
-            'tweet_results', 'result'
+            'tweet_results', 'result', ('tweet', None),
         ), expected_type=dict, default={}, get_all=False)
 
+        if result.get('__typename') not in ('Tweet', None):
+            self.report_warning(f'Unknown typename: {result.get("__typename")}', twid, only_once=True)
+
         if 'tombstone' in result:
             cause = traverse_obj(result, ('tombstone', 'text', 'text'), expected_type=str)
             raise ExtractorError(f'Twitter API says: {cause or "Unknown error"}', expected=True)

From 9cfdbcbf3f17be51f5b6bb9bb6d880b2f3d67362 Mon Sep 17 00:00:00 2001
From: Jasper Rebane <rebane2001@gmail.com>
Date: Fri, 3 Feb 2023 16:38:51 +0200
Subject: [PATCH 011/871] [extractor/freesound] Workaround invalid URL in
 webpage (#6147)

Authored by: rebane2001
Closes #6146
---
 yt_dlp/extractor/freesound.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/yt_dlp/extractor/freesound.py b/yt_dlp/extractor/freesound.py
index 8b5f2278c..fcde04469 100644
--- a/yt_dlp/extractor/freesound.py
+++ b/yt_dlp/extractor/freesound.py
@@ -52,6 +52,7 @@ def _real_extract(self, url):
         tags_str = get_element_by_class('tags', webpage)
         tags = re.findall(r'<a[^>]+>([^<]+)', tags_str) if tags_str else None
 
+        audio_url = re.sub(r'^https?://freesound\.org(https?://)', r'\1', audio_url)
         audio_urls = [audio_url]
 
         LQ_FORMAT = '-lq.mp3'

From dad2210c0cb9cf03702a9511817ee5ec646d7bc8 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 3 Feb 2023 23:47:13 +0530
Subject: [PATCH 012/871] [extractor/youtube] Support `/live/` URL

---
 yt_dlp/extractor/youtube.py | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 855a76012..3d4c496ba 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -1012,7 +1012,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
                          (?:                                                  # the various things that can precede the ID:
-                             (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
+                             (?:(?:v|embed|e|shorts|live)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
                              |(?:                                             # or the v= param in all its forms
                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
@@ -2573,7 +2573,38 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 106,
             },
             'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
-        }
+        },
+        {
+            'url': 'https://www.youtube.com/live/qVv6vCqciTM',
+            'info_dict': {
+                'id': 'qVv6vCqciTM',
+                'ext': 'mp4',
+                'age_limit': 0,
+                'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
+                'comment_count': int,
+                'chapters': 'count:13',
+                'upload_date': '20221223',
+                'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
+                'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
+                'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
+                'like_count': int,
+                'release_date': '20221223',
+                'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
+                'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
+                'view_count': int,
+                'playable_in_embed': True,
+                'duration': 4438,
+                'availability': 'public',
+                'channel_follower_count': int,
+                'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
+                'categories': ['Entertainment'],
+                'live_status': 'was_live',
+                'release_timestamp': 1671793345,
+                'channel': 'さなちゃんねる',
+                'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
+                'uploader': 'さなちゃんねる',
+            },
+        },
     ]
 
     _WEBPAGE_TESTS = [

From b032ff0f032512bd6fc70c9c1994d906eacc06cb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 3 Feb 2023 23:53:35 +0530
Subject: [PATCH 013/871] [extractor/youtube] Handle `consent.youtube`

---
 yt_dlp/extractor/_extractors.py |  3 ++-
 yt_dlp/extractor/youtube.py     | 47 ++++++++++++++++++++++++++++++++-
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index dc5e50e2f..62d652f27 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -21,7 +21,8 @@
     YoutubeYtBeIE,
     YoutubeYtUserIE,
     YoutubeWatchLaterIE,
-    YoutubeShortsAudioPivotIE
+    YoutubeShortsAudioPivotIE,
+    YoutubeConsentRedirectIE,
 )
 
 from .abc import (
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 3d4c496ba..f7b0772df 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -5139,7 +5139,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
     IE_DESC = 'YouTube Tabs'
     _VALID_URL = r'''(?x:
         https?://
-            (?:\w+\.)?
+            (?!consent\.)(?:\w+\.)?
             (?:
                 youtube(?:kids)?\.com|
                 %(invidious)s
@@ -6949,6 +6949,51 @@ def _real_extract(self, url):
         }
 
 
+class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
+    IE_NAME = 'youtube:consent'
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'https?://consent\.youtube\.com/m\?'
+    _TESTS = [{
+        'url': 'https://consent.youtube.com/m?continue=https%3A%2F%2Fwww.youtube.com%2Flive%2FqVv6vCqciTM%3Fcbrd%3D1&gl=NL&m=0&pc=yt&hl=en&src=1',
+        'info_dict': {
+            'id': 'qVv6vCqciTM',
+            'ext': 'mp4',
+            'age_limit': 0,
+            'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
+            'comment_count': int,
+            'chapters': 'count:13',
+            'upload_date': '20221223',
+            'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
+            'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
+            'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
+            'like_count': int,
+            'release_date': '20221223',
+            'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
+            'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
+            'view_count': int,
+            'playable_in_embed': True,
+            'duration': 4438,
+            'availability': 'public',
+            'channel_follower_count': int,
+            'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
+            'categories': ['Entertainment'],
+            'live_status': 'was_live',
+            'release_timestamp': 1671793345,
+            'channel': 'さなちゃんねる',
+            'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
+            'uploader': 'さなちゃんねる',
+        },
+        'add_ie': ['Youtube'],
+        'params': {'skip_download': 'Youtube'},
+    }]
+
+    def _real_extract(self, url):
+        redirect_url = url_or_none(parse_qs(url).get('continue', [None])[-1])
+        if not redirect_url:
+            raise ExtractorError('Invalid cookie consent redirect URL', expected=True)
+        return self.url_result(redirect_url)
+
+
 class YoutubeTruncatedIDIE(InfoExtractor):
     IE_NAME = 'youtube:truncated_id'
     IE_DESC = False  # Do not list

From 389896df85ed14eaf74f72531da6c4491d6b73b0 Mon Sep 17 00:00:00 2001
From: chio0hai <94094996+chio0hai@users.noreply.github.com>
Date: Fri, 3 Feb 2023 13:47:00 -0500
Subject: [PATCH 014/871] [extractor/txxx] Add extractors (#5240)

Authored by: chio0hai
Closes #5021
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/generic.py     |  14 --
 yt_dlp/extractor/txxx.py        | 418 ++++++++++++++++++++++++++++++++
 yt_dlp/utils.py                 |   2 +
 4 files changed, 424 insertions(+), 14 deletions(-)
 create mode 100644 yt_dlp/extractor/txxx.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 62d652f27..a67c39479 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2048,6 +2048,10 @@
     TwitterSpacesIE,
     TwitterShortenerIE,
 )
+from .txxx import (
+    TxxxIE,
+    PornTopIE,
+)
 from .udemy import (
     UdemyIE,
     UdemyCourseIE
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 9e4df4cea..55e55d524 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -864,20 +864,6 @@ class GenericIE(InfoExtractor):
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
-        {
-            # JWPlayer config passed as variable
-            'url': 'http://www.txxx.com/videos/3326530/ariele/',
-            'info_dict': {
-                'id': '3326530_hq',
-                'ext': 'mp4',
-                'title': 'ARIELE | Tube Cup',
-                'uploader': 'www.txxx.com',
-                'age_limit': 18,
-            },
-            'params': {
-                'skip_download': True,
-            }
-        },
         {
             # Video.js embed, multiple formats
             'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
diff --git a/yt_dlp/extractor/txxx.py b/yt_dlp/extractor/txxx.py
new file mode 100644
index 000000000..fff7a5d76
--- /dev/null
+++ b/yt_dlp/extractor/txxx.py
@@ -0,0 +1,418 @@
+import base64
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    js_to_json,
+    merge_dicts,
+    parse_duration,
+    traverse_obj,
+    try_call,
+    urljoin,
+    variadic,
+)
+
+
+def decode_base64(text):
+    return base64.b64decode(text.translate(text.maketrans({
+        '\u0405': 'S',
+        '\u0406': 'I',
+        '\u0408': 'J',
+        '\u0410': 'A',
+        '\u0412': 'B',
+        '\u0415': 'E',
+        '\u041a': 'K',
+        '\u041c': 'M',
+        '\u041d': 'H',
+        '\u041e': 'O',
+        '\u0420': 'P',
+        '\u0421': 'C',
+        '\u0425': 'X',
+        ',': '/',
+        '.': '+',
+        '~': '=',
+    }))).decode()
+
+
+def get_formats(host, video_file):
+    return [{
+        'url': urljoin(f'https://{host}', decode_base64(video['video_url'])),
+        'format_id': try_call(lambda: variadic(video['format'])[0].lstrip('_')),
+        'quality': index,
+    } for index, video in enumerate(video_file) if video.get('video_url')]
+
+
+class TxxxIE(InfoExtractor):
+    _DOMAINS = (
+        'hclips.com',
+        'hdzog.com',
+        'hdzog.tube',
+        'hotmovs.com',
+        'hotmovs.tube',
+        'inporn.com',
+        'privatehomeclips.com',
+        'tubepornclassic.com',
+        'txxx.com',
+        'txxx.tube',
+        'upornia.com',
+        'upornia.tube',
+        'vjav.com',
+        'vjav.tube',
+        'vxxx.com',
+        'voyeurhit.com',
+        'voyeurhit.tube',
+    )
+    _VALID_URL = rf'''(?x)
+        https?://(?:www\.)?(?P<host>{"|".join(map(re.escape, _DOMAINS))})/
+        (?:videos?[/-]|embed/)(?P<id>\d+)(?:/(?P<display_id>[^/?#]+))?
+    '''
+    _EMBED_REGEX = [rf'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?(?:{"|".join(map(re.escape, _DOMAINS))})/embed/[^"\']*)\1']
+    _TESTS = [{
+        'url': 'https://txxx.com/videos/16574965/digital-desire-malena-morgan/',
+        'md5': 'c54e4ace54320aaf8e2a72df87859391',
+        'info_dict': {
+            'id': '16574965',
+            'display_id': 'digital-desire-malena-morgan',
+            'ext': 'mp4',
+            'title': 'Digital Desire - Malena Morgan',
+            'uploader': 'Lois Argentum',
+            'duration': 694,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://txxx.tube/videos/16574965/digital-desire-malena-morgan/',
+        'md5': 'c54e4ace54320aaf8e2a72df87859391',
+        'info_dict': {
+            'id': '16574965',
+            'display_id': 'digital-desire-malena-morgan',
+            'ext': 'mp4',
+            'title': 'Digital Desire - Malena Morgan',
+            'uploader': 'Lois Argentum',
+            'duration': 694,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://vxxx.com/video-68925/',
+        'md5': '1fcff3748b0c5b41fe41d0afa22409e1',
+        'info_dict': {
+            'id': '68925',
+            'display_id': '68925',
+            'ext': 'mp4',
+            'title': 'Malena Morgan',
+            'uploader': 'Huge Hughes',
+            'duration': 694,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://hclips.com/videos/6291073/malena-morgan-masturbates-her-sweet/',
+        'md5': 'a5dd4f83363972ee043313cff85e7e26',
+        'info_dict': {
+            'id': '6291073',
+            'display_id': 'malena-morgan-masturbates-her-sweet',
+            'ext': 'mp4',
+            'title': 'Malena Morgan masturbates her sweet',
+            'uploader': 'John Salt',
+            'duration': 426,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://hdzog.com/videos/67063/gorgeous-malena-morgan-will-seduce-you-at-the-first-glance/',
+        'md5': 'f8bdedafd45d1ec2875c43fe33a846d3',
+        'info_dict': {
+            'id': '67063',
+            'display_id': 'gorgeous-malena-morgan-will-seduce-you-at-the-first-glance',
+            'ext': 'mp4',
+            'title': 'Gorgeous Malena Morgan will seduce you at the first glance',
+            'uploader': 'momlesson',
+            'duration': 601,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://hdzog.tube/videos/67063/gorgeous-malena-morgan-will-seduce-you-at-the-first-glance/',
+        'md5': 'f8bdedafd45d1ec2875c43fe33a846d3',
+        'info_dict': {
+            'id': '67063',
+            'display_id': 'gorgeous-malena-morgan-will-seduce-you-at-the-first-glance',
+            'ext': 'mp4',
+            'title': 'Gorgeous Malena Morgan will seduce you at the first glance',
+            'uploader': 'momlesson',
+            'duration': 601,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://hotmovs.com/videos/8789287/unbelievable-malena-morgan-performing-in-incredible-masturantion/',
+        'md5': '71d32c51584876472db87e561171a386',
+        'info_dict': {
+            'id': '8789287',
+            'display_id': 'unbelievable-malena-morgan-performing-in-incredible-masturantion',
+            'ext': 'mp4',
+            'title': 'Unbelievable Malena Morgan performing in incredible masturantion',
+            'uploader': 'Davit Sanchez',
+            'duration': 940,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://hotmovs.tube/videos/8789287/unbelievable-malena-morgan-performing-in-incredible-masturantion/',
+        'md5': '71d32c51584876472db87e561171a386',
+        'info_dict': {
+            'id': '8789287',
+            'display_id': 'unbelievable-malena-morgan-performing-in-incredible-masturantion',
+            'ext': 'mp4',
+            'title': 'Unbelievable Malena Morgan performing in incredible masturantion',
+            'uploader': 'Davit Sanchez',
+            'duration': 940,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://inporn.com/video/517897/malena-morgan-solo/',
+        'md5': '344db467481edf78f193cdf5820a7cfb',
+        'info_dict': {
+            'id': '517897',
+            'display_id': 'malena-morgan-solo',
+            'ext': 'mp4',
+            'title': 'Malena Morgan - Solo',
+            'uploader': 'Ashley Oxy',
+            'duration': 480,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://privatehomeclips.com/videos/3630599/malena-morgan-cam-show/',
+        'md5': 'ea657273e352493c5fb6357fbfa4f126',
+        'info_dict': {
+            'id': '3630599',
+            'display_id': 'malena-morgan-cam-show',
+            'ext': 'mp4',
+            'title': 'malena morgan cam show',
+            'uploader': 'Member9915',
+            'duration': 290,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://tubepornclassic.com/videos/1015455/mimi-rogers-full-body-massage-nude-compilation/',
+        'md5': '2e9a6cf610c9862e86e0ce24f08f4427',
+        'info_dict': {
+            'id': '1015455',
+            'display_id': 'mimi-rogers-full-body-massage-nude-compilation',
+            'ext': 'mp4',
+            'title': 'Mimi Rogers - Full Body Massage (Nude) compilation',
+            'uploader': '88bhuto',
+            'duration': 286,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://upornia.com/videos/1498858/twistys-malena-morgan-starring-at-dr-morgan-baller/',
+        'md5': '7ff7033340bc88a173198b7c22600e4f',
+        'info_dict': {
+            'id': '1498858',
+            'display_id': 'twistys-malena-morgan-starring-at-dr-morgan-baller',
+            'ext': 'mp4',
+            'title': 'Twistys - Malena Morgan starring at Dr. Morgan-Baller',
+            'uploader': 'mindgeek',
+            'duration': 480,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://upornia.tube/videos/1498858/twistys-malena-morgan-starring-at-dr-morgan-baller/',
+        'md5': '7ff7033340bc88a173198b7c22600e4f',
+        'info_dict': {
+            'id': '1498858',
+            'display_id': 'twistys-malena-morgan-starring-at-dr-morgan-baller',
+            'ext': 'mp4',
+            'title': 'Twistys - Malena Morgan starring at Dr. Morgan-Baller',
+            'uploader': 'mindgeek',
+            'duration': 480,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://vjav.com/videos/11761/yui-hatano-in-if-yui-was-my-girlfriend2/',
+        'md5': '6de5bc1f13bdfc3491a77f23edb1676f',
+        'info_dict': {
+            'id': '11761',
+            'display_id': 'yui-hatano-in-if-yui-was-my-girlfriend2',
+            'ext': 'mp4',
+            'title': 'Yui Hatano in If Yui Was My Girlfriend',
+            'uploader': 'Matheus69',
+            'duration': 3310,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://vjav.tube/videos/11761/yui-hatano-in-if-yui-was-my-girlfriend2/',
+        'md5': '6de5bc1f13bdfc3491a77f23edb1676f',
+        'info_dict': {
+            'id': '11761',
+            'display_id': 'yui-hatano-in-if-yui-was-my-girlfriend2',
+            'ext': 'mp4',
+            'title': 'Yui Hatano in If Yui Was My Girlfriend',
+            'uploader': 'Matheus69',
+            'duration': 3310,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://voyeurhit.com/videos/332875/charlotte-stokely-elle-alexandra-malena-morgan-lingerie/',
+        'md5': '12b4666e9c3e60dafe9182e5d12aae33',
+        'info_dict': {
+            'id': '332875',
+            'display_id': 'charlotte-stokely-elle-alexandra-malena-morgan-lingerie',
+            'ext': 'mp4',
+            'title': 'Charlotte Stokely, Elle Alexandra, Malena Morgan-Lingerie',
+            'uploader': 'Kyle Roberts',
+            'duration': 655,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://voyeurhit.tube/videos/332875/charlotte-stokely-elle-alexandra-malena-morgan-lingerie/',
+        'md5': '12b4666e9c3e60dafe9182e5d12aae33',
+        'info_dict': {
+            'id': '332875',
+            'display_id': 'charlotte-stokely-elle-alexandra-malena-morgan-lingerie',
+            'ext': 'mp4',
+            'title': 'Charlotte Stokely, Elle Alexandra, Malena Morgan-Lingerie',
+            'uploader': 'Kyle Roberts',
+            'duration': 655,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://pornzog.com/video/9125519/michelle-malone-dreamgirls-wild-wet-3/',
+        'info_dict': {
+            'id': '5119660',
+            'display_id': '5119660',
+            'ext': 'mp4',
+            'title': 'Michelle Malone - Dreamgirls - Wild Wet 3',
+            'uploader': 'FallenAngel12',
+            'duration': 402,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+        }
+    }]
+
+    def _call_api(self, url, video_id, fatal=False, **kwargs):
+        content = self._download_json(url, video_id, fatal=fatal, **kwargs)
+        if traverse_obj(content, 'error'):
+            raise self._error_or_warning(ExtractorError(
+                f'Txxx said: {content["error"]}', expected=True), fatal=fatal)
+        return content or {}
+
+    def _real_extract(self, url):
+        video_id, host, display_id = self._match_valid_url(url).group('id', 'host', 'display_id')
+        headers = {'Referer': url, 'X-Requested-With': 'XMLHttpRequest'}
+
+        video_file = self._call_api(
+            f'https://{host}/api/videofile.php?video_id={video_id}&lifetime=8640000',
+            video_id, fatal=True, note='Downloading video file info', headers=headers)
+
+        slug = f'{int(1E6 * (int(video_id) // 1E6))}/{1000 * (int(video_id) // 1000)}'
+        video_info = self._call_api(
+            f'https://{host}/api/json/video/86400/{slug}/{video_id}.json',
+            video_id, note='Downloading video info', headers=headers)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': traverse_obj(video_info, ('video', 'title')),
+            'uploader': traverse_obj(video_info, ('video', 'user', 'username')),
+            'duration': parse_duration(traverse_obj(video_info, ('video', 'duration'))),
+            'view_count': int_or_none(traverse_obj(video_info, ('video', 'statistics', 'viewed'))),
+            'like_count': int_or_none(traverse_obj(video_info, ('video', 'statistics', 'likes'))),
+            'dislike_count': int_or_none(traverse_obj(video_info, ('video', 'statistics', 'dislikes'))),
+            'age_limit': 18,
+            'formats': get_formats(host, video_file),
+        }
+
+
+class PornTopIE(InfoExtractor):
+    _VALID_URL = r'https?://(?P<host>(?:www\.)?porntop\.com)/video/(?P<id>\d+)(?:/(?P<display_id>[^/?]+))?'
+    _TESTS = [{
+        'url': 'https://porntop.com/video/101569/triple-threat-with-lia-lor-malena-morgan-and-dani-daniels/',
+        'md5': '612ba7b3cb99455b382972948e200b08',
+        'info_dict': {
+            'id': '101569',
+            'display_id': 'triple-threat-with-lia-lor-malena-morgan-and-dani-daniels',
+            'ext': 'mp4',
+            'title': 'Triple Threat With Lia Lor, Malena Morgan And Dani Daniels',
+            'description': 'md5:285357d9d3a00ce5acb29f39f826dbf6',
+            'uploader': 'PatrickBush',
+            'duration': 480,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+            'timestamp': 1609455029,
+            'upload_date': '20201231',
+            'thumbnail': 'https://tn.porntop.com/media/tn/sources/101569_1.jpg',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id, host, display_id = self._match_valid_url(url).group('id', 'host', 'display_id')
+        webpage = self._download_webpage(url, video_id)
+
+        json_ld = self._json_ld(self._search_json(
+            r'\bschemaJson\s*=', webpage, 'JSON-LD', video_id, transform_source=js_to_json,
+            contains_pattern='{[^<]+?VideoObject[^<]+};'), video_id, fatal=True)
+
+        video_file = self._parse_json(decode_base64(self._search_regex(
+            r"window\.initPlayer\(.*}}},\s*'(?P<json_b64c>[^']+)'",
+            webpage, 'json_urls', group='json_b64c')), video_id)
+
+        return merge_dicts({
+            'id': video_id,
+            'display_id': display_id,
+            'age_limit': 18,
+            'formats': get_formats(host, video_file),
+        }, json_ld)
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 55e1c4415..e1e0f7b25 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3385,6 +3385,8 @@ def create_map(mobj):
     if not strict:
         code = re.sub(r'new Date\((".+")\)', r'\g<1>', code)
         code = re.sub(r'new \w+\((.*?)\)', lambda m: json.dumps(m.group(0)), code)
+        code = re.sub(r'parseInt\([^\d]+(\d+)[^\d]+\)', r'\1', code)
+        code = re.sub(r'\(function\([^)]*\)\s*\{[^}]*\}\s*\)\s*\(\s*(["\'][^)]*["\'])\s*\)', r'\1', code)
 
     return re.sub(rf'''(?sx)
         {STRING_RE}|

From 3b161265add30613bde2e46fca214fe94d09e651 Mon Sep 17 00:00:00 2001
From: Matumo <dev@matumo.com>
Date: Sat, 4 Feb 2023 03:50:06 +0900
Subject: [PATCH 015/871] [extractor/niconico] Add support for like history
 (#5705)

Authored by: Matumo, pukkandan
---
 yt_dlp/extractor/niconico.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py
index 210303759..9c3a5a4bc 100644
--- a/yt_dlp/extractor/niconico.py
+++ b/yt_dlp/extractor/niconico.py
@@ -675,8 +675,8 @@ def _real_extract(self, url):
 
 class NiconicoHistoryIE(NiconicoPlaylistBaseIE):
     IE_NAME = 'niconico:history'
-    IE_DESC = 'NicoNico user history. Requires cookies.'
-    _VALID_URL = r'https?://(?:www\.|sp\.)?nicovideo\.jp/my/history'
+    IE_DESC = 'NicoNico user history or likes. Requires cookies.'
+    _VALID_URL = r'https?://(?:www\.|sp\.)?nicovideo\.jp/my/(?P<id>history(?:/like)?)'
 
     _TESTS = [{
         'note': 'PC page, with /video',
@@ -694,23 +694,29 @@ class NiconicoHistoryIE(NiconicoPlaylistBaseIE):
         'note': 'mobile page, without /video',
         'url': 'https://sp.nicovideo.jp/my/history',
         'only_matching': True,
+    }, {
+        'note': 'PC page',
+        'url': 'https://www.nicovideo.jp/my/history/like',
+        'only_matching': True,
+    }, {
+        'note': 'Mobile page',
+        'url': 'https://sp.nicovideo.jp/my/history/like',
+        'only_matching': True,
     }]
 
     def _call_api(self, list_id, resource, query):
+        path = 'likes' if list_id == 'history/like' else 'watch/history'
         return self._download_json(
-            'https://nvapi.nicovideo.jp/v1/users/me/watch/history', 'history',
-            f'Downloading {resource}', query=query,
-            headers=self._API_HEADERS)['data']
+            f'https://nvapi.nicovideo.jp/v1/users/me/{path}', list_id,
+            f'Downloading {resource}', query=query, headers=self._API_HEADERS)['data']
 
     def _real_extract(self, url):
-        list_id = 'history'
+        list_id = self._match_id(url)
         try:
-            mylist = self._call_api(list_id, 'list', {
-                'pageSize': 1,
-            })
+            mylist = self._call_api(list_id, 'list', {'pageSize': 1})
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                self.raise_login_required('You have to be logged in to get your watch history')
+                self.raise_login_required('You have to be logged in to get your history')
             raise
         return self.playlist_result(self._entries(list_id), list_id, **self._parse_owner(mylist))
 

From 0fe87a8730638490415d630f48e61d264d89c358 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Fri, 3 Feb 2023 23:38:29 +0100
Subject: [PATCH 016/871] [extractor/zdf] Use android API endpoint for UHD
 downloads (#6150)

Authored by: seproDev
---
 yt_dlp/extractor/zdf.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/extractor/zdf.py b/yt_dlp/extractor/zdf.py
index fca426a50..c863c46ed 100644
--- a/yt_dlp/extractor/zdf.py
+++ b/yt_dlp/extractor/zdf.py
@@ -24,7 +24,7 @@
 
 class ZDFBaseIE(InfoExtractor):
     _GEO_COUNTRIES = ['DE']
-    _QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh', 'hd')
+    _QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh', 'hd', 'uhd')
 
     def _call_api(self, url, video_id, item, api_token=None, referrer=None):
         headers = {}
@@ -174,7 +174,8 @@ class ZDFIE(ZDFBaseIE):
             'thumbnail': 'md5:e65f459f741be5455c952cd820eb188e',
             'title': 'heute journal vom 30.12.2021',
             'timestamp': 1640897100,
-        }
+        },
+        'skip': 'No longer available: "Diese Seite wurde leider nicht gefunden"',
     }, {
         'url': 'https://www.zdf.de/dokumentation/terra-x/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100.html',
         'info_dict': {
@@ -189,7 +190,7 @@ class ZDFIE(ZDFBaseIE):
         },
     }, {
         'url': 'https://www.zdf.de/funk/druck-11790/funk-alles-ist-verzaubert-102.html',
-        'md5': '1b93bdec7d02fc0b703c5e7687461628',
+        'md5': '57af4423db0455a3975d2dc4578536bc',
         'info_dict': {
             'ext': 'mp4',
             'id': 'video_funk_1770473',
@@ -198,7 +199,7 @@ class ZDFIE(ZDFBaseIE):
             'title': 'Alles ist verzaubert',
             'timestamp': 1635520560,
             'upload_date': '20211029',
-            'thumbnail': 'https://www.zdf.de/assets/teaser-funk-alles-ist-verzaubert-100~1920x1080?cb=1636466431799',
+            'thumbnail': 'https://www.zdf.de/assets/teaser-funk-alles-ist-verzaubert-102~1920x1080?cb=1663848412907',
         },
     }, {
         # Same as https://www.phoenix.de/sendungen/dokumentationen/gesten-der-maechtigen-i-a-89468.html?ref=suche
@@ -241,10 +242,23 @@ class ZDFIE(ZDFBaseIE):
             'title': 'Das Geld anderer Leute',
             'description': 'md5:cb6f660850dc5eb7d1ab776ea094959d',
             'duration': 2581.0,
-            'timestamp': 1654790700,
-            'upload_date': '20220609',
+            'timestamp': 1675160100,
+            'upload_date': '20230131',
             'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/e2d7e55a-09f0-424e-ac73-6cac4dd65f35?layout=2400x1350',
         },
+    }, {
+        'url': 'https://www.zdf.de/dokumentation/terra-x/unser-gruener-planet-wuesten-doku-100.html',
+        'info_dict': {
+            'id': '220605_dk_gruener_planet_wuesten_tex',
+            'ext': 'mp4',
+            'title': 'Unser grüner Planet - Wüsten',
+            'description': 'md5:4fc647b6f9c3796eea66f4a0baea2862',
+            'duration': 2613.0,
+            'timestamp': 1654450200,
+            'upload_date': '20220605',
+            'format_note': 'uhd, main',
+            'thumbnail': 'https://www.zdf.de/assets/saguaro-kakteen-102~3840x2160?cb=1655910690796',
+        },
     }]
 
     def _extract_entry(self, url, player, content, video_id):
@@ -259,7 +273,7 @@ def _extract_entry(self, url, player, content, video_id):
             raise ExtractorError('Could not extract ptmd_path')
 
         info = self._extract_ptmd(
-            urljoin(url, ptmd_path.replace('{playerId}', 'ngplayer_2_4')), video_id, player['apiToken'], url)
+            urljoin(url, ptmd_path.replace('{playerId}', 'android_native_5')), video_id, player['apiToken'], url)
 
         thumbnails = []
         layouts = try_get(

From d27bde98832e3b7ffb39f3cf6346011b97bb3bc3 Mon Sep 17 00:00:00 2001
From: Jeroen Jacobs <git@jeroenj.be>
Date: Fri, 3 Feb 2023 23:42:43 +0100
Subject: [PATCH 017/871] [extractor/GoPlay] Use new API (#6151)

Authored by: jeroenj
Closes #6032
---
 yt_dlp/extractor/goplay.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/goplay.py b/yt_dlp/extractor/goplay.py
index 2882b49dd..960d7d7bc 100644
--- a/yt_dlp/extractor/goplay.py
+++ b/yt_dlp/extractor/goplay.py
@@ -76,11 +76,11 @@ def _real_extract(self, url):
             }
 
         api = self._download_json(
-            f'https://api.viervijfzes.be/content/{video_id}',
-            video_id, headers={'Authorization': self._id_token})
+            f'https://api.goplay.be/web/v1/videos/long-form/{video_id}',
+            video_id, headers={'Authorization': 'Bearer %s' % self._id_token})
 
         formats, subs = self._extract_m3u8_formats_and_subtitles(
-            api['video']['S'], video_id, ext='mp4', m3u8_id='HLS')
+            api['manifestUrls']['hls'], video_id, ext='mp4', m3u8_id='HLS')
 
         info_dict.update({
             'id': video_id,

From c77df98b1a477a020a57141464d10c0f4d0fdbc9 Mon Sep 17 00:00:00 2001
From: OMEGA_RAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Mon, 6 Feb 2023 08:51:39 -0500
Subject: [PATCH 018/871] [extractor/reddit] Support user posts (#6173)

Authored by: OMEGARAZER
---
 yt_dlp/extractor/reddit.py | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/reddit.py b/yt_dlp/extractor/reddit.py
index 836b3a7ae..9dba3eca8 100644
--- a/yt_dlp/extractor/reddit.py
+++ b/yt_dlp/extractor/reddit.py
@@ -14,7 +14,7 @@
 
 
 class RedditIE(InfoExtractor):
-    _VALID_URL = r'https?://(?P<subdomain>[^/]+\.)?reddit(?:media)?\.com/r/(?P<slug>[^/]+/comments/(?P<id>[^/?#&]+))'
+    _VALID_URL = r'https?://(?P<subdomain>[^/]+\.)?reddit(?:media)?\.com/(?P<slug>(?:r|user)/[^/]+/comments/(?P<id>[^/?#&]+))'
     _TESTS = [{
         'url': 'https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/',
         'info_dict': {
@@ -58,6 +58,29 @@ class RedditIE(InfoExtractor):
             'age_limit': 0,
             'channel_id': 'aww',
         },
+    }, {
+        # User post
+        'url': 'https://www.reddit.com/user/creepyt0es/comments/nip71r/i_plan_to_make_more_stickers_and_prints_check/',
+        'info_dict': {
+            'id': 'zasobba6wp071',
+            'ext': 'mp4',
+            'display_id': 'nip71r',
+            'title': 'I plan to make more stickers and prints! Check them out on my Etsy! Or get them through my Patreon. Links below.',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'thumbnails': 'count:5',
+            'timestamp': 1621709093,
+            'upload_date': '20210522',
+            'uploader': 'creepyt0es',
+            'duration': 6,
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'age_limit': 0,
+            'channel_id': 'u_creepyt0es',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         # videos embedded in reddit text post
         'url': 'https://www.reddit.com/r/KamenRider/comments/wzqkxp/finale_kamen_rider_revice_episode_50_family_to/',
@@ -84,6 +107,7 @@ class RedditIE(InfoExtractor):
             'dislike_count': int,
             'comment_count': int,
             'age_limit': 0,
+            'channel_id': 'dumbfuckers_club',
         },
     }, {
         'url': 'https://www.reddit.com/r/videos/comments/6rrwyj',
@@ -124,10 +148,10 @@ def _real_extract(self, url):
 
         self._set_cookie('.reddit.com', 'reddit_session', self._gen_session_id())
         self._set_cookie('.reddit.com', '_options', '%7B%22pref_quarantine_optin%22%3A%20true%7D')
-        data = self._download_json(f'https://{subdomain}reddit.com/r/{slug}/.json', video_id, fatal=False)
+        data = self._download_json(f'https://{subdomain}reddit.com/{slug}/.json', video_id, fatal=False)
         if not data:
             # Fall back to old.reddit.com in case the requested subdomain fails
-            data = self._download_json(f'https://old.reddit.com/r/{slug}/.json', video_id)
+            data = self._download_json(f'https://old.reddit.com/{slug}/.json', video_id)
         data = data[0]['data']['children'][0]['data']
         video_url = data['url']
 

From fbbb5508ea98ed8709847f5ecced7d70ff05e0ee Mon Sep 17 00:00:00 2001
From: Felix Yan <felixonmars@archlinux.org>
Date: Tue, 7 Feb 2023 03:24:47 +0800
Subject: [PATCH 019/871] [extractor/huya] Support HD streams (#6172)

Authored by: felixonmars
---
 yt_dlp/extractor/huya.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/huya.py b/yt_dlp/extractor/huya.py
index b6e9eec24..c4965f9bc 100644
--- a/yt_dlp/extractor/huya.py
+++ b/yt_dlp/extractor/huya.py
@@ -1,5 +1,6 @@
 import hashlib
 import random
+import re
 
 from ..compat import compat_urlparse, compat_b64decode
 
@@ -37,7 +38,7 @@ class HuyaLiveIE(InfoExtractor):
     }]
 
     _RESOLUTION = {
-        '蓝光4M': {
+        '蓝光': {
             'width': 1920,
             'height': 1080,
         },
@@ -76,11 +77,15 @@ def _real_extract(self, url):
             if re_secret:
                 fm, ss = self.encrypt(params, stream_info, stream_name)
             for si in stream_data.get('vMultiStreamInfo'):
+                display_name, bitrate = re.fullmatch(
+                    r'(.+?)(?:(\d+)M)?', si.get('sDisplayName')).groups()
                 rate = si.get('iBitRate')
                 if rate:
                     params['ratio'] = rate
                 else:
                     params.pop('ratio', None)
+                    if bitrate:
+                        rate = int(bitrate) * 1000
                 if re_secret:
                     params['wsSecret'] = hashlib.md5(
                         '_'.join([fm, params['u'], stream_name, ss, params['wsTime']]))
@@ -90,7 +95,7 @@ def _real_extract(self, url):
                     'tbr': rate,
                     'url': update_url_query(f'{stream_url}/{stream_name}.{stream_info.get("sFlvUrlSuffix")}',
                                             query=params),
-                    **self._RESOLUTION.get(si.get('sDisplayName'), {}),
+                    **self._RESOLUTION.get(display_name, {}),
                 })
 
         return {

From 7aefd19afed357c80743405ec2ace2148cba42e3 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 7 Feb 2023 01:17:11 +0530
Subject: [PATCH 020/871] Make `title` completely non-fatal

Ref: https://github.com/yt-dlp/yt-dlp/pull/6158#discussion_r1096984349
---
 yt_dlp/YoutubeDL.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index fd280726f..e092aed67 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2411,11 +2411,7 @@ def check_thumbnails(thumbnails):
     def _fill_common_fields(self, info_dict, final=True):
         # TODO: move sanitization here
         if final:
-            title = info_dict.get('title', NO_DEFAULT)
-            if title is NO_DEFAULT:
-                raise ExtractorError('Missing "title" field in extractor result',
-                                     video_id=info_dict['id'], ie=info_dict['extractor'])
-            info_dict['fulltitle'] = title
+            title = info_dict['fulltitle'] = info_dict.get('title')
             if not title:
                 if title == '':
                     self.write_debug('Extractor gave empty title. Creating a generic title')

From 754c84e2e416cf6609dd0e4632b4985a08d34043 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 8 Feb 2023 07:25:36 +0530
Subject: [PATCH 021/871] Support module level `__bool__` and `property`

---
 yt_dlp/compat/__init__.py     |  2 +-
 yt_dlp/compat/compat_utils.py | 99 ++++++++++++++++++++++-------------
 2 files changed, 64 insertions(+), 37 deletions(-)

diff --git a/yt_dlp/compat/__init__.py b/yt_dlp/compat/__init__.py
index 5d3db4b4c..5cc78ebc2 100644
--- a/yt_dlp/compat/__init__.py
+++ b/yt_dlp/compat/__init__.py
@@ -8,7 +8,7 @@
 
 # XXX: Implement this the same way as other DeprecationWarnings without circular import
 passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
-    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=3))
+    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=5))
 
 
 # HTMLParseError has been deprecated in Python 3.3 and removed in
diff --git a/yt_dlp/compat/compat_utils.py b/yt_dlp/compat/compat_utils.py
index 82e176281..b67944e6b 100644
--- a/yt_dlp/compat/compat_utils.py
+++ b/yt_dlp/compat/compat_utils.py
@@ -23,48 +23,75 @@ def get_package_info(module):
 
 
 def _is_package(module):
-    try:
-        module.__getattribute__('__path__')
-    except AttributeError:
-        return False
-    return True
+    return '__path__' in vars(module)
+
+
+class EnhancedModule(types.ModuleType):
+    def __new__(cls, name, *args, **kwargs):
+        if name not in sys.modules:
+            return super().__new__(cls, name, *args, **kwargs)
+
+        assert not args and not kwargs, 'Cannot pass additional arguments to an existing module'
+        module = sys.modules[name]
+        module.__class__ = cls
+        return module
+
+    def __init__(self, name, *args, **kwargs):
+        # Prevent __new__ from trigerring __init__ again
+        if name not in sys.modules:
+            super().__init__(name, *args, **kwargs)
+
+    def __bool__(self):
+        return vars(self).get('__bool__', lambda: True)()
+
+    def __getattribute__(self, attr):
+        try:
+            ret = super().__getattribute__(attr)
+        except AttributeError:
+            if attr.startswith('__') and attr.endswith('__'):
+                raise
+            getter = getattr(self, '__getattr__', None)
+            if not getter:
+                raise
+            ret = getter(attr)
+        return ret.fget() if isinstance(ret, property) else ret
 
 
 def passthrough_module(parent, child, allowed_attributes=None, *, callback=lambda _: None):
-    parent_module = importlib.import_module(parent)
-    child_module = None  # Import child module only as needed
+    """Passthrough parent module into a child module, creating the parent if necessary"""
+    parent = EnhancedModule(parent)
 
-    class PassthroughModule(types.ModuleType):
-        def __getattr__(self, attr):
-            if _is_package(parent_module):
-                with contextlib.suppress(ImportError):
-                    return importlib.import_module(f'.{attr}', parent)
+    def __getattr__(attr):
+        if _is_package(parent):
+            with contextlib.suppress(ImportError):
+                return importlib.import_module(f'.{attr}', parent.__name__)
 
-            ret = self.__from_child(attr)
-            if ret is _NO_ATTRIBUTE:
-                raise AttributeError(f'module {parent} has no attribute {attr}')
-            callback(attr)
-            return ret
+        ret = from_child(attr)
+        if ret is _NO_ATTRIBUTE:
+            raise AttributeError(f'module {parent.__name__} has no attribute {attr}')
+        callback(attr)
+        return ret
 
-        def __from_child(self, attr):
-            if allowed_attributes is None:
-                if attr.startswith('__') and attr.endswith('__'):
-                    return _NO_ATTRIBUTE
-            elif attr not in allowed_attributes:
+    def from_child(attr):
+        nonlocal child
+
+        if allowed_attributes is None:
+            if attr.startswith('__') and attr.endswith('__'):
                 return _NO_ATTRIBUTE
-
-            nonlocal child_module
-            child_module = child_module or importlib.import_module(child, parent)
-
-            with contextlib.suppress(AttributeError):
-                return getattr(child_module, attr)
-
-            if _is_package(child_module):
-                with contextlib.suppress(ImportError):
-                    return importlib.import_module(f'.{attr}', child)
-
+        elif attr not in allowed_attributes:
             return _NO_ATTRIBUTE
 
-    # Python 3.6 does not have module level __getattr__
-    # https://peps.python.org/pep-0562/
-    sys.modules[parent].__class__ = PassthroughModule
+        if isinstance(child, str):
+            child = importlib.import_module(child, parent.__name__)
+
+        with contextlib.suppress(AttributeError):
+            return getattr(child, attr)
+
+        if _is_package(child):
+            with contextlib.suppress(ImportError):
+                return importlib.import_module(f'.{attr}', child.__name__)
+
+        return _NO_ATTRIBUTE
+
+    parent.__getattr__ = __getattr__
+    return parent

From f6a765ceb59c55aea06921880c1c87d1ff36e5de Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 7 Feb 2023 03:22:29 +0530
Subject: [PATCH 022/871] [dependencies] Standardize `Cryptodome` imports

---
 test/test_aes.py                              |  6 +--
 test/test_compat.py                           |  3 ++
 yt_dlp/aes.py                                 |  8 ++--
 yt_dlp/compat/compat_utils.py                 | 16 +++-----
 yt_dlp/dependencies/Cryptodome.py             | 38 +++++++++++++++++++
 .../__init__.py}                              | 26 +++----------
 yt_dlp/downloader/hls.py                      |  4 +-
 yt_dlp/extractor/bilibili.py                  | 16 +++-----
 yt_dlp/extractor/ivi.py                       | 26 ++++---------
 9 files changed, 74 insertions(+), 69 deletions(-)
 create mode 100644 yt_dlp/dependencies/Cryptodome.py
 rename yt_dlp/{dependencies.py => dependencies/__init__.py} (75%)

diff --git a/test/test_aes.py b/test/test_aes.py
index 8e8fc0b3e..18f15fecb 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -26,7 +26,7 @@
     key_expansion,
     pad_block,
 )
-from yt_dlp.dependencies import Cryptodome_AES
+from yt_dlp.dependencies import Cryptodome
 from yt_dlp.utils import bytes_to_intlist, intlist_to_bytes
 
 # the encrypted data can be generate with 'devscripts/generate_aes_testdata.py'
@@ -48,7 +48,7 @@ def test_cbc_decrypt(self):
         data = b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\x27\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd'
         decrypted = intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(data), self.key, self.iv))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
-        if Cryptodome_AES:
+        if Cryptodome:
             decrypted = aes_cbc_decrypt_bytes(data, intlist_to_bytes(self.key), intlist_to_bytes(self.iv))
             self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
@@ -78,7 +78,7 @@ def test_gcm_decrypt(self):
         decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify(
             bytes_to_intlist(data), self.key, bytes_to_intlist(authentication_tag), self.iv[:12]))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
-        if Cryptodome_AES:
+        if Cryptodome:
             decrypted = aes_gcm_decrypt_and_verify_bytes(
                 data, intlist_to_bytes(self.key), authentication_tag, intlist_to_bytes(self.iv[:12]))
             self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
diff --git a/test/test_compat.py b/test/test_compat.py
index e3d775bc1..003a97abf 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -31,6 +31,9 @@ def test_compat_passthrough(self):
         # TODO: Test submodule
         # compat.asyncio.events  # Must not raise error
 
+        with self.assertWarns(DeprecationWarning):
+            compat.compat_pycrypto_AES  # Must not raise error
+
     def test_compat_expanduser(self):
         old_home = os.environ.get('HOME')
         test_str = R'C:\Documents and Settings\тест\Application Data'
diff --git a/yt_dlp/aes.py b/yt_dlp/aes.py
index 60ce99cb1..deff0a2b3 100644
--- a/yt_dlp/aes.py
+++ b/yt_dlp/aes.py
@@ -2,17 +2,17 @@
 from math import ceil
 
 from .compat import compat_ord
-from .dependencies import Cryptodome_AES
+from .dependencies import Cryptodome
 from .utils import bytes_to_intlist, intlist_to_bytes
 
-if Cryptodome_AES:
+if Cryptodome:
     def aes_cbc_decrypt_bytes(data, key, iv):
         """ Decrypt bytes with AES-CBC using pycryptodome """
-        return Cryptodome_AES.new(key, Cryptodome_AES.MODE_CBC, iv).decrypt(data)
+        return Cryptodome.Cipher.AES.new(key, Cryptodome.Cipher.AES.MODE_CBC, iv).decrypt(data)
 
     def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce):
         """ Decrypt bytes with AES-GCM using pycryptodome """
-        return Cryptodome_AES.new(key, Cryptodome_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag)
+        return Cryptodome.Cipher.AES.new(key, Cryptodome.Cipher.AES.MODE_GCM, nonce).decrypt_and_verify(data, tag)
 
 else:
     def aes_cbc_decrypt_bytes(data, key, iv):
diff --git a/yt_dlp/compat/compat_utils.py b/yt_dlp/compat/compat_utils.py
index b67944e6b..373389a46 100644
--- a/yt_dlp/compat/compat_utils.py
+++ b/yt_dlp/compat/compat_utils.py
@@ -10,16 +10,12 @@
 
 
 def get_package_info(module):
-    parent = module.__name__.split('.')[0]
-    parent_module = None
-    with contextlib.suppress(ImportError):
-        parent_module = importlib.import_module(parent)
-
-    for attr in ('__version__', 'version_string', 'version'):
-        version = getattr(parent_module, attr, None)
-        if version is not None:
-            break
-    return _Package(getattr(module, '_yt_dlp__identifier', parent), str(version))
+    return _Package(
+        name=getattr(module, '_yt_dlp__identifier', module.__name__),
+        version=str(next(filter(None, (
+            getattr(module, attr, None)
+            for attr in ('__version__', 'version_string', 'version')
+        )), None)))
 
 
 def _is_package(module):
diff --git a/yt_dlp/dependencies/Cryptodome.py b/yt_dlp/dependencies/Cryptodome.py
new file mode 100644
index 000000000..b95f45d72
--- /dev/null
+++ b/yt_dlp/dependencies/Cryptodome.py
@@ -0,0 +1,38 @@
+import importlib
+
+from ..compat import functools
+from ..compat.compat_utils import EnhancedModule, passthrough_module
+
+EnhancedModule(__name__)
+
+try:
+    import Cryptodome as _parent
+except ImportError:
+    try:
+        import Crypto as _parent
+    except (ImportError, SyntaxError):  # Old Crypto gives SyntaxError in newer Python
+        _parent = EnhancedModule('Cryptodome')
+        __bool__ = lambda: False
+
+
+@functools.cache
+def __getattr__(name):
+    try:
+        submodule = importlib.import_module(f'.{name}', _parent.__name__)
+    except ImportError:
+        return getattr(_parent, name)
+    return passthrough_module(f'{__name__}.{name}', submodule)
+
+
+@property
+@functools.cache
+def _yt_dlp__identifier():
+    if _parent.__name__ == 'Crypto':
+        from Crypto.Cipher import AES
+        try:
+            # In pycrypto, mode defaults to ECB. See:
+            # https://www.pycryptodome.org/en/latest/src/vs_pycrypto.html#:~:text=not%20have%20ECB%20as%20default%20mode
+            AES.new(b'abcdefghijklmnop')
+        except TypeError:
+            return 'pycrypto'
+    return _parent.__name__
diff --git a/yt_dlp/dependencies.py b/yt_dlp/dependencies/__init__.py
similarity index 75%
rename from yt_dlp/dependencies.py
rename to yt_dlp/dependencies/__init__.py
index 5a5363adb..c2214e6db 100644
--- a/yt_dlp/dependencies.py
+++ b/yt_dlp/dependencies/__init__.py
@@ -23,24 +23,6 @@
         certifi = None
 
 
-try:
-    from Cryptodome.Cipher import AES as Cryptodome_AES
-except ImportError:
-    try:
-        from Crypto.Cipher import AES as Cryptodome_AES
-    except (ImportError, SyntaxError):  # Old Crypto gives SyntaxError in newer Python
-        Cryptodome_AES = None
-    else:
-        try:
-            # In pycrypto, mode defaults to ECB. See:
-            # https://www.pycryptodome.org/en/latest/src/vs_pycrypto.html#:~:text=not%20have%20ECB%20as%20default%20mode
-            Cryptodome_AES.new(b'abcdefghijklmnop')
-        except TypeError:
-            pass
-        else:
-            Cryptodome_AES._yt_dlp__identifier = 'pycrypto'
-
-
 try:
     import mutagen
 except ImportError:
@@ -84,12 +66,16 @@
         xattr._yt_dlp__identifier = 'pyxattr'
 
 
+from . import Cryptodome
+
 all_dependencies = {k: v for k, v in globals().items() if not k.startswith('_')}
-
-
 available_dependencies = {k: v for k, v in all_dependencies.items() if v}
 
 
+# Deprecated
+Cryptodome_AES = Cryptodome.Cipher.AES if Cryptodome else None
+
+
 __all__ = [
     'all_dependencies',
     'available_dependencies',
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index 2010f3dc9..ae18ac419 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -7,7 +7,7 @@
 from .external import FFmpegFD
 from .fragment import FragmentFD
 from .. import webvtt
-from ..dependencies import Cryptodome_AES
+from ..dependencies import Cryptodome
 from ..utils import bug_reports_message, parse_m3u8_attributes, update_url_query
 
 
@@ -63,7 +63,7 @@ def real_download(self, filename, info_dict):
         can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
         if can_download:
             has_ffmpeg = FFmpegFD.available()
-            no_crypto = not Cryptodome_AES and '#EXT-X-KEY:METHOD=AES-128' in s
+            no_crypto = not Cryptodome and '#EXT-X-KEY:METHOD=AES-128' in s
             if no_crypto and has_ffmpeg:
                 can_download, message = False, 'The stream has AES-128 encryption and pycryptodomex is not available'
             elif no_crypto:
diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index d4b05248f..266d57871 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -6,6 +6,7 @@
 import urllib.parse
 
 from .common import InfoExtractor, SearchInfoExtractor
+from ..dependencies import Cryptodome
 from ..utils import (
     ExtractorError,
     GeoRestrictedError,
@@ -893,22 +894,15 @@ def _parse_video_metadata(self, video_data):
         }
 
     def _perform_login(self, username, password):
-        try:
-            from Cryptodome.PublicKey import RSA
-            from Cryptodome.Cipher import PKCS1_v1_5
-        except ImportError:
-            try:
-                from Crypto.PublicKey import RSA
-                from Crypto.Cipher import PKCS1_v1_5
-            except ImportError:
-                raise ExtractorError('pycryptodomex not found. Please install', expected=True)
+        if not Cryptodome:
+            raise ExtractorError('pycryptodomex not found. Please install', expected=True)
 
         key_data = self._download_json(
             'https://passport.bilibili.tv/x/intl/passport-login/web/key?lang=en-US', None,
             note='Downloading login key', errnote='Unable to download login key')['data']
 
-        public_key = RSA.importKey(key_data['key'])
-        password_hash = PKCS1_v1_5.new(public_key).encrypt((key_data['hash'] + password).encode('utf-8'))
+        public_key = Cryptodome.PublicKey.RSA.importKey(key_data['key'])
+        password_hash = Cryptodome.Cipher.PKCS1_v1_5.new(public_key).encrypt((key_data['hash'] + password).encode('utf-8'))
         login_post = self._download_json(
             'https://passport.bilibili.tv/x/intl/passport-login/web/login/password?lang=en-US', None, data=urlencode_postdata({
                 'username': username,
diff --git a/yt_dlp/extractor/ivi.py b/yt_dlp/extractor/ivi.py
index dc6a48196..96220bea9 100644
--- a/yt_dlp/extractor/ivi.py
+++ b/yt_dlp/extractor/ivi.py
@@ -2,11 +2,8 @@
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    qualities,
-)
+from ..dependencies import Cryptodome
+from ..utils import ExtractorError, int_or_none, qualities
 
 
 class IviIE(InfoExtractor):
@@ -94,18 +91,8 @@ def _real_extract(self, url):
         for site in (353, 183):
             content_data = (data % site).encode()
             if site == 353:
-                try:
-                    from Cryptodome.Cipher import Blowfish
-                    from Cryptodome.Hash import CMAC
-                    pycryptodome_found = True
-                except ImportError:
-                    try:
-                        from Crypto.Cipher import Blowfish
-                        from Crypto.Hash import CMAC
-                        pycryptodome_found = True
-                    except ImportError:
-                        pycryptodome_found = False
-                        continue
+                if not Cryptodome:
+                    continue
 
                 timestamp = (self._download_json(
                     self._LIGHT_URL, video_id,
@@ -118,7 +105,8 @@ def _real_extract(self, url):
 
                 query = {
                     'ts': timestamp,
-                    'sign': CMAC.new(self._LIGHT_KEY, timestamp.encode() + content_data, Blowfish).hexdigest(),
+                    'sign': Cryptodome.Hash.CMAC.new(self._LIGHT_KEY, timestamp.encode() + content_data,
+                                                     Cryptodome.Cipher.Blowfish).hexdigest(),
                 }
             else:
                 query = {}
@@ -138,7 +126,7 @@ def _real_extract(self, url):
                     extractor_msg = 'Video %s does not exist'
                 elif site == 353:
                     continue
-                elif not pycryptodome_found:
+                elif not Cryptodome:
                     raise ExtractorError('pycryptodomex not found. Please install', expected=True)
                 elif message:
                     extractor_msg += ': ' + message

From 88426d9446758c707fb511408f2d6f56de952db4 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 8 Feb 2023 08:14:36 +0530
Subject: [PATCH 023/871] [compat_utils] Improve `passthrough_module`

---
 Makefile                          |  2 +-
 yt_dlp/compat/compat_utils.py     | 26 +++++++++++++++-----------
 yt_dlp/dependencies/Cryptodome.py | 14 ++------------
 3 files changed, 18 insertions(+), 24 deletions(-)

diff --git a/Makefile b/Makefile
index ca7d641ab..d5d47629b 100644
--- a/Makefile
+++ b/Makefile
@@ -74,7 +74,7 @@ offlinetest: codetest
 	$(PYTHON) -m pytest -k "not download"
 
 # XXX: This is hard to maintain
-CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat
+CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/dependencies
 yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
 	mkdir -p zip
 	for d in $(CODE_FOLDERS) ; do \
diff --git a/yt_dlp/compat/compat_utils.py b/yt_dlp/compat/compat_utils.py
index 373389a46..f8679c98e 100644
--- a/yt_dlp/compat/compat_utils.py
+++ b/yt_dlp/compat/compat_utils.py
@@ -1,5 +1,6 @@
 import collections
 import contextlib
+import functools
 import importlib
 import sys
 import types
@@ -22,6 +23,10 @@ def _is_package(module):
     return '__path__' in vars(module)
 
 
+def _is_dunder(name):
+    return name.startswith('__') and name.endswith('__')
+
+
 class EnhancedModule(types.ModuleType):
     def __new__(cls, name, *args, **kwargs):
         if name not in sys.modules:
@@ -44,7 +49,7 @@ def __getattribute__(self, attr):
         try:
             ret = super().__getattribute__(attr)
         except AttributeError:
-            if attr.startswith('__') and attr.endswith('__'):
+            if _is_dunder(attr):
                 raise
             getter = getattr(self, '__getattr__', None)
             if not getter:
@@ -53,7 +58,7 @@ def __getattribute__(self, attr):
         return ret.fget() if isinstance(ret, property) else ret
 
 
-def passthrough_module(parent, child, allowed_attributes=None, *, callback=lambda _: None):
+def passthrough_module(parent, child, allowed_attributes=(..., ), *, callback=lambda _: None):
     """Passthrough parent module into a child module, creating the parent if necessary"""
     parent = EnhancedModule(parent)
 
@@ -68,24 +73,23 @@ def __getattr__(attr):
         callback(attr)
         return ret
 
+    @functools.lru_cache(maxsize=None)
     def from_child(attr):
         nonlocal child
-
-        if allowed_attributes is None:
-            if attr.startswith('__') and attr.endswith('__'):
+        if attr not in allowed_attributes:
+            if ... not in allowed_attributes or _is_dunder(attr):
                 return _NO_ATTRIBUTE
-        elif attr not in allowed_attributes:
-            return _NO_ATTRIBUTE
 
         if isinstance(child, str):
             child = importlib.import_module(child, parent.__name__)
 
-        with contextlib.suppress(AttributeError):
-            return getattr(child, attr)
-
         if _is_package(child):
             with contextlib.suppress(ImportError):
-                return importlib.import_module(f'.{attr}', child.__name__)
+                return passthrough_module(f'{parent.__name__}.{attr}',
+                                          importlib.import_module(f'.{attr}', child.__name__))
+
+        with contextlib.suppress(AttributeError):
+            return getattr(child, attr)
 
         return _NO_ATTRIBUTE
 
diff --git a/yt_dlp/dependencies/Cryptodome.py b/yt_dlp/dependencies/Cryptodome.py
index b95f45d72..580ce0753 100644
--- a/yt_dlp/dependencies/Cryptodome.py
+++ b/yt_dlp/dependencies/Cryptodome.py
@@ -1,10 +1,6 @@
-import importlib
-
 from ..compat import functools
 from ..compat.compat_utils import EnhancedModule, passthrough_module
 
-EnhancedModule(__name__)
-
 try:
     import Cryptodome as _parent
 except ImportError:
@@ -14,14 +10,8 @@
         _parent = EnhancedModule('Cryptodome')
         __bool__ = lambda: False
 
-
-@functools.cache
-def __getattr__(name):
-    try:
-        submodule = importlib.import_module(f'.{name}', _parent.__name__)
-    except ImportError:
-        return getattr(_parent, name)
-    return passthrough_module(f'{__name__}.{name}', submodule)
+passthrough_module(__name__, _parent, (..., '__version__'))
+del passthrough_module, EnhancedModule
 
 
 @property

From b1bde57bef878478e3503ab07190fd207914ade9 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 8 Feb 2023 04:11:08 +0100
Subject: [PATCH 024/871] [utils] `traverse_obj`: Fix several behavioral
 problems

See #6180 for further info

Authored by: Grub4K
---
 test/test_utils.py |  43 +++++++++-----
 yt_dlp/utils.py    | 141 ++++++++++++++++++++++++++-------------------
 2 files changed, 108 insertions(+), 76 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index ffe1b729f..190e4ef9b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2000,8 +2000,8 @@ def test_traverse_obj(self):
 
         # Test Ellipsis behavior
         self.assertCountEqual(traverse_obj(_TEST_DATA, ...),
-                              (item for item in _TEST_DATA.values() if item is not None),
-                              msg='`...` should give all values except `None`')
+                              (item for item in _TEST_DATA.values() if item not in (None, [], {})),
+                              msg='`...` should give all non discarded values')
         self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, ...)), _TEST_DATA['urls'][0].values(),
                               msg='`...` selection for dicts should select all values')
         self.assertEqual(traverse_obj(_TEST_DATA, (..., ..., 'url')),
@@ -2084,15 +2084,23 @@ def test_traverse_obj(self):
                          {0: ['https://www.example.com/1', 'https://www.example.com/0']},
                          msg='tripple nesting in dict path should be treated as branches')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}), {},
-                         msg='remove `None` values when dict key')
+                         msg='remove `None` values when top level dict key fails')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}, default=...), {0: ...},
-                         msg='do not remove `None` values if `default`')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {0: {}},
-                         msg='do not remove empty values when dict key')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=...), {0: {}},
-                         msg='do not remove empty values when dict key and a default')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', ...)}), {0: []},
-                         msg='if branch in dict key not successful, return `[]`')
+                         msg='use `default` if key fails and `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {},
+                         msg='remove empty values when dict key')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=...), {0: ...},
+                         msg='use `default` when dict key and `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}), {},
+                         msg='remove empty values when nested dict key fails')
+        self.assertEqual(traverse_obj(None, {0: 'fail'}), {},
+                         msg='default to dict if pruned')
+        self.assertEqual(traverse_obj(None, {0: 'fail'}, default=...), {},
+                         msg='default to dict if pruned and default is given')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}, default=...), {0: {0: ...}},
+                         msg='use nested `default` when nested dict key fails and `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', ...)}), {},
+                         msg='remove key if branch in dict key not successful')
 
         # Testing default parameter behavior
         _DEFAULT_DATA = {'None': None, 'int': 0, 'list': []}
@@ -2183,14 +2191,17 @@ def test_traverse_obj(self):
                                       traverse_string=True), '.',
                          msg='traverse into converted data if `traverse_string`')
         self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', ...),
-                                      traverse_string=True), list('str'),
-                         msg='`...` branching into string should result in list')
+                                      traverse_string=True), 'str',
+                         msg='`...` should result in string (same value) if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', slice(0, None, 2)),
+                                      traverse_string=True), 'sr',
+                         msg='`slice` should result in string if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda i, v: i or v == "s"),
+                                      traverse_string=True), 'str',
+                         msg='function should result in string if `traverse_string`')
         self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', (0, 2)),
                                       traverse_string=True), ['s', 'r'],
-                         msg='branching into string should result in list')
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda _, x: x),
-                                      traverse_string=True), list('str'),
-                         msg='function branching into string should result in list')
+                         msg='branching should result in list if `traverse_string`')
 
         # Test is_user_input behavior
         _IS_USER_INPUT_DATA = {'range8': list(range(8))}
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index e1e0f7b25..878b2b6a8 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5420,7 +5420,7 @@ def traverse_obj(
     Each of the provided `paths` is tested and the first producing a valid result will be returned.
     The next path will also be tested if the path branched but no results could be found.
     Supported values for traversal are `Mapping`, `Sequence` and `re.Match`.
-    A value of None is treated as the absence of a value.
+    Unhelpful values (`[]`, `{}`, `None`) are treated as the absence of a value and discarded.
 
     The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
 
@@ -5446,6 +5446,8 @@ def traverse_obj(
 
     @params paths           Paths which to traverse by.
     @param default          Value to return if the paths do not match.
+                            If the last key in the path is a `dict`, it will apply to each value inside
+                            the dict instead, depth first. Try to avoid if using nested `dict` keys.
     @param expected_type    If a `type`, only accept final values of this type.
                             If any other callable, try to call the function on each result.
                             If the last key in the path is a `dict`, it will apply to each value inside
@@ -5460,12 +5462,15 @@ def traverse_obj(
     @param traverse_string  Whether to traverse into objects as strings.
                             If `True`, any non-compatible object will first be
                             converted into a string and then traversed into.
+                            The return value of that path will be a string instead,
+                            not respecting any further branching.
 
 
     @returns                The result of the object traversal.
                             If successful, `get_all=True`, and the path branches at least once,
                             then a list of results is returned instead.
-                            A list is always returned if the last path branches and no `default` is given.
+                            If no `default` is given and the last path branches, a `list` of results
+                            is always returned. If a path ends on a `dict` that result will always be a `dict`.
     """
     is_sequence = lambda x: isinstance(x, collections.abc.Sequence) and not isinstance(x, (str, bytes))
     casefold = lambda k: k.casefold() if isinstance(k, str) else k
@@ -5475,87 +5480,94 @@ def traverse_obj(
     else:
         type_test = lambda val: try_call(expected_type or IDENTITY, args=(val,))
 
-    def apply_key(key, test_type, obj):
+    def apply_key(key, obj, is_last):
+        branching = False
+        result = None
+
         if obj is None:
-            return
+            pass
 
         elif key is None:
-            yield obj
+            result = obj
 
         elif isinstance(key, set):
             assert len(key) == 1, 'Set should only be used to wrap a single item'
             item = next(iter(key))
             if isinstance(item, type):
                 if isinstance(obj, item):
-                    yield obj
+                    result = obj
             else:
-                yield try_call(item, args=(obj,))
+                result = try_call(item, args=(obj,))
 
         elif isinstance(key, (list, tuple)):
-            for branch in key:
-                _, result = apply_path(obj, branch, test_type)
-                yield from result
+            branching = True
+            result = itertools.chain.from_iterable(
+                apply_path(obj, branch, is_last)[0] for branch in key)
 
         elif key is ...:
+            branching = True
             if isinstance(obj, collections.abc.Mapping):
-                yield from obj.values()
+                result = obj.values()
             elif is_sequence(obj):
-                yield from obj
+                result = obj
             elif isinstance(obj, re.Match):
-                yield from obj.groups()
+                result = obj.groups()
             elif traverse_string:
-                yield from str(obj)
+                branching = False
+                result = str(obj)
+            else:
+                result = ()
 
         elif callable(key):
-            if is_sequence(obj):
-                iter_obj = enumerate(obj)
-            elif isinstance(obj, collections.abc.Mapping):
+            branching = True
+            if isinstance(obj, collections.abc.Mapping):
                 iter_obj = obj.items()
+            elif is_sequence(obj):
+                iter_obj = enumerate(obj)
             elif isinstance(obj, re.Match):
                 iter_obj = itertools.chain(
                     enumerate((obj.group(), *obj.groups())),
                     obj.groupdict().items())
             elif traverse_string:
+                branching = False
                 iter_obj = enumerate(str(obj))
             else:
-                return
-            yield from (v for k, v in iter_obj if try_call(key, args=(k, v)))
+                iter_obj = ()
+
+            result = (v for k, v in iter_obj if try_call(key, args=(k, v)))
+            if not branching:  # string traversal
+                result = ''.join(result)
 
         elif isinstance(key, dict):
-            iter_obj = ((k, _traverse_obj(obj, v, test_type=test_type)) for k, v in key.items())
-            yield {k: v if v is not None else default for k, v in iter_obj
-                   if v is not None or default is not NO_DEFAULT}
+            iter_obj = ((k, _traverse_obj(obj, v, False, is_last)) for k, v in key.items())
+            result = {
+                k: v if v is not None else default for k, v in iter_obj
+                if v is not None or default is not NO_DEFAULT
+            } or None
 
         elif isinstance(obj, collections.abc.Mapping):
-            yield (obj.get(key) if casesense or (key in obj)
-                   else next((v for k, v in obj.items() if casefold(k) == key), None))
+            result = (obj.get(key) if casesense or (key in obj) else
+                      next((v for k, v in obj.items() if casefold(k) == key), None))
 
         elif isinstance(obj, re.Match):
             if isinstance(key, int) or casesense:
                 with contextlib.suppress(IndexError):
-                    yield obj.group(key)
-                    return
+                    result = obj.group(key)
 
-            if not isinstance(key, str):
-                return
-
-            yield next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
-
-        else:
-            if is_user_input:
-                key = (int_or_none(key) if ':' not in key
-                       else slice(*map(int_or_none, key.split(':'))))
-
-            if not isinstance(key, (int, slice)):
-                return
+            elif isinstance(key, str):
+                result = next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
 
+        elif isinstance(key, (int, slice)):
             if not is_sequence(obj):
-                if not traverse_string:
-                    return
-                obj = str(obj)
+                if traverse_string:
+                    with contextlib.suppress(IndexError):
+                        result = str(obj)[key]
+            else:
+                branching = isinstance(key, slice)
+                with contextlib.suppress(IndexError):
+                    result = obj[key]
 
-            with contextlib.suppress(IndexError):
-                yield obj[key]
+        return branching, result if branching else (result,)
 
     def lazy_last(iterable):
         iterator = iter(iterable)
@@ -5569,45 +5581,54 @@ def lazy_last(iterable):
 
         yield True, prev
 
-    def apply_path(start_obj, path, test_type=False):
+    def apply_path(start_obj, path, test_type):
         objs = (start_obj,)
         has_branched = False
 
         key = None
         for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
-            if is_user_input and key == ':':
-                key = ...
+            if is_user_input and isinstance(key, str):
+                if key == ':':
+                    key = ...
+                elif ':' in key:
+                    key = slice(*map(int_or_none, key.split(':')))
+                elif int_or_none(key) is not None:
+                    key = int(key)
 
             if not casesense and isinstance(key, str):
                 key = key.casefold()
 
-            if key is ... or isinstance(key, (list, tuple)) or callable(key):
-                has_branched = True
-
             if __debug__ and callable(key):
                 # Verify function signature
                 inspect.signature(key).bind(None, None)
 
-            key_func = functools.partial(apply_key, key, last)
-            objs = itertools.chain.from_iterable(map(key_func, objs))
+            new_objs = []
+            for obj in objs:
+                branching, results = apply_key(key, obj, last)
+                has_branched |= branching
+                new_objs.append(results)
+
+            objs = itertools.chain.from_iterable(new_objs)
 
         if test_type and not isinstance(key, (dict, list, tuple)):
             objs = map(type_test, objs)
 
-        return has_branched, objs
-
-    def _traverse_obj(obj, path, use_list=True, test_type=True):
-        has_branched, results = apply_path(obj, path, test_type)
-        results = LazyList(x for x in results if x is not None)
+        return objs, has_branched, isinstance(key, dict)
 
+    def _traverse_obj(obj, path, allow_empty, test_type):
+        results, has_branched, is_dict = apply_path(obj, path, test_type)
+        results = LazyList(item for item in results if item not in (None, [], {}))
         if get_all and has_branched:
-            return results.exhaust() if results or use_list else None
+            if results:
+                return results.exhaust()
+            if allow_empty:
+                return [] if default is NO_DEFAULT else default
+            return None
 
-        return results[0] if results else None
+        return results[0] if results else {} if allow_empty and is_dict else None
 
     for index, path in enumerate(paths, 1):
-        use_list = default is NO_DEFAULT and index == len(paths)
-        result = _traverse_obj(obj, path, use_list)
+        result = _traverse_obj(obj, path, index == len(paths), True)
         if result is not None:
             return result
 

From f7efe6dc958eb0689cb9534ff0b4e592040be8df Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Wed, 8 Feb 2023 01:20:19 -0400
Subject: [PATCH 025/871] [extractor/pornez] Handle relative URLs in iframe
 (#6171)

Authored by: JChris246
Closes #6162
---
 yt_dlp/extractor/pornez.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/pornez.py b/yt_dlp/extractor/pornez.py
index df0e44a69..3a22cb821 100644
--- a/yt_dlp/extractor/pornez.py
+++ b/yt_dlp/extractor/pornez.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import int_or_none, urljoin
 
 
 class PornezIE(InfoExtractor):
@@ -20,7 +20,8 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         iframe_src = self._html_search_regex(
-            r'<iframe[^>]+src="(https?://pornez\.net/player/\?[^"]+)"', webpage, 'iframe', fatal=True)
+            r'<iframe[^>]+src="([^"]+)"', webpage, 'iframe', fatal=True)
+        iframe_src = urljoin('https://pornez.net', iframe_src)
         title = self._html_search_meta(['name', 'twitter:title', 'og:title'], webpage, 'title', default=None)
         if title is None:
             title = self._search_regex(r'<h1>(.*?)</h1>', webpage, 'title', fatal=True)

From 7e68567e508168b345266c0c19812ad50a829eaa Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Wed, 8 Feb 2023 11:03:54 +0530
Subject: [PATCH 026/871] [downloader/hls] Allow extractors to provide AES key
 (#6158)

and related cleanup

Authored by: bashonly, Grub4K

Co-authored-by: Simon Sawicki <contact@grub4k.xyz>
---
 yt_dlp/YoutubeDL.py           |  2 +-
 yt_dlp/downloader/external.py |  1 +
 yt_dlp/downloader/fragment.py |  3 ++-
 yt_dlp/downloader/hls.py      | 45 ++++++++++++++++++++++-------------
 yt_dlp/extractor/common.py    | 13 +++++++++-
 yt_dlp/extractor/vzaar.py     |  2 +-
 6 files changed, 45 insertions(+), 21 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index e092aed67..8f88104ef 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -554,7 +554,7 @@ class YoutubeDL:
         'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns',
         'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start',
         'preference', 'language', 'language_preference', 'quality', 'source_preference',
-        'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'downloader_options',
+        'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'hls_aes', 'downloader_options',
         'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time'
     }
     _format_selection_exts = {
diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index 3917af448..5f54017a8 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -104,6 +104,7 @@ def supports(cls, info_dict):
         return all((
             not info_dict.get('to_stdout') or Features.TO_STDOUT in cls.SUPPORTED_FEATURES,
             '+' not in info_dict['protocol'] or Features.MULTIPLE_FORMATS in cls.SUPPORTED_FEATURES,
+            not traverse_obj(info_dict, ('hls_aes', ...), 'extra_param_to_segment_url'),
             all(proto in cls.SUPPORTED_PROTOCOLS for proto in info_dict['protocol'].split('+')),
         ))
 
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 83f7870ed..02f8559cc 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -360,7 +360,8 @@ def decrypt_fragment(fragment, frag_content):
             if not decrypt_info or decrypt_info['METHOD'] != 'AES-128':
                 return frag_content
             iv = decrypt_info.get('IV') or struct.pack('>8xq', fragment['media_sequence'])
-            decrypt_info['KEY'] = decrypt_info.get('KEY') or _get_key(info_dict.get('_decryption_key_url') or decrypt_info['URI'])
+            decrypt_info['KEY'] = (decrypt_info.get('KEY')
+                                   or _get_key(traverse_obj(info_dict, ('hls_aes', 'uri')) or decrypt_info['URI']))
             # Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block
             # size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded,
             # not what it decrypts to.
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index ae18ac419..29d6f6241 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -8,7 +8,14 @@
 from .fragment import FragmentFD
 from .. import webvtt
 from ..dependencies import Cryptodome
-from ..utils import bug_reports_message, parse_m3u8_attributes, update_url_query
+from ..utils import (
+    bug_reports_message,
+    parse_m3u8_attributes,
+    remove_start,
+    traverse_obj,
+    update_url_query,
+    urljoin,
+)
 
 
 class HlsFD(FragmentFD):
@@ -150,6 +157,13 @@ def is_ad_fragment_end(s):
         i = 0
         media_sequence = 0
         decrypt_info = {'METHOD': 'NONE'}
+        external_aes_key = traverse_obj(info_dict, ('hls_aes', 'key'))
+        if external_aes_key:
+            external_aes_key = binascii.unhexlify(remove_start(external_aes_key, '0x'))
+            assert len(external_aes_key) in (16, 24, 32), 'Invalid length for HLS AES-128 key'
+        external_aes_iv = traverse_obj(info_dict, ('hls_aes', 'iv'))
+        if external_aes_iv:
+            external_aes_iv = binascii.unhexlify(remove_start(external_aes_iv, '0x').zfill(32))
         byte_range = {}
         discontinuity_count = 0
         frag_index = 0
@@ -165,10 +179,7 @@ def is_ad_fragment_end(s):
                     frag_index += 1
                     if frag_index <= ctx['fragment_index']:
                         continue
-                    frag_url = (
-                        line
-                        if re.match(r'^https?://', line)
-                        else urllib.parse.urljoin(man_url, line))
+                    frag_url = urljoin(man_url, line)
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
 
@@ -190,10 +201,7 @@ def is_ad_fragment_end(s):
                         return False
                     frag_index += 1
                     map_info = parse_m3u8_attributes(line[11:])
-                    frag_url = (
-                        map_info.get('URI')
-                        if re.match(r'^https?://', map_info.get('URI'))
-                        else urllib.parse.urljoin(man_url, map_info.get('URI')))
+                    frag_url = urljoin(man_url, map_info.get('URI'))
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
 
@@ -218,15 +226,18 @@ def is_ad_fragment_end(s):
                     decrypt_url = decrypt_info.get('URI')
                     decrypt_info = parse_m3u8_attributes(line[11:])
                     if decrypt_info['METHOD'] == 'AES-128':
-                        if 'IV' in decrypt_info:
+                        if external_aes_iv:
+                            decrypt_info['IV'] = external_aes_iv
+                        elif 'IV' in decrypt_info:
                             decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:].zfill(32))
-                        if not re.match(r'^https?://', decrypt_info['URI']):
-                            decrypt_info['URI'] = urllib.parse.urljoin(
-                                man_url, decrypt_info['URI'])
-                        if extra_query:
-                            decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
-                        if decrypt_url != decrypt_info['URI']:
-                            decrypt_info['KEY'] = None
+                        if external_aes_key:
+                            decrypt_info['KEY'] = external_aes_key
+                        else:
+                            decrypt_info['URI'] = urljoin(man_url, decrypt_info['URI'])
+                            if extra_query:
+                                decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
+                            if decrypt_url != decrypt_info['URI']:
+                                decrypt_info['KEY'] = None
 
                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                     media_sequence = int(line[22:])
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index f80536470..09b03e69a 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -81,8 +81,8 @@
     update_Request,
     update_url_query,
     url_basename,
-    urlhandle_detect_ext,
     url_or_none,
+    urlhandle_detect_ext,
     urljoin,
     variadic,
     xpath_element,
@@ -220,6 +220,17 @@ class InfoExtractor:
                     * no_resume  The server does not support resuming the
                                  (HTTP or RTMP) download. Boolean.
                     * has_drm    The format has DRM and cannot be downloaded. Boolean
+                    * extra_param_to_segment_url  A query string to append to each
+                                 fragment's URL, or to update each existing query string
+                                 with. Only applied by the native HLS/DASH downloaders.
+                    * hls_aes    A dictionary of HLS AES-128 decryption information
+                                 used by the native HLS downloader to override the
+                                 values in the media playlist when an '#EXT-X-KEY' tag
+                                 is present in the playlist:
+                                 * uri  The URI from which the key will be downloaded
+                                 * key  The key (as hex) used to decrypt fragments.
+                                        If `key` is given, any key URI will be ignored
+                                 * iv   The IV (as hex) used to decrypt fragments
                     * downloader_options  A dictionary of downloader options
                                  (For internal use only)
                                  * http_chunk_size Chunk size for HTTP downloads
diff --git a/yt_dlp/extractor/vzaar.py b/yt_dlp/extractor/vzaar.py
index 6b9817c9e..19908a929 100644
--- a/yt_dlp/extractor/vzaar.py
+++ b/yt_dlp/extractor/vzaar.py
@@ -87,7 +87,7 @@ def _real_extract(self, url):
                 m3u8_id='hls', fatal=False)
             if hls_aes:
                 for f in m3u8_formats:
-                    f['_decryption_key_url'] = url_templ % ('goose', '') + qs
+                    f['hls_aes'] = {'uri': url_templ % ('goose', '') + qs}
             formats.extend(m3u8_formats)
 
         return {

From e61acb40b2cb6ef45508d72235026d458c9d5dff Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 7 Feb 2023 23:42:11 -0600
Subject: [PATCH 027/871] [extractor/wrestleuniverse] Add extractors (#6158)

Authored by bashonly, Grub4K
Closes #6120

Co-authored-by: Simon Sawicki <contact@grub4k.xyz>
---
 yt_dlp/extractor/_extractors.py     |   4 +
 yt_dlp/extractor/wrestleuniverse.py | 233 ++++++++++++++++++++++++++++
 2 files changed, 237 insertions(+)
 create mode 100644 yt_dlp/extractor/wrestleuniverse.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index a67c39479..50dfe2eb1 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2274,6 +2274,10 @@
     WPPilotIE,
     WPPilotChannelsIE,
 )
+from .wrestleuniverse import (
+    WrestleUniverseVODIE,
+    WrestleUniversePPVIE,
+)
 from .wsj import (
     WSJIE,
     WSJArticleIE,
diff --git a/yt_dlp/extractor/wrestleuniverse.py b/yt_dlp/extractor/wrestleuniverse.py
new file mode 100644
index 000000000..78e7c83ab
--- /dev/null
+++ b/yt_dlp/extractor/wrestleuniverse.py
@@ -0,0 +1,233 @@
+import base64
+import binascii
+import json
+import time
+
+from .common import InfoExtractor
+from ..dependencies import Cryptodome
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    jwt_decode_hs256,
+    traverse_obj,
+    try_call,
+    url_or_none,
+)
+
+
+class WrestleUniverseBaseIE(InfoExtractor):
+    _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
+    _API_PATH = None
+    _TOKEN = None
+    _TOKEN_EXPIRY = None
+
+    def _get_token_cookie(self):
+        if not self._TOKEN or not self._TOKEN_EXPIRY:
+            self._TOKEN = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
+            if not self._TOKEN:
+                self.raise_login_required()
+            expiry = traverse_obj(jwt_decode_hs256(self._TOKEN), ('exp', {int_or_none}))
+            if not expiry:
+                raise ExtractorError('There was a problem with the token cookie')
+            self._TOKEN_EXPIRY = expiry
+
+        if self._TOKEN_EXPIRY <= int(time.time()):
+            raise ExtractorError(
+                'Expired token. Refresh your cookies in browser and try again', expected=True)
+
+        return self._TOKEN
+
+    def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True):
+        headers = {'CA-CID': ''}
+        if data:
+            headers['Content-Type'] = 'application/json;charset=utf-8'
+            data = json.dumps(data, separators=(',', ':')).encode()
+        if auth:
+            headers['Authorization'] = f'Bearer {self._get_token_cookie()}'
+        return self._download_json(
+            f'https://api.wrestle-universe.com/v1/{self._API_PATH}/{video_id}{param}', video_id,
+            note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
+            data=data, headers=headers, query=query, fatal=fatal)
+
+    def _call_encrypted_api(self, video_id, param='', msg='API', data={}, query={}, fatal=True):
+        if not Cryptodome:
+            raise ExtractorError('pycryptodomex not found. Please install', expected=True)
+        private_key = Cryptodome.PublicKey.RSA.generate(2048)
+        cipher = Cryptodome.Cipher.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.Hash.SHA1)
+
+        def decrypt(data):
+            if not data:
+                return None
+            try:
+                return cipher.decrypt(base64.b64decode(data)).decode()
+            except (ValueError, binascii.Error) as e:
+                raise ExtractorError(f'Could not decrypt data: {e}')
+
+        token = base64.b64encode(private_key.public_key().export_key('DER')).decode()
+        api_json = self._call_api(video_id, param, msg, data={
+            # 'deviceId' (random uuid4 generated at login) is not required yet
+            'token': token,
+            **data,
+        }, query=query, fatal=fatal)
+        return api_json, decrypt
+
+    def _download_metadata(self, url, video_id, lang, props_key):
+        metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
+        if not metadata:
+            webpage = self._download_webpage(url, video_id)
+            nextjs_data = self._search_nextjs_data(webpage, video_id)
+            metadata = traverse_obj(nextjs_data, ('props', 'pageProps', props_key, {dict})) or {}
+        return metadata
+
+    def _get_formats(self, data, path, video_id=None):
+        hls_url = traverse_obj(data, path, get_all=False)
+        if not hls_url and not data.get('canWatch'):
+            self.raise_no_formats(
+                'This account does not have access to the requested content', expected=True)
+        elif not hls_url:
+            self.raise_no_formats('No supported formats found')
+        return self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', live=True)
+
+
+class WrestleUniverseVODIE(WrestleUniverseBaseIE):
+    _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'videos'
+    _TESTS = [{
+        'url': 'https://www.wrestle-universe.com/en/videos/dp8mpjmcKfxzUhEHM2uFws',
+        'info_dict': {
+            'id': 'dp8mpjmcKfxzUhEHM2uFws',
+            'ext': 'mp4',
+            'title': 'The 3rd “Futari wa Princess” Max Heart Tournament',
+            'description': 'md5:318d5061e944797fbbb81d5c7dd00bf5',
+            'location': '埼玉・春日部ふれあいキューブ',
+            'channel': 'tjpw',
+            'duration': 7119,
+            'timestamp': 1674979200,
+            'upload_date': '20230129',
+            'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN',
+            'chapters': 'count:7',
+            'cast': 'count:18',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }]
+
+    _API_PATH = 'videoEpisodes'
+
+    def _real_extract(self, url):
+        lang, video_id = self._match_valid_url(url).group('lang', 'id')
+        metadata = self._download_metadata(url, video_id, lang, 'videoEpisodeFallbackData')
+        video_data = self._call_api(video_id, ':watch', 'watch', data={
+            # 'deviceId' is required if ignoreDeviceRestriction is False
+            'ignoreDeviceRestriction': True,
+        })
+
+        return {
+            'id': video_id,
+            'formats': self._get_formats(video_data, (
+                (('protocolHls', 'url'), ('chromecastUrls', ...)), {url_or_none}), video_id),
+            **traverse_obj(metadata, {
+                'title': ('displayName', {str}),
+                'description': ('description', {str}),
+                'channel': ('labels', 'group', {str}),
+                'location': ('labels', 'venue', {str}),
+                'timestamp': ('watchStartTime', {int_or_none}),
+                'thumbnail': ('keyVisualUrl', {url_or_none}),
+                'cast': ('casts', ..., 'displayName', {str}),
+                'duration': ('duration', {int}),
+                'chapters': ('videoChapters', lambda _, v: isinstance(v.get('start'), int), {
+                    'title': ('displayName', {str}),
+                    'start_time': ('start', {int}),
+                    'end_time': ('end', {int}),
+                }),
+            }),
+        }
+
+
+class WrestleUniversePPVIE(WrestleUniverseBaseIE):
+    _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'lives'
+    _TESTS = [{
+        'note': 'HLS AES-128 key obtained via API',
+        'url': 'https://www.wrestle-universe.com/en/lives/buH9ibbfhdJAY4GKZcEuJX',
+        'info_dict': {
+            'id': 'buH9ibbfhdJAY4GKZcEuJX',
+            'ext': 'mp4',
+            'title': '【PPV】Beyond the origins, into the future',
+            'description': 'md5:9a872db68cd09be4a1e35a3ee8b0bdfc',
+            'channel': 'tjpw',
+            'location': '東京・Twin Box AKIHABARA',
+            'duration': 10098,
+            'timestamp': 1675076400,
+            'upload_date': '20230130',
+            'thumbnail': 'https://image.asset.wrestle-universe.com/rJs2m7cBaLXrwCcxMdQGRM/rJs2m7cBaLXrwCcxMdQGRM',
+            'thumbnails': 'count:3',
+            'hls_aes': {
+                'key': '5633184acd6e43f1f1ac71c6447a4186',
+                'iv': '5bac71beb33197d5600337ce86de7862',
+            },
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }, {
+        'note': 'unencrypted HLS',
+        'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx',
+        'info_dict': {
+            'id': 'wUG8hP5iApC63jbtQzhVVx',
+            'ext': 'mp4',
+            'title': 'GRAND PRINCESS \'22',
+            'description': 'md5:e4f43d0d4262de3952ff34831bc99858',
+            'channel': 'tjpw',
+            'location': '東京・両国国技館',
+            'duration': 18044,
+            'timestamp': 1647665400,
+            'upload_date': '20220319',
+            'thumbnail': 'https://image.asset.wrestle-universe.com/i8jxSTCHPfdAKD4zN41Psx/i8jxSTCHPfdAKD4zN41Psx',
+            'thumbnails': 'count:3',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }]
+
+    _API_PATH = 'events'
+
+    def _real_extract(self, url):
+        lang, video_id = self._match_valid_url(url).group('lang', 'id')
+        metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData')
+
+        info = traverse_obj(metadata, {
+            'title': ('displayName', {str}),
+            'description': ('description', {str}),
+            'channel': ('labels', 'group', {str}),
+            'location': ('labels', 'venue', {str}),
+            'timestamp': ('startTime', {int_or_none}),
+            'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
+        })
+
+        ended_time = traverse_obj(metadata, ('endedTime', {int_or_none}))
+        if info.get('timestamp') and ended_time:
+            info['duration'] = ended_time - info['timestamp']
+
+        video_data, decrypt = self._call_encrypted_api(
+            video_id, ':watchArchive', 'watch archive', data={'method': 1})
+        formats = self._get_formats(video_data, (
+            ('hls', None), ('urls', 'chromecastUrls'), ..., {url_or_none}), video_id)
+        for f in formats:
+            # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
+            if f.get('tbr'):
+                f['tbr'] = int(f['tbr'] / 2.5)
+
+        hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt}))
+        if not hls_aes_key and traverse_obj(video_data, ('hls', 'encryptType', {int}), default=0) > 0:
+            self.report_warning('HLS AES-128 key was not found in API response')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'hls_aes': {
+                'key': hls_aes_key,
+                'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
+            },
+            **info,
+        }

From f40e32fb1ac67be5bdbc8e32a3c235abfc4be260 Mon Sep 17 00:00:00 2001
From: Stefan Lobbenmeier <Stefan.Lobbenmeier@gmail.com>
Date: Wed, 8 Feb 2023 07:05:32 +0100
Subject: [PATCH 028/871] [extractor/servus] Rewrite extractor (#6036)

Closes #1076, closes #4240, closes #2748, closes #1045, closes #1498
Authored by: FrankZ85, Ashish0804, StefanLobbenmeier

Co-authored-by: FrankZ85 <43293037+FrankZ85@users.noreply.github.com>
---
 yt_dlp/extractor/servus.py | 169 ++++++++++++++++++-------------------
 1 file changed, 80 insertions(+), 89 deletions(-)

diff --git a/yt_dlp/extractor/servus.py b/yt_dlp/extractor/servus.py
index 490d56267..dda195883 100644
--- a/yt_dlp/extractor/servus.py
+++ b/yt_dlp/extractor/servus.py
@@ -1,11 +1,13 @@
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
+    ExtractorError,
     float_or_none,
+    format_field,
     int_or_none,
+    join_nonempty,
+    traverse_obj,
+    unescapeHTML,
     unified_timestamp,
-    urlencode_postdata,
-    url_or_none,
 )
 
 
@@ -15,32 +17,41 @@ class ServusIE(InfoExtractor):
                         (?:www\.)?
                         (?:
                             servus\.com/(?:(?:at|de)/p/[^/]+|tv/videos)|
-                            (?:servustv|pm-wissen)\.com/videos
+                            (?:servustv|pm-wissen)\.com/(?:[^/]+/)?v(?:ideos)?
                         )
-                        /(?P<id>[aA]{2}-\w+|\d+-\d+)
+                        /(?P<id>[aA]{2}-?\w+|\d+-\d+)
                     '''
     _TESTS = [{
-        # new URL schema
-        'url': 'https://www.servustv.com/videos/aa-1t6vbu5pw1w12/',
-        'md5': '60474d4c21f3eb148838f215c37f02b9',
+        # URL schema v3
+        'url': 'https://www.servustv.com/natur/v/aa-28bycqnh92111/',
         'info_dict': {
-            'id': 'AA-1T6VBU5PW1W12',
+            'id': 'AA-28BYCQNH92111',
             'ext': 'mp4',
-            'title': 'Die Grünen aus Sicht des Volkes',
-            'alt_title': 'Talk im Hangar-7 Voxpops Gruene',
-            'description': 'md5:1247204d85783afe3682644398ff2ec4',
+            'title': 'Klettersteige in den Alpen',
+            'description': 'md5:25e47ddd83a009a0f9789ba18f2850ce',
             'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 62.442,
-            'timestamp': 1605193976,
-            'upload_date': '20201112',
-            'series': 'Talk im Hangar-7',
-            'season': 'Season 9',
-            'season_number': 9,
-            'episode': 'Episode 31 - September 14',
-            'episode_number': 31,
-        }
+            'duration': 2823,
+            'timestamp': 1655752333,
+            'upload_date': '20220620',
+            'series': 'Bergwelten',
+            'season': 'Season 11',
+            'season_number': 11,
+            'episode': 'Episode 8 - Vie Ferrate – Klettersteige in den Alpen',
+            'episode_number': 8,
+        },
+        'params': {'skip_download': 'm3u8'}
     }, {
-        # old URL schema
+        'url': 'https://www.servustv.com/natur/v/aa-1xg5xwmgw2112/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.servustv.com/natur/v/aansszcx3yi9jmlmhdc1/',
+        'only_matching': True,
+    }, {
+        # URL schema v2
+        'url': 'https://www.servustv.com/videos/aa-1t6vbu5pw1w12/',
+        'only_matching': True,
+    }, {
+        # URL schema v1
         'url': 'https://www.servus.com/de/p/Die-Gr%C3%BCnen-aus-Sicht-des-Volkes/AA-1T6VBU5PW1W12/',
         'only_matching': True,
     }, {
@@ -60,85 +71,65 @@ class ServusIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url).upper()
 
-        token = self._download_json(
-            'https://auth.redbullmediahouse.com/token', video_id,
-            'Downloading token', data=urlencode_postdata({
-                'grant_type': 'client_credentials',
-            }), headers={
-                'Authorization': 'Basic SVgtMjJYNEhBNFdEM1cxMTpEdDRVSkFLd2ZOMG5IMjB1NGFBWTBmUFpDNlpoQ1EzNA==',
-            })
-        access_token = token['access_token']
-        token_type = token.get('token_type', 'Bearer')
-
         video = self._download_json(
-            'https://sparkle-api.liiift.io/api/v1/stv/channels/international/assets/%s' % video_id,
-            video_id, 'Downloading video JSON', headers={
-                'Authorization': '%s %s' % (token_type, access_token),
-            })
+            'https://api-player.redbull.com/stv/servus-tv?timeZone=Europe/Berlin',
+            video_id, 'Downloading video JSON', query={'videoId': video_id})
+        if not video.get('videoUrl'):
+            self._report_errors(video)
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            video['videoUrl'], video_id, 'mp4', m3u8_id='hls')
 
-        formats = []
-        thumbnail = None
-        for resource in video['resources']:
-            if not isinstance(resource, dict):
-                continue
-            format_url = url_or_none(resource.get('url'))
-            if not format_url:
-                continue
-            extension = resource.get('extension')
-            type_ = resource.get('type')
-            if extension == 'jpg' or type_ == 'reference_keyframe':
-                thumbnail = format_url
-                continue
-            ext = determine_ext(format_url)
-            if type_ == 'dash' or ext == 'mpd':
-                formats.extend(self._extract_mpd_formats(
-                    format_url, video_id, mpd_id='dash', fatal=False))
-            elif type_ == 'hls' or ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False))
-            elif extension == 'mp4' or ext == 'mp4':
-                formats.append({
-                    'url': format_url,
-                    'format_id': type_,
-                    'width': int_or_none(resource.get('width')),
-                    'height': int_or_none(resource.get('height')),
-                })
-
-        attrs = {}
-        for attribute in video['attributes']:
-            if not isinstance(attribute, dict):
-                continue
-            key = attribute.get('fieldKey')
-            value = attribute.get('fieldValue')
-            if not key or not value:
-                continue
-            attrs[key] = value
-
-        title = attrs.get('title_stv') or video_id
-        alt_title = attrs.get('title')
-        description = attrs.get('long_description') or attrs.get('short_description')
-        series = attrs.get('label')
-        season = attrs.get('season')
-        episode = attrs.get('chapter')
-        duration = float_or_none(attrs.get('duration'), scale=1000)
+        season = video.get('season')
         season_number = int_or_none(self._search_regex(
             r'Season (\d+)', season or '', 'season number', default=None))
+        episode = video.get('chapter')
         episode_number = int_or_none(self._search_regex(
             r'Episode (\d+)', episode or '', 'episode number', default=None))
 
         return {
             'id': video_id,
-            'title': title,
-            'alt_title': alt_title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': unified_timestamp(video.get('lastPublished')),
-            'series': series,
+            'title': video.get('title'),
+            'description': self._get_description(video_id) or video.get('description'),
+            'thumbnail': video.get('poster'),
+            'duration': float_or_none(video.get('duration')),
+            'timestamp': unified_timestamp(video.get('currentSunrise')),
+            'series': video.get('label'),
             'season': season,
             'season_number': season_number,
             'episode': episode,
             'episode_number': episode_number,
             'formats': formats,
+            'subtitles': subtitles,
         }
+
+    def _get_description(self, video_id):
+        info = self._download_json(
+            f'https://backend.servustv.com/wp-json/rbmh/v2/media_asset/aa_id/{video_id}?fieldset=page',
+            video_id, fatal=False)
+
+        return join_nonempty(*traverse_obj(info, (
+            ('stv_short_description', 'stv_long_description'),
+            {lambda x: unescapeHTML(x.replace('\n\n', '\n'))})), delim='\n\n')
+
+    def _report_errors(self, video):
+        playability_errors = traverse_obj(video, ('playabilityErrors', ...))
+        if not playability_errors:
+            raise ExtractorError('No videoUrl and no information about errors')
+
+        elif 'FSK_BLOCKED' in playability_errors:
+            details = traverse_obj(video, ('playabilityErrorDetails', 'FSK_BLOCKED'), expected_type=dict)
+            message = format_field(''.join((
+                format_field(details, 'minEveningHour', ' from %02d:00'),
+                format_field(details, 'maxMorningHour', ' to %02d:00'),
+                format_field(details, 'minAge', ' (Minimum age %d)'),
+            )), None, 'Only available%s') or 'Blocked by FSK with unknown availability'
+
+        elif 'NOT_YET_AVAILABLE' in playability_errors:
+            message = format_field(
+                video, (('playabilityErrorDetails', 'NOT_YET_AVAILABLE', 'availableFrom'), 'currentSunrise'),
+                'Only available from %s') or 'Video not yet available with unknown availability'
+
+        else:
+            message = f'Video unavailable: {", ".join(playability_errors)}'
+
+        raise ExtractorError(message, expected=True)

From acb1042a9ffa8769fe691beac1011d6da1fcf321 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Feb 2023 01:12:08 +0530
Subject: [PATCH 029/871] [devscripts] Provide pyinstaller hooks

Closes #6185
---
 pyinst.py                           | 26 +-------------------------
 setup.py                            |  5 ++++-
 yt_dlp/__pyinstaller/__init__.py    |  5 +++++
 yt_dlp/__pyinstaller/hook-yt_dlp.py | 29 +++++++++++++++++++++++++++++
 yt_dlp/compat/__init__.py           |  6 ------
 5 files changed, 39 insertions(+), 32 deletions(-)
 create mode 100644 yt_dlp/__pyinstaller/__init__.py
 create mode 100644 yt_dlp/__pyinstaller/hook-yt_dlp.py

diff --git a/pyinst.py b/pyinst.py
index 17c950563..22df672c9 100644
--- a/pyinst.py
+++ b/pyinst.py
@@ -37,7 +37,7 @@ def main():
         '--icon=devscripts/logo.ico',
         '--upx-exclude=vcruntime140.dll',
         '--noconfirm',
-        *dependency_options(),
+        '--additional-hooks-dir=yt_dlp/__pyinstaller',
         *opts,
         'yt_dlp/__main__.py',
     ]
@@ -77,30 +77,6 @@ def version_to_list(version):
     return list(map(int, version_list)) + [0] * (4 - len(version_list))
 
 
-def dependency_options():
-    # Due to the current implementation, these are auto-detected, but explicitly add them just in case
-    dependencies = [pycryptodome_module(), 'mutagen', 'brotli', 'certifi', 'websockets']
-    excluded_modules = ('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts')
-
-    yield from (f'--hidden-import={module}' for module in dependencies)
-    yield '--collect-submodules=websockets'
-    yield from (f'--exclude-module={module}' for module in excluded_modules)
-
-
-def pycryptodome_module():
-    try:
-        import Cryptodome  # noqa: F401
-    except ImportError:
-        try:
-            import Crypto  # noqa: F401
-            print('WARNING: Using Crypto since Cryptodome is not available. '
-                  'Install with: pip install pycryptodomex', file=sys.stderr)
-            return 'Crypto'
-        except ImportError:
-            pass
-    return 'Cryptodome'
-
-
 def set_version_info(exe, version):
     if OS_NAME == 'win32':
         windows_set_version(exe, version)
diff --git a/setup.py b/setup.py
index e2520ff6f..ccfcf4252 100644
--- a/setup.py
+++ b/setup.py
@@ -92,7 +92,10 @@ def build_params():
     params = {'data_files': data_files}
 
     if setuptools_available:
-        params['entry_points'] = {'console_scripts': ['yt-dlp = yt_dlp:main']}
+        params['entry_points'] = {
+            'console_scripts': ['yt-dlp = yt_dlp:main'],
+            'pyinstaller40': ['hook-dirs = yt_dlp.__pyinstaller:get_hook_dirs'],
+        }
     else:
         params['scripts'] = ['yt-dlp']
     return params
diff --git a/yt_dlp/__pyinstaller/__init__.py b/yt_dlp/__pyinstaller/__init__.py
new file mode 100644
index 000000000..1c52aadf4
--- /dev/null
+++ b/yt_dlp/__pyinstaller/__init__.py
@@ -0,0 +1,5 @@
+import os
+
+
+def get_hook_dirs():
+    return [os.path.dirname(__file__)]
diff --git a/yt_dlp/__pyinstaller/hook-yt_dlp.py b/yt_dlp/__pyinstaller/hook-yt_dlp.py
new file mode 100644
index 000000000..66d1b6369
--- /dev/null
+++ b/yt_dlp/__pyinstaller/hook-yt_dlp.py
@@ -0,0 +1,29 @@
+import sys
+
+from PyInstaller.utils.hooks import collect_submodules
+
+
+def _pycryptodome_module():
+    try:
+        import Cryptodome  # noqa: F401
+    except ImportError:
+        try:
+            import Crypto  # noqa: F401
+            print('WARNING: Using Crypto since Cryptodome is not available. '
+                  'Install with: pip install pycryptodomex', file=sys.stderr)
+            return 'Crypto'
+        except ImportError:
+            pass
+    return 'Cryptodome'
+
+
+def _hidden_imports():
+    yield 'yt_dlp.compat._legacy'
+    for m in [_pycryptodome_module(), 'websockets']:
+        yield from collect_submodules(m)
+    # These are auto-detected, but explicitly add them just in case
+    yield from ('mutagen', 'brotli', 'certifi')
+
+
+hiddenimports = list(_hidden_imports())
+excludedimports = ['youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts']
diff --git a/yt_dlp/compat/__init__.py b/yt_dlp/compat/__init__.py
index 5cc78ebc2..c6c02541c 100644
--- a/yt_dlp/compat/__init__.py
+++ b/yt_dlp/compat/__init__.py
@@ -70,9 +70,3 @@ def compat_expanduser(path):
         return userhome + path[i:]
 else:
     compat_expanduser = os.path.expanduser
-
-
-# NB: Add modules that are imported dynamically here so that PyInstaller can find them
-# See https://github.com/pyinstaller/pyinstaller-hooks-contrib/issues/438
-if False:
-    from . import _legacy  # noqa: F401

From 768a00178109508893488e53a0e720b117fbccf6 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Feb 2023 01:34:39 +0530
Subject: [PATCH 030/871] [compat_utils] Simplify `EnhancedModule`

---
 yt_dlp/compat/compat_utils.py     | 18 ++----------------
 yt_dlp/dependencies/Cryptodome.py |  8 +++++---
 2 files changed, 7 insertions(+), 19 deletions(-)

diff --git a/yt_dlp/compat/compat_utils.py b/yt_dlp/compat/compat_utils.py
index f8679c98e..8956b3bf1 100644
--- a/yt_dlp/compat/compat_utils.py
+++ b/yt_dlp/compat/compat_utils.py
@@ -28,20 +28,6 @@ def _is_dunder(name):
 
 
 class EnhancedModule(types.ModuleType):
-    def __new__(cls, name, *args, **kwargs):
-        if name not in sys.modules:
-            return super().__new__(cls, name, *args, **kwargs)
-
-        assert not args and not kwargs, 'Cannot pass additional arguments to an existing module'
-        module = sys.modules[name]
-        module.__class__ = cls
-        return module
-
-    def __init__(self, name, *args, **kwargs):
-        # Prevent __new__ from trigerring __init__ again
-        if name not in sys.modules:
-            super().__init__(name, *args, **kwargs)
-
     def __bool__(self):
         return vars(self).get('__bool__', lambda: True)()
 
@@ -60,8 +46,6 @@ def __getattribute__(self, attr):
 
 def passthrough_module(parent, child, allowed_attributes=(..., ), *, callback=lambda _: None):
     """Passthrough parent module into a child module, creating the parent if necessary"""
-    parent = EnhancedModule(parent)
-
     def __getattr__(attr):
         if _is_package(parent):
             with contextlib.suppress(ImportError):
@@ -93,5 +77,7 @@ def from_child(attr):
 
         return _NO_ATTRIBUTE
 
+    parent = sys.modules.get(parent, types.ModuleType(parent))
+    parent.__class__ = EnhancedModule
     parent.__getattr__ = __getattr__
     return parent
diff --git a/yt_dlp/dependencies/Cryptodome.py b/yt_dlp/dependencies/Cryptodome.py
index 580ce0753..2adc51374 100644
--- a/yt_dlp/dependencies/Cryptodome.py
+++ b/yt_dlp/dependencies/Cryptodome.py
@@ -1,5 +1,7 @@
+import types
+
 from ..compat import functools
-from ..compat.compat_utils import EnhancedModule, passthrough_module
+from ..compat.compat_utils import passthrough_module
 
 try:
     import Cryptodome as _parent
@@ -7,11 +9,11 @@
     try:
         import Crypto as _parent
     except (ImportError, SyntaxError):  # Old Crypto gives SyntaxError in newer Python
-        _parent = EnhancedModule('Cryptodome')
+        _parent = types.ModuleType('no_Cryptodome')
         __bool__ = lambda: False
 
 passthrough_module(__name__, _parent, (..., '__version__'))
-del passthrough_module, EnhancedModule
+del passthrough_module
 
 
 @property

From f14c2333481c63c24017a41ded7d8f36726504b7 Mon Sep 17 00:00:00 2001
From: Ha Tien Loi <loiht.b17vt220@stu.ptit.edu.vn>
Date: Thu, 9 Feb 2023 03:41:04 +0700
Subject: [PATCH 031/871] [extractor/DouyuTV]: Use new API (#6074)

Authored by: hatienl0i261299
---
 yt_dlp/extractor/douyutv.py | 55 +++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 21 deletions(-)

diff --git a/yt_dlp/extractor/douyutv.py b/yt_dlp/extractor/douyutv.py
index 477f4687c..fa40844df 100644
--- a/yt_dlp/extractor/douyutv.py
+++ b/yt_dlp/extractor/douyutv.py
@@ -1,6 +1,7 @@
 import time
 import hashlib
 import re
+import urllib
 
 from .common import InfoExtractor
 from ..utils import (
@@ -13,7 +14,7 @@
 
 class DouyuTVIE(InfoExtractor):
     IE_DESC = '斗鱼'
-    _VALID_URL = r'https?://(?:www\.)?douyu(?:tv)?\.com/(?:[^/]+/)*(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?douyu(?:tv)?\.com/(topic/\w+\?rid=|(?:[^/]+/))*(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
@@ -22,7 +23,7 @@ class DouyuTVIE(InfoExtractor):
             'ext': 'flv',
             'title': 're:^清晨醒脑！根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.png',
             'uploader': '7师傅',
             'is_live': True,
         },
@@ -37,7 +38,7 @@ class DouyuTVIE(InfoExtractor):
             'ext': 'flv',
             'title': 're:^小漠从零单排记！——CSOL2躲猫猫 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': 'md5:746a2f7a253966a06755a912f0acc0d2',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.png',
             'uploader': 'douyu小漠',
             'is_live': True,
         },
@@ -53,13 +54,28 @@ class DouyuTVIE(InfoExtractor):
             'ext': 'flv',
             'title': 're:^清晨醒脑！根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.png',
             'uploader': '7师傅',
             'is_live': True,
         },
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.douyu.com/topic/ydxc?rid=6560603',
+        'info_dict': {
+            'id': '6560603',
+            'display_id': '6560603',
+            'ext': 'flv',
+            'title': 're:^阿余：新年快乐恭喜发财！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 're:.*直播时间.*',
+            'thumbnail': r're:^https?://.*\.png',
+            'uploader': '阿涛皎月Carry',
+            'live_status': 'is_live',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.douyu.com/xiaocang',
         'only_matching': True,
@@ -79,28 +95,24 @@ def _real_extract(self, url):
             room_id = self._html_search_regex(
                 r'"room_id\\?"\s*:\s*(\d+),', page, 'room id')
 
-        # Grab metadata from mobile API
+        # Grab metadata from API
+        params = {
+            'aid': 'wp',
+            'client_sys': 'wp',
+            'time': int(time.time()),
+        }
+        params['auth'] = hashlib.md5(
+            f'room/{video_id}?{urllib.parse.urlencode(params)}zNzMV1y4EMxOHS6I5WKm'.encode()).hexdigest()
         room = self._download_json(
-            'http://m.douyu.com/html5/live?roomId=%s' % room_id, video_id,
-            note='Downloading room info')['data']
+            f'http://www.douyutv.com/api/v1/room/{room_id}', video_id,
+            note='Downloading room info', query=params)['data']
 
         # 1 = live, 2 = offline
         if room.get('show_status') == '2':
             raise ExtractorError('Live stream is offline', expected=True)
 
-        # Grab the URL from PC client API
-        # The m3u8 url from mobile API requires re-authentication every 5 minutes
-        tt = int(time.time())
-        signContent = 'lapi/live/thirdPart/getPlay/%s?aid=pcclient&rate=0&time=%d9TUk5fjjUjg9qIMH3sdnh' % (room_id, tt)
-        sign = hashlib.md5(signContent.encode('ascii')).hexdigest()
-        video_url = self._download_json(
-            'http://coapi.douyucdn.cn/lapi/live/thirdPart/getPlay/' + room_id,
-            video_id, note='Downloading video URL info',
-            query={'rate': 0}, headers={
-                'auth': sign,
-                'time': str(tt),
-                'aid': 'pcclient'
-            })['data']['live_url']
+        video_url = urljoin('https://hls3-akm.douyucdn.cn/', self._search_regex(r'(live/.*)', room['hls_url'], 'URL'))
+        formats, subs = self._extract_m3u8_formats_and_subtitles(video_url, room_id)
 
         title = unescapeHTML(room['room_name'])
         description = room.get('show_details')
@@ -110,12 +122,13 @@ def _real_extract(self, url):
         return {
             'id': room_id,
             'display_id': video_id,
-            'url': video_url,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
             'is_live': True,
+            'subtitles': subs,
+            'formats': formats,
         }
 
 

From c0cd13fb1c71b842c3d272d0273c03542b467766 Mon Sep 17 00:00:00 2001
From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com>
Date: Fri, 10 Feb 2023 14:50:29 +0100
Subject: [PATCH 032/871] [extractor/vimeo] Fix `playerConfig` extraction
 (#6203)

Authored by: bashonly, LeoniePhiline
Closes #6149
---
 yt_dlp/extractor/vimeo.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py
index 97b99fc50..88d143e71 100644
--- a/yt_dlp/extractor/vimeo.py
+++ b/yt_dlp/extractor/vimeo.py
@@ -358,7 +358,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         },
         {
             'url': 'http://player.vimeo.com/video/54469442',
-            'md5': 'b3e7f4d2cbb53bd7dc3bb6ff4ed5cfbd',
+            'md5': '619b811a4417aa4abe78dc653becf511',
             'note': 'Videos that embed the url in the player page',
             'info_dict': {
                 'id': '54469442',
@@ -389,8 +389,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user18948128',
                 'uploader': 'Jaime Marquínez Ferrándiz',
                 'duration': 10,
-                'description': 'md5:dca3ea23adb29ee387127bc4ddfce63f',
-                'thumbnail': 'https://i.vimeocdn.com/video/440665496-b2c5aee2b61089442c794f64113a8e8f7d5763c3e6b3ebfaf696ae6413f8b1f4-d_960',
+                'description': 'md5:6173f270cd0c0119f22817204b3eb86c',
+                'thumbnail': 'https://i.vimeocdn.com/video/440665496-b2c5aee2b61089442c794f64113a8e8f7d5763c3e6b3ebfaf696ae6413f8b1f4-d_1280',
                 'view_count': int,
                 'comment_count': int,
                 'like_count': int,
@@ -407,7 +407,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'id': '75629013',
                 'ext': 'mp4',
                 'title': 'Key & Peele: Terrorist Interrogation',
-                'description': 'md5:8678b246399b070816b12313e8b4eb5c',
+                'description': 'md5:6173f270cd0c0119f22817204b3eb86c',
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/atencio',
                 'uploader_id': 'atencio',
                 'uploader': 'Peter Atencio',
@@ -559,8 +559,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user18948128',
                 'uploader': 'Jaime Marquínez Ferrándiz',
                 'duration': 10,
-                'description': 'md5:dca3ea23adb29ee387127bc4ddfce63f',
-                'thumbnail': 'https://i.vimeocdn.com/video/440665496-b2c5aee2b61089442c794f64113a8e8f7d5763c3e6b3ebfaf696ae6413f8b1f4-d_960',
+                'description': 'md5:6173f270cd0c0119f22817204b3eb86c',
+                'thumbnail': 'https://i.vimeocdn.com/video/440665496-b2c5aee2b61089442c794f64113a8e8f7d5763c3e6b3ebfaf696ae6413f8b1f4-d_1280',
                 'view_count': int,
                 'comment_count': int,
                 'like_count': int,
@@ -834,8 +834,8 @@ def _real_extract(self, url):
             raise
 
         if '://player.vimeo.com/video/' in url:
-            config = self._parse_json(self._search_regex(
-                r'\b(?:playerC|c)onfig\s*=\s*({.+?})\s*;', webpage, 'info section'), video_id)
+            config = self._search_json(
+                r'\b(?:playerC|c)onfig\s*=', webpage, 'info section', video_id)
             if config.get('view') == 4:
                 config = self._verify_player_video_password(
                     redirect_url, video_id, headers)

From 6839ae1f6dde4c0442619e351b3f0442312ab4f9 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Fri, 10 Feb 2023 03:56:26 +0530
Subject: [PATCH 033/871] [utils] `traverse_obj`: Fix more bugs

and cleanup uses of `default=[]`

Continued from b1bde57bef878478e3503ab07190fd207914ade9
---
 test/test_utils.py            | 75 ++++++++++++++++++++++-------------
 yt_dlp/downloader/fragment.py |  2 +-
 yt_dlp/extractor/abematv.py   |  4 +-
 yt_dlp/extractor/gamejolt.py  |  2 +-
 yt_dlp/extractor/iqiyi.py     |  8 ++--
 yt_dlp/extractor/panopto.py   |  4 +-
 yt_dlp/extractor/patreon.py   |  2 +-
 yt_dlp/extractor/tiktok.py    |  4 +-
 yt_dlp/extractor/youtube.py   | 35 ++++++++--------
 yt_dlp/utils.py               | 15 ++++---
 10 files changed, 84 insertions(+), 67 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 190e4ef9b..3045b6d7e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2000,7 +2000,7 @@ def test_traverse_obj(self):
 
         # Test Ellipsis behavior
         self.assertCountEqual(traverse_obj(_TEST_DATA, ...),
-                              (item for item in _TEST_DATA.values() if item not in (None, [], {})),
+                              (item for item in _TEST_DATA.values() if item not in (None, {})),
                               msg='`...` should give all non discarded values')
         self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, ...)), _TEST_DATA['urls'][0].values(),
                               msg='`...` selection for dicts should select all values')
@@ -2095,7 +2095,7 @@ def test_traverse_obj(self):
                          msg='remove empty values when nested dict key fails')
         self.assertEqual(traverse_obj(None, {0: 'fail'}), {},
                          msg='default to dict if pruned')
-        self.assertEqual(traverse_obj(None, {0: 'fail'}, default=...), {},
+        self.assertEqual(traverse_obj(None, {0: 'fail'}, default=...), {0: ...},
                          msg='default to dict if pruned and default is given')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}, default=...), {0: {0: ...}},
                          msg='use nested `default` when nested dict key fails and `default`')
@@ -2124,34 +2124,55 @@ def test_traverse_obj(self):
                          msg='if branched but not successful return `[]`, not `default`')
         self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', ...)), [],
                          msg='if branched but object is empty return `[]`, not `default`')
+        self.assertEqual(traverse_obj(None, ...), [],
+                         msg='if branched but object is `None` return `[]`, not `default`')
+        self.assertEqual(traverse_obj({0: None}, (0, ...)), [],
+                         msg='if branched but state is `None` return `[]`, not `default`')
+
+        branching_paths = [
+            ('fail', ...),
+            (..., 'fail'),
+            100 * ('fail',) + (...,),
+            (...,) + 100 * ('fail',),
+        ]
+        for branching_path in branching_paths:
+            self.assertEqual(traverse_obj({}, branching_path), [],
+                             msg='if branched but state is `None`, return `[]` (not `default`)')
+            self.assertEqual(traverse_obj({}, 'fail', branching_path), [],
+                             msg='if branching in last alternative and previous did not match, return `[]` (not `default`)')
+            self.assertEqual(traverse_obj({0: 'x'}, 0, branching_path), 'x',
+                             msg='if branching in last alternative and previous did match, return single value')
+            self.assertEqual(traverse_obj({0: 'x'}, branching_path, 0), 'x',
+                             msg='if branching in first alternative and non-branching path does match, return single value')
+            self.assertEqual(traverse_obj({}, branching_path, 'fail'), None,
+                             msg='if branching in first alternative and non-branching path does not match, return `default`')
 
         # Testing expected_type behavior
         _EXPECTED_TYPE_DATA = {'str': 'str', 'int': 0}
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=str), 'str',
-                         msg='accept matching `expected_type` type')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int), None,
-                         msg='reject non matching `expected_type` type')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: str(x)), '0',
-                         msg='transform type using type function')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str',
-                                      expected_type=lambda _: 1 / 0), None,
-                         msg='wrap expected_type fuction in try_call')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, ..., expected_type=str), ['str'],
-                         msg='eliminate items that expected_type fails on')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}, expected_type=int), {0: 100},
-                         msg='type as expected_type should filter dict values')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2, 2: 'None'}, expected_type=str_or_none), {0: '100', 1: '1.2'},
-                         msg='function as expected_type should transform dict values')
-        self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, {int_or_none}), expected_type=int), 1,
-                         msg='expected_type should not filter non final dict values')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 100, 1: 'str'}}, expected_type=int), {0: {0: 100}},
-                         msg='expected_type should transform deep dict values')
-        self.assertEqual(traverse_obj(_TEST_DATA, [({0: '...'}, {0: '...'})], expected_type=type(...)), [{0: ...}, {0: ...}],
-                         msg='expected_type should transform branched dict values')
-        self.assertEqual(traverse_obj({1: {3: 4}}, [(1, 2), 3], expected_type=int), [4],
-                         msg='expected_type regression for type matching in tuple branching')
-        self.assertEqual(traverse_obj(_TEST_DATA, ['data', ...], expected_type=int), [],
-                         msg='expected_type regression for type matching in dict result')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=str),
+                         'str', msg='accept matching `expected_type` type')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int),
+                         None, msg='reject non matching `expected_type` type')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: str(x)),
+                         '0', msg='transform type using type function')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=lambda _: 1 / 0),
+                         None, msg='wrap expected_type fuction in try_call')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, ..., expected_type=str),
+                         ['str'], msg='eliminate items that expected_type fails on')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}, expected_type=int),
+                         {0: 100}, msg='type as expected_type should filter dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2, 2: 'None'}, expected_type=str_or_none),
+                         {0: '100', 1: '1.2'}, msg='function as expected_type should transform dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, {int_or_none}), expected_type=int),
+                         1, msg='expected_type should not filter non final dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 100, 1: 'str'}}, expected_type=int),
+                         {0: {0: 100}}, msg='expected_type should transform deep dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, [({0: '...'}, {0: '...'})], expected_type=type(...)),
+                         [{0: ...}, {0: ...}], msg='expected_type should transform branched dict values')
+        self.assertEqual(traverse_obj({1: {3: 4}}, [(1, 2), 3], expected_type=int),
+                         [4], msg='expected_type regression for type matching in tuple branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, ['data', ...], expected_type=int),
+                         [], msg='expected_type regression for type matching in dict result')
 
         # Test get_all behavior
         _GET_ALL_DATA = {'key': [0, 1, 2]}
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 02f8559cc..039cb1492 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -383,7 +383,7 @@ def download_and_append_fragments_multiple(self, *args, **kwargs):
         max_workers = self.params.get('concurrent_fragment_downloads', 1)
         if max_progress > 1:
             self._prepare_multiline_status(max_progress)
-        is_live = any(traverse_obj(args, (..., 2, 'is_live'), default=[]))
+        is_live = any(traverse_obj(args, (..., 2, 'is_live')))
 
         def thread_func(idx, ctx, fragments, info_dict, tpe):
             ctx['max_progress'] = max_progress
diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index 9955fb289..7552e3e57 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -416,7 +416,7 @@ def _real_extract(self, url):
                 f'https://api.abema.io/v1/video/programs/{video_id}', video_id,
                 note='Checking playability',
                 headers=headers)
-            ondemand_types = traverse_obj(api_response, ('terms', ..., 'onDemandType'), default=[])
+            ondemand_types = traverse_obj(api_response, ('terms', ..., 'onDemandType'))
             if 3 not in ondemand_types:
                 # cannot acquire decryption key for these streams
                 self.report_warning('This is a premium-only stream')
@@ -489,7 +489,7 @@ def _fetch_page(self, playlist_id, series_version, page):
             })
         yield from (
             self.url_result(f'https://abema.tv/video/episode/{x}')
-            for x in traverse_obj(programs, ('programs', ..., 'id'), default=[]))
+            for x in traverse_obj(programs, ('programs', ..., 'id')))
 
     def _entries(self, playlist_id, series_version):
         return OnDemandPagedList(
diff --git a/yt_dlp/extractor/gamejolt.py b/yt_dlp/extractor/gamejolt.py
index 440b832fc..8ec046bb3 100644
--- a/yt_dlp/extractor/gamejolt.py
+++ b/yt_dlp/extractor/gamejolt.py
@@ -48,7 +48,7 @@ def _get_comments(self, post_num_id, post_hash_id):
                 post_hash_id, note='Downloading comments list page %d' % page)
             if not comments_data.get('comments'):
                 break
-            for comment in traverse_obj(comments_data, (('comments', 'childComments'), ...), expected_type=dict, default=[]):
+            for comment in traverse_obj(comments_data, (('comments', 'childComments'), ...), expected_type=dict):
                 yield {
                     'id': comment['id'],
                     'text': self._parse_content_as_text(
diff --git a/yt_dlp/extractor/iqiyi.py b/yt_dlp/extractor/iqiyi.py
index eba89f787..4443b1991 100644
--- a/yt_dlp/extractor/iqiyi.py
+++ b/yt_dlp/extractor/iqiyi.py
@@ -585,7 +585,7 @@ def _real_extract(self, url):
                     'langCode': self._get_cookie('lang', 'en_us'),
                     'deviceId': self._get_cookie('QC005', '')
                 }, fatal=False)
-            ut_list = traverse_obj(vip_data, ('data', 'all_vip', ..., 'vipType'), expected_type=str_or_none, default=[])
+            ut_list = traverse_obj(vip_data, ('data', 'all_vip', ..., 'vipType'), expected_type=str_or_none)
         else:
             ut_list = ['0']
 
@@ -617,7 +617,7 @@ def _real_extract(self, url):
             self.report_warning('This preview video is limited%s' % format_field(preview_time, None, ' to %s seconds'))
 
         # TODO: Extract audio-only formats
-        for bid in set(traverse_obj(initial_format_data, ('program', 'video', ..., 'bid'), expected_type=str_or_none, default=[])):
+        for bid in set(traverse_obj(initial_format_data, ('program', 'video', ..., 'bid'), expected_type=str_or_none)):
             dash_path = dash_paths.get(bid)
             if not dash_path:
                 self.report_warning(f'Unknown format id: {bid}. It is currently not being extracted')
@@ -628,7 +628,7 @@ def _real_extract(self, url):
                 fatal=False), 'data', expected_type=dict)
 
             video_format = traverse_obj(format_data, ('program', 'video', lambda _, v: str(v['bid']) == bid),
-                                        expected_type=dict, default=[], get_all=False) or {}
+                                        expected_type=dict, get_all=False) or {}
             extracted_formats = []
             if video_format.get('m3u8Url'):
                 extracted_formats.extend(self._extract_m3u8_formats(
@@ -669,7 +669,7 @@ def _real_extract(self, url):
                 })
             formats.extend(extracted_formats)
 
-        for sub_format in traverse_obj(initial_format_data, ('program', 'stl', ...), expected_type=dict, default=[]):
+        for sub_format in traverse_obj(initial_format_data, ('program', 'stl', ...), expected_type=dict):
             lang = self._LID_TAGS.get(str_or_none(sub_format.get('lid')), sub_format.get('_name'))
             subtitles.setdefault(lang, []).extend([{
                 'ext': format_ext,
diff --git a/yt_dlp/extractor/panopto.py b/yt_dlp/extractor/panopto.py
index 32c103bc1..6e3c9f442 100644
--- a/yt_dlp/extractor/panopto.py
+++ b/yt_dlp/extractor/panopto.py
@@ -412,7 +412,7 @@ def _real_extract(self, url):
         return {
             'id': video_id,
             'title': delivery.get('SessionName'),
-            'cast': traverse_obj(delivery, ('Contributors', ..., 'DisplayName'), default=[], expected_type=lambda x: x or None),
+            'cast': traverse_obj(delivery, ('Contributors', ..., 'DisplayName'), expected_type=lambda x: x or None),
             'timestamp': session_start_time - 11640000000 if session_start_time else None,
             'duration': delivery.get('Duration'),
             'thumbnail': base_url + f'/Services/FrameGrabber.svc/FrameRedirect?objectId={video_id}&mode=Delivery&random={random()}',
@@ -563,7 +563,7 @@ def _extract_folder_metadata(self, base_url, folder_id):
             base_url, '/Services/Data.svc/GetFolderInfo', folder_id,
             data={'folderID': folder_id}, fatal=False)
         return {
-            'title': get_first(response, 'Name', default=[])
+            'title': get_first(response, 'Name')
         }
 
     def _real_extract(self, url):
diff --git a/yt_dlp/extractor/patreon.py b/yt_dlp/extractor/patreon.py
index 529aba178..e93e37eb9 100644
--- a/yt_dlp/extractor/patreon.py
+++ b/yt_dlp/extractor/patreon.py
@@ -310,7 +310,7 @@ def _get_comments(self, post_id):
                 f'posts/{post_id}/comments', post_id, query=params, note='Downloading comments page %d' % page)
 
             cursor = None
-            for comment in traverse_obj(response, (('data', ('included', lambda _, v: v['type'] == 'comment')), ...), default=[]):
+            for comment in traverse_obj(response, (('data', ('included', lambda _, v: v['type'] == 'comment')), ...)):
                 count += 1
                 comment_id = comment.get('id')
                 attributes = comment.get('attributes') or {}
diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index cc96de364..096748bf7 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -285,7 +285,7 @@ def extract_addr(addr, add_meta={}):
         user_url = self._UPLOADER_URL_FORMAT % (traverse_obj(author_info,
                                                              'sec_uid', 'id', 'uid', 'unique_id',
                                                              expected_type=str_or_none, get_all=False))
-        labels = traverse_obj(aweme_detail, ('hybrid_label', ..., 'text'), expected_type=str, default=[])
+        labels = traverse_obj(aweme_detail, ('hybrid_label', ..., 'text'), expected_type=str)
 
         contained_music_track = traverse_obj(
             music_info, ('matched_song', 'title'), ('matched_pgc_sound', 'title'), expected_type=str)
@@ -355,7 +355,7 @@ def _parse_aweme_video_web(self, aweme_detail, webpage_url):
                 'ext': 'mp4',
                 'width': width,
                 'height': height,
-            } for url in traverse_obj(play_url, (..., 'src'), expected_type=url_or_none, default=[]) if url]
+            } for url in traverse_obj(play_url, (..., 'src'), expected_type=url_or_none) if url]
 
         download_url = url_or_none(video_info.get('downloadAddr')) or traverse_obj(video_info, ('download', 'url'), expected_type=url_or_none)
         if download_url:
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index f7b0772df..aff89f8ac 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -745,7 +745,7 @@ def _extract_badges(self, renderer: dict):
         }
 
         badges = []
-        for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer'), default=[]):
+        for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer')):
             badge_type = (
                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
                 or badge_style_map.get(traverse_obj(badge, 'style'))
@@ -785,7 +785,7 @@ def _get_text(data, *path_list, max_runs=None):
                     runs = item
 
                 runs = runs[:min(len(runs), max_runs or len(runs))]
-                text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
+                text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str))
                 if text:
                     return text
 
@@ -805,7 +805,7 @@ def _extract_thumbnails(data, *path_list):
         """
         thumbnails = []
         for path in path_list or [()]:
-            for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
+            for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...)):
                 thumbnail_url = url_or_none(thumbnail.get('url'))
                 if not thumbnail_url:
                     continue
@@ -2668,11 +2668,10 @@ def refetch_manifest(format_id, delay):
                 return
 
             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
-            video_details = traverse_obj(
-                prs, (..., 'videoDetails'), expected_type=dict, default=[])
+            video_details = traverse_obj(prs, (..., 'videoDetails'), expected_type=dict)
             microformats = traverse_obj(
                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
-                expected_type=dict, default=[])
+                expected_type=dict)
             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
             is_live = live_status == 'is_live'
             start_time = time.time()
@@ -3173,7 +3172,7 @@ def _extract_chapters_from_engagement_panel(self, data, duration):
         content_list = traverse_obj(
             data,
             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
-            expected_type=list, default=[])
+            expected_type=list)
         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
         chapter_title = lambda chapter: self._get_text(chapter, 'title')
 
@@ -3450,7 +3449,7 @@ def _is_agegated(player_response):
         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
             return True
 
-        reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
+        reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')))
         AGE_GATE_REASONS = (
             'confirm your age', 'age-restricted', 'inappropriate',  # reason
             'age_verification_required', 'age_check_required',  # status
@@ -3606,7 +3605,7 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
         ])
-        streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
+        streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
 
         for fmt in streaming_formats:
             if fmt.get('targetDurationSec'):
@@ -3872,7 +3871,7 @@ def _list_formats(self, video_id, microformats, video_details, player_responses,
                        else 'was_live' if live_content
                        else 'not_live' if False in (is_live, live_content)
                        else None)
-        streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
+        streaming_data = traverse_obj(player_responses, (..., 'streamingData'))
         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
 
         return live_broadcast_details, live_status, streaming_data, formats, subtitles
@@ -3887,7 +3886,7 @@ def _real_extract(self, url):
         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
 
         playability_statuses = traverse_obj(
-            player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
+            player_responses, (..., 'playabilityStatus'), expected_type=dict)
 
         trailer_video_id = get_first(
             playability_statuses,
@@ -3900,11 +3899,10 @@ def _real_extract(self, url):
         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
                        if webpage else (lambda x: None))
 
-        video_details = traverse_obj(
-            player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
+        video_details = traverse_obj(player_responses, (..., 'videoDetails'), expected_type=dict)
         microformats = traverse_obj(
             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
-            expected_type=dict, default=[])
+            expected_type=dict)
 
         translated_title = self._get_text(microformats, (..., 'title'))
         video_title = (self._preferred_lang and translated_title
@@ -4110,10 +4108,10 @@ def get_lang_code(track):
             # Converted into dicts to remove duplicates
             captions = {
                 get_lang_code(sub): sub
-                for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
+                for sub in traverse_obj(pctr, (..., 'captionTracks', ...))}
             translation_languages = {
                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
-                for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
+                for lang in traverse_obj(pctr, (..., 'translationLanguages', ...))}
 
             def process_language(container, base_url, lang_code, sub_name, query):
                 lang_subs = container.setdefault(lang_code, [])
@@ -4267,9 +4265,8 @@ def process_language(container, base_url, lang_code, sub_name, query):
                     list) or []):
                 tbrs = variadic(
                     traverse_obj(
-                        tlb, 'toggleButtonRenderer',
-                        ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer'),
-                        default=[]))
+                        tlb, ('toggleButtonRenderer', ...),
+                        ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer')))
                 for tbr in tbrs:
                     for getter, regex in [(
                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 878b2b6a8..7cf151e3a 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5420,7 +5420,7 @@ def traverse_obj(
     Each of the provided `paths` is tested and the first producing a valid result will be returned.
     The next path will also be tested if the path branched but no results could be found.
     Supported values for traversal are `Mapping`, `Sequence` and `re.Match`.
-    Unhelpful values (`[]`, `{}`, `None`) are treated as the absence of a value and discarded.
+    Unhelpful values (`{}`, `None`) are treated as the absence of a value and discarded.
 
     The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
 
@@ -5484,7 +5484,7 @@ def apply_key(key, obj, is_last):
         branching = False
         result = None
 
-        if obj is None:
+        if obj is None and traverse_string:
             pass
 
         elif key is None:
@@ -5558,14 +5558,13 @@ def apply_key(key, obj, is_last):
                 result = next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
 
         elif isinstance(key, (int, slice)):
-            if not is_sequence(obj):
-                if traverse_string:
-                    with contextlib.suppress(IndexError):
-                        result = str(obj)[key]
-            else:
+            if is_sequence(obj):
                 branching = isinstance(key, slice)
                 with contextlib.suppress(IndexError):
                     result = obj[key]
+            elif traverse_string:
+                with contextlib.suppress(IndexError):
+                    result = str(obj)[key]
 
         return branching, result if branching else (result,)
 
@@ -5617,7 +5616,7 @@ def apply_path(start_obj, path, test_type):
 
     def _traverse_obj(obj, path, allow_empty, test_type):
         results, has_branched, is_dict = apply_path(obj, path, test_type)
-        results = LazyList(item for item in results if item not in (None, [], {}))
+        results = LazyList(item for item in results if item not in (None, {}))
         if get_all and has_branched:
             if results:
                 return results.exhaust()

From 203a06f8554df6db07d8f20f465ecbfe8a14e591 Mon Sep 17 00:00:00 2001
From: Master <roadmasternavi@gmail.com>
Date: Sat, 11 Feb 2023 19:24:10 +0900
Subject: [PATCH 034/871] [extractor/radiko] Fix format sorting for Time Free
 (#6159)

Authored by: road-master
---
 yt_dlp/extractor/radiko.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/radiko.py b/yt_dlp/extractor/radiko.py
index f10292203..43eecba5f 100644
--- a/yt_dlp/extractor/radiko.py
+++ b/yt_dlp/extractor/radiko.py
@@ -1,5 +1,4 @@
 import base64
-import re
 import urllib.parse
 
 from .common import InfoExtractor
@@ -15,6 +14,23 @@
 
 class RadikoBaseIE(InfoExtractor):
     _FULL_KEY = None
+    _HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED = (
+        'https://c-rpaa.smartstream.ne.jp',
+        'https://si-c-radiko.smartstream.ne.jp',
+        'https://tf-f-rpaa-radiko.smartstream.ne.jp',
+        'https://tf-c-rpaa-radiko.smartstream.ne.jp',
+        'https://si-f-radiko.smartstream.ne.jp',
+        'https://rpaa.smartstream.ne.jp',
+    )
+    _HOSTS_FOR_TIME_FREE_FFMPEG_SUPPORTED = (
+        'https://rd-wowza-radiko.radiko-cf.com',
+        'https://radiko.jp',
+        'https://f-radiko.smartstream.ne.jp',
+    )
+    # Following URL forcibly connects not Time Free but Live
+    _HOSTS_FOR_LIVE = (
+        'https://c-radiko.smartstream.ne.jp',
+    )
 
     def _auth_client(self):
         _, auth1_handle = self._download_webpage_handle(
@@ -92,9 +108,9 @@ def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token,
         formats = []
         found = set()
         for url_tag in m3u8_urls:
-            pcu = url_tag.find('playlist_create_url')
+            pcu = url_tag.find('playlist_create_url').text
             url_attrib = url_tag.attrib
-            playlist_url = update_url_query(pcu.text, {
+            playlist_url = update_url_query(pcu, {
                 'station_id': station,
                 **query,
                 'l': '15',
@@ -117,10 +133,11 @@ def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token,
                     'X-Radiko-AreaId': area_id,
                     'X-Radiko-AuthToken': auth_token,
                 })
+            not_preferred = is_onair and not pcu.startswith(self._HOSTS_FOR_LIVE) or (not is_onair and (pcu.startswith(self._HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED) or pcu.startswith(self._HOSTS_FOR_LIVE)))
             for sf in subformats:
-                if re.fullmatch(r'[cf]-radiko\.smartstream\.ne\.jp', domain):
-                    # Prioritize live radio vs playback based on extractor
-                    sf['preference'] = 100 if is_onair else -100
+                if not_preferred:
+                    sf['preference'] = -100
+                    sf['format_note'] = 'not preferred'
                 if not is_onair and url_attrib['timefree'] == '1' and time_to_skip:
                     sf['downloader_options'] = {'ffmpeg_args': ['-ss', time_to_skip]}
             formats.extend(subformats)

From b85faf6ffb700058e774e99c04304a7a9257cdd0 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Feb 2023 11:56:12 +0530
Subject: [PATCH 035/871] [devscripts/pyinstaller] Analyze sub-modules of
 `Cryptodome`

Ref: https://github.com/yt-dlp/yt-dlp/issues/6185#issuecomment-1423523986
---
 yt_dlp/__pyinstaller/hook-yt_dlp.py | 38 +++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/__pyinstaller/hook-yt_dlp.py b/yt_dlp/__pyinstaller/hook-yt_dlp.py
index 66d1b6369..057cfef2f 100644
--- a/yt_dlp/__pyinstaller/hook-yt_dlp.py
+++ b/yt_dlp/__pyinstaller/hook-yt_dlp.py
@@ -1,9 +1,31 @@
+import ast
+import os
 import sys
+from pathlib import Path
 
 from PyInstaller.utils.hooks import collect_submodules
 
 
-def _pycryptodome_module():
+def find_attribute_accesses(node, name, path=()):
+    if isinstance(node, ast.Attribute):
+        path = [*path, node.attr]
+        if isinstance(node.value, ast.Name) and node.value.id == name:
+            yield path[::-1]
+    for child in ast.iter_child_nodes(node):
+        yield from find_attribute_accesses(child, name, path)
+
+
+def collect_used_submodules(name, level):
+    for dirpath, _, filenames in os.walk(Path(__file__).parent.parent):
+        for filename in filenames:
+            if not filename.endswith('.py'):
+                continue
+            with open(Path(dirpath) / filename, encoding='utf8') as f:
+                for submodule in find_attribute_accesses(ast.parse(f.read()), name):
+                    yield '.'.join(submodule[:level])
+
+
+def pycryptodome_module():
     try:
         import Cryptodome  # noqa: F401
     except ImportError:
@@ -17,13 +39,19 @@ def _pycryptodome_module():
     return 'Cryptodome'
 
 
-def _hidden_imports():
+def get_hidden_imports():
     yield 'yt_dlp.compat._legacy'
-    for m in [_pycryptodome_module(), 'websockets']:
-        yield from collect_submodules(m)
+    yield from collect_submodules('websockets')
+
+    crypto = pycryptodome_module()
+    for sm in set(collect_used_submodules('Cryptodome', 2)):
+        yield f'{crypto}.{sm}'
+
     # These are auto-detected, but explicitly add them just in case
     yield from ('mutagen', 'brotli', 'certifi')
 
 
-hiddenimports = list(_hidden_imports())
+hiddenimports = list(get_hidden_imports())
+print(f'Adding imports: {hiddenimports}')
+
 excludedimports = ['youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts']

From 7708df8da05c94270b43e0630e4e20f6d2d62c55 Mon Sep 17 00:00:00 2001
From: Alex Berg <chexxor@users.noreply.github.com>
Date: Sat, 11 Feb 2023 21:47:52 -0600
Subject: [PATCH 036/871] [extractor/Hidive] Fix subtitles and age-restriction
 (#5828)

Authored by: chexxor
Closes #408
---
 yt_dlp/extractor/hidive.py | 50 +++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/yt_dlp/extractor/hidive.py b/yt_dlp/extractor/hidive.py
index 3a53f2c45..8a8749859 100644
--- a/yt_dlp/extractor/hidive.py
+++ b/yt_dlp/extractor/hidive.py
@@ -1,5 +1,3 @@
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -39,15 +37,27 @@ def _perform_login(self, username, password):
         form = self._search_regex(
             r'(?s)<form[^>]+action="/account/login"[^>]*>(.+?)</form>',
             webpage, 'login form', default=None)
-        if not form:  # logged in
+        if not form:
             return
         data = self._hidden_inputs(form)
         data.update({
             'Email': username,
             'Password': password,
         })
-        self._download_webpage(
+        login_webpage = self._download_webpage(
             self._LOGIN_URL, None, 'Logging in', data=urlencode_postdata(data))
+        # If the user has multiple profiles on their account, select one. For now pick the first profile.
+        profile_id = self._search_regex(r'<button [^>]+?data-profile-id="(\w+)"', login_webpage, 'profile_id')
+        if profile_id is None:
+            return  # If only one profile, Hidive auto-selects it
+        profile_id_hash = self._search_regex(r'\<button [^>]+?data-hash="(\w+)"', login_webpage, 'profile_id_hash')
+        self._request_webpage(
+            'https://www.hidive.com/ajax/chooseprofile', None,
+            data=urlencode_postdata({
+                'profileId': profile_id,
+                'hash': profile_id_hash,
+                'returnUrl': '/dashboard'
+            }))
 
     def _call_api(self, video_id, title, key, data={}, **kwargs):
         data = {
@@ -60,26 +70,6 @@ def _call_api(self, video_id, title, key, data={}, **kwargs):
             'https://www.hidive.com/play/settings', video_id,
             data=urlencode_postdata(data), **kwargs) or {}
 
-    def _extract_subtitles_from_rendition(self, rendition, subtitles, parsed_urls):
-        for cc_file in rendition.get('ccFiles', []):
-            cc_url = url_or_none(try_get(cc_file, lambda x: x[2]))
-            # name is used since we cant distinguish subs with same language code
-            cc_lang = try_get(cc_file, (lambda x: x[1].replace(' ', '-').lower(), lambda x: x[0]), str)
-            if cc_url not in parsed_urls and cc_lang:
-                parsed_urls.add(cc_url)
-                subtitles.setdefault(cc_lang, []).append({'url': cc_url})
-
-    def _get_subtitles(self, url, video_id, title, key, parsed_urls):
-        webpage = self._download_webpage(url, video_id, fatal=False) or ''
-        subtitles = {}
-        for caption in set(re.findall(r'data-captions=\"([^\"]+)\"', webpage)):
-            renditions = self._call_api(
-                video_id, title, key, {'Captions': caption}, fatal=False,
-                note=f'Downloading {caption} subtitle information').get('renditions') or {}
-            for rendition_id, rendition in renditions.items():
-                self._extract_subtitles_from_rendition(rendition, subtitles, parsed_urls)
-        return subtitles
-
     def _real_extract(self, url):
         video_id, title, key = self._match_valid_url(url).group('id', 'title', 'key')
         settings = self._call_api(video_id, title, key)
@@ -104,10 +94,20 @@ def _real_extract(self, url):
                     f['format_note'] = f'{version}, {extra}'
                 formats.extend(frmt)
 
+        subtitles = {}
+        for rendition_id, rendition in settings['renditions'].items():
+            audio, version, extra = rendition_id.split('_')
+            for cc_file in rendition.get('ccFiles') or []:
+                cc_url = url_or_none(try_get(cc_file, lambda x: x[2]))
+                cc_lang = try_get(cc_file, (lambda x: x[1].replace(' ', '-').lower(), lambda x: x[0]), str)
+                if cc_url not in parsed_urls and cc_lang:
+                    parsed_urls.add(cc_url)
+                    subtitles.setdefault(cc_lang, []).append({'url': cc_url})
+
         return {
             'id': video_id,
             'title': video_id,
-            'subtitles': self.extract_subtitles(url, video_id, title, key, parsed_urls),
+            'subtitles': subtitles,
             'formats': formats,
             'series': title,
             'season_number': int_or_none(

From c085cc2def9862ac8a7619ce8ea5dcc177325719 Mon Sep 17 00:00:00 2001
From: pmitchell86 <121815598+pmitchell86@users.noreply.github.com>
Date: Sat, 11 Feb 2023 20:13:31 -0800
Subject: [PATCH 037/871] [extractor/91porn] Fix title and comment extraction
 (#5932)

Authored by: pmitchell86
Fixes #3256
---
 yt_dlp/extractor/porn91.py | 89 ++++++++++++++++++++++++++------------
 1 file changed, 62 insertions(+), 27 deletions(-)

diff --git a/yt_dlp/extractor/porn91.py b/yt_dlp/extractor/porn91.py
index af4a0dc9c..7d16a1631 100644
--- a/yt_dlp/extractor/porn91.py
+++ b/yt_dlp/extractor/porn91.py
@@ -1,26 +1,48 @@
+import urllib.parse
 from .common import InfoExtractor
 from ..utils import (
-    parse_duration,
+    determine_ext,
     int_or_none,
+    parse_duration,
+    remove_end,
+    unified_strdate,
     ExtractorError,
 )
 
 
 class Porn91IE(InfoExtractor):
     IE_NAME = '91porn'
-    _VALID_URL = r'(?:https?://)(?:www\.|)91porn\.com/.+?\?viewkey=(?P<id>[\w\d]+)'
+    _VALID_URL = r'(?:https?://)(?:www\.|)91porn\.com/view_video.php\?([^#]+&)?viewkey=(?P<id>\w+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
-        'md5': '7fcdb5349354f40d41689bd0fa8db05a',
+        'md5': 'd869db281402e0ef4ddef3c38b866f86',
         'info_dict': {
             'id': '7e42283b4f5ab36da134',
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
+            'description': 'md5:1ff241f579b07ae936a54e810ad2e891',
             'ext': 'mp4',
             'duration': 431,
+            'upload_date': '20150520',
+            'comment_count': int,
+            'view_count': int,
             'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'https://91porn.com/view_video.php?viewkey=7ef0cf3d362c699ab91c',
+        'md5': 'f8fd50540468a6d795378cd778b40226',
+        'info_dict': {
+            'id': '7ef0cf3d362c699ab91c',
+            'title': '真实空乘，冲上云霄第二部',
+            'description': 'md5:618bf9652cafcc66cd277bd96789baea',
+            'ext': 'mp4',
+            'duration': 248,
+            'upload_date': '20221119',
+            'comment_count': int,
+            'view_count': int,
+            'age_limit': 18,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -29,32 +51,45 @@ def _real_extract(self, url):
         webpage = self._download_webpage(
             'http://91porn.com/view_video.php?viewkey=%s' % video_id, video_id)
 
-        if '作为游客，你每天只可观看10个视频' in webpage:
-            raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
+        if '视频不存在,可能已经被删除或者被举报为不良内容!' in webpage:
+            raise ExtractorError('91 Porn says: Video does not exist', expected=True)
 
-        title = self._search_regex(
-            r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
-        title = title.replace('\n', '')
+        daily_limit = self._search_regex(
+            r'作为游客，你每天只可观看([\d]+)个视频', webpage, 'exceeded daily limit', default=None, fatal=False)
+        if daily_limit:
+            raise ExtractorError(f'91 Porn says: Daily limit {daily_limit} videos exceeded', expected=True)
 
         video_link_url = self._search_regex(
-            r'<textarea[^>]+id=["\']fm-video_link[^>]+>([^<]+)</textarea>',
-            webpage, 'video link')
-        videopage = self._download_webpage(video_link_url, video_id)
+            r'document\.write\(\s*strencode2\s*\(\s*((?:"[^"]+")|(?:\'[^\']+\'))', webpage, 'video link')
+        video_link_url = self._search_regex(
+            r'src=["\']([^"\']+)["\']', urllib.parse.unquote(video_link_url), 'unquoted video link')
 
-        info_dict = self._parse_html5_media_entries(url, videopage, video_id)[0]
+        formats, subtitles = self._get_formats_and_subtitle(video_link_url, video_id)
 
-        duration = parse_duration(self._search_regex(
-            r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
-
-        comment_count = int_or_none(self._search_regex(
-            r'留言:\s*</span>\s*(\d+)', webpage, 'comment count', fatal=False))
-
-        info_dict.update({
+        return {
             'id': video_id,
-            'title': title,
-            'duration': duration,
-            'comment_count': comment_count,
-            'age_limit': self._rta_search(webpage),
-        })
+            'title': remove_end(self._html_extract_title(webpage).replace('\n', ''), 'Chinese homemade video').strip(),
+            'formats': formats,
+            'subtitles': subtitles,
+            'upload_date': unified_strdate(self._search_regex(
+                r'<span\s+class=["\']title-yakov["\']>(\d{4}-\d{2}-\d{2})</span>', webpage, 'upload_date', fatal=False)),
+            'description': self._html_search_regex(
+                r'<span\s+class=["\']more title["\']>\s*([^<]+)', webpage, 'description', fatal=False),
+            'duration': parse_duration(self._search_regex(
+                r'时长:\s*<span[^>]*>\s*(\d+(?::\d+){1,2})', webpage, 'duration', fatal=False)),
+            'comment_count': int_or_none(self._search_regex(
+                r'留言:\s*<span[^>]*>\s*(\d+)\s*</span>', webpage, 'comment count', fatal=False)),
+            'view_count': int_or_none(self._search_regex(
+                r'热度:\s*<span[^>]*>\s*(\d+)\s*</span>', webpage, 'view count', fatal=False)),
+            'age_limit': 18,
+        }
 
-        return info_dict
+    def _get_formats_and_subtitle(self, video_link_url, video_id):
+        ext = determine_ext(video_link_url)
+        if ext == 'm3u8':
+            formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_link_url, video_id, ext='mp4')
+        else:
+            formats = [{'url': video_link_url, 'ext': ext}]
+            subtitles = {}
+
+        return formats, subtitles

From c62e64cf0122e52fa2175dd1b004ca6b8e1d82af Mon Sep 17 00:00:00 2001
From: panatexxa <91012623+panatexxa@users.noreply.github.com>
Date: Sun, 12 Feb 2023 05:15:16 +0100
Subject: [PATCH 038/871] [extractor/moviepilot] Fix extractor (#5954)

Authored by: panatexxa
---
 yt_dlp/extractor/moviepilot.py | 53 ++++++++++++----------------------
 1 file changed, 19 insertions(+), 34 deletions(-)

diff --git a/yt_dlp/extractor/moviepilot.py b/yt_dlp/extractor/moviepilot.py
index ca541567a..668c0984e 100644
--- a/yt_dlp/extractor/moviepilot.py
+++ b/yt_dlp/extractor/moviepilot.py
@@ -1,11 +1,5 @@
 from .dailymotion import DailymotionIE
 from .common import InfoExtractor
-from ..utils import (
-    parse_iso8601,
-    try_get,
-)
-
-import re
 
 
 class MoviepilotIE(InfoExtractor):
@@ -16,21 +10,21 @@ class MoviepilotIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://www.moviepilot.de/movies/interstellar-2/',
         'info_dict': {
-            'id': 'x7xdut5',
+            'id': 'x7xdpkk',
             'display_id': 'interstellar-2',
             'ext': 'mp4',
             'title': 'Interstellar',
-            'thumbnail': r're:https://\w+\.dmcdn\.net/v/SaXev1VvzitVZMFsR/x720',
-            'timestamp': 1400491705,
-            'description': 'md5:7dfc5c1758e7322a7346934f1f0c489c',
+            'thumbnail': r're:https://\w+\.dmcdn\.net/v/SaV-q1ZganMw4HVXg/x1080',
+            'timestamp': 1605010596,
+            'description': 'md5:0ae9cb452af52610c9ffc60f2fd0474c',
             'uploader': 'Moviepilot',
             'like_count': int,
             'view_count': int,
             'uploader_id': 'x6nd9k',
-            'upload_date': '20140519',
-            'duration': 140,
+            'upload_date': '20201110',
+            'duration': 97,
             'age_limit': 0,
-            'tags': ['Alle Trailer', 'Movie', 'Third Party'],
+            'tags': ['Alle Trailer', 'Movie', 'Verleih'],
         },
     }, {
         'url': 'https://www.moviepilot.de/movies/interstellar-2/trailer',
@@ -45,14 +39,14 @@ class MoviepilotIE(InfoExtractor):
             'display_id': 'queen-slim',
             'title': 'Queen & Slim',
             'ext': 'mp4',
-            'thumbnail': r're:https://\w+\.dmcdn\.net/v/SbUM71WtomSjVmI_q/x720',
-            'timestamp': 1571838685,
-            'description': 'md5:73058bcd030aa12d991e4280d65fbebe',
+            'thumbnail': r're:https://\w+\.dmcdn\.net/v/SbUM71ZeG2N975lf2/x1080',
+            'timestamp': 1605555825,
+            'description': 'md5:83228bb86f5367dd181447fdc4873989',
             'uploader': 'Moviepilot',
             'like_count': int,
             'view_count': int,
             'uploader_id': 'x6nd9k',
-            'upload_date': '20191023',
+            'upload_date': '20201116',
             'duration': 138,
             'age_limit': 0,
             'tags': ['Movie', 'Verleih', 'Neue Trailer'],
@@ -72,12 +66,12 @@ class MoviepilotIE(InfoExtractor):
             'display_id': 'muellers-buero',
             'title': 'Müllers Büro',
             'ext': 'mp4',
-            'description': 'md5:57501251c05cdc61ca314b7633e0312e',
-            'timestamp': 1287584475,
+            'description': 'md5:4d23a8f4ca035196cd4523863c4fe5a4',
+            'timestamp': 1604958457,
             'age_limit': 0,
             'duration': 82,
-            'upload_date': '20101020',
-            'thumbnail': r're:https://\w+\.dmcdn\.net/v/SaMes1WfAm1d6maq_/x720',
+            'upload_date': '20201109',
+            'thumbnail': r're:https://\w+\.dmcdn\.net/v/SaMes1Zg3lxLv9j5u/x1080',
             'uploader': 'Moviepilot',
             'like_count': int,
             'view_count': int,
@@ -91,22 +85,13 @@ def _real_extract(self, url):
 
         webpage = self._download_webpage(f'https://www.moviepilot.de/movies/{video_id}/trailer', video_id)
 
-        duration = try_get(
-            re.match(r'P(?P<hours>\d+)H(?P<mins>\d+)M(?P<secs>\d+)S',
-                     self._html_search_meta('duration', webpage, fatal=False) or ''),
-            lambda mobj: sum(float(x) * y for x, y in zip(mobj.groups(), (3600, 60, 1))))
-        # _html_search_meta is not used since we don't want name=description to match
-        description = self._html_search_regex(
-            '<meta[^>]+itemprop="description"[^>]+content="([^>"]+)"', webpage, 'description', fatal=False)
+        clip = self._search_nextjs_data(webpage, video_id)['props']['initialProps']['pageProps']
 
         return {
             '_type': 'url_transparent',
             'ie_key': DailymotionIE.ie_key(),
             'display_id': video_id,
-            'title': self._og_search_title(webpage),
-            'url': self._html_search_meta('embedURL', webpage),
-            'thumbnail': self._html_search_meta('thumbnailURL', webpage),
-            'description': description,
-            'duration': duration,
-            'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage), delimiter=' ')
+            'title': clip.get('title'),
+            'url': f'https://www.dailymotion.com/video/{clip["videoRemoteId"]}',
+            'description': clip.get('summary'),
         }

From 7d5f919bad07017f4b39b55725491b1e9717d47a Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Sun, 12 Feb 2023 00:17:37 -0400
Subject: [PATCH 039/871] [extractor/Stripchat] Fix extractor (#5985)

Authored by bashonly, JChris246
Closes #5963, closes #5866
---
 yt_dlp/extractor/stripchat.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/stripchat.py b/yt_dlp/extractor/stripchat.py
index 4229a0bf1..4d2fb0608 100644
--- a/yt_dlp/extractor/stripchat.py
+++ b/yt_dlp/extractor/stripchat.py
@@ -1,5 +1,10 @@
 from .common import InfoExtractor
-from ..utils import ExtractorError, lowercase_escape, traverse_obj
+from ..utils import (
+    ExtractorError,
+    UserNotLive,
+    lowercase_escape,
+    traverse_obj
+)
 
 
 class StripchatIE(InfoExtractor):
@@ -35,16 +40,16 @@ def _real_extract(self, url):
         if traverse_obj(data, ('viewCam', 'show'), expected_type=dict):
             raise ExtractorError('Model is in private show', expected=True)
         elif not traverse_obj(data, ('viewCam', 'model', 'isLive'), expected_type=bool):
-            raise ExtractorError('Model is offline', expected=True)
+            raise UserNotLive(video_id=video_id)
 
         server = traverse_obj(data, ('viewCam', 'viewServers', 'flashphoner-hls'), expected_type=str)
         model_id = traverse_obj(data, ('viewCam', 'model', 'id'), expected_type=int)
 
         formats = []
-        for host in traverse_obj(data, (
-                'config', 'data', (('featuresV2', 'hlsFallback', 'fallbackDomains', ...), 'hlsStreamHost'))):
+        for host in traverse_obj(data, ('config', 'data', (
+                (('features', 'featuresV2'), 'hlsFallback', 'fallbackDomains', ...), 'hlsStreamHost'))):
             formats = self._extract_m3u8_formats(
-                f'https://b-{server}.{host}/hls/{model_id}/{model_id}.m3u8',
+                f'https://b-{server}.{host}/hls/{model_id}/master/{model_id}_auto.m3u8',
                 video_id, ext='mp4', m3u8_id='hls', fatal=False, live=True)
             if formats:
                 break

From 989f47b6315541989bb507f26b431d9586430995 Mon Sep 17 00:00:00 2001
From: oxamun <123496932+oxamun@users.noreply.github.com>
Date: Sat, 11 Feb 2023 23:21:29 -0500
Subject: [PATCH 040/871] [extractor/tnaflix] Fix extractor (#6086)

Closes #6085
Authored by: oxamun, bashonly
---
 yt_dlp/extractor/tnaflix.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/yt_dlp/extractor/tnaflix.py b/yt_dlp/extractor/tnaflix.py
index 4482c8474..b2baf2e87 100644
--- a/yt_dlp/extractor/tnaflix.py
+++ b/yt_dlp/extractor/tnaflix.py
@@ -81,26 +81,27 @@ def _real_extract(self, url):
             display_id = video_id
 
         webpage = self._download_webpage(url, display_id)
+        inputs = self._hidden_inputs(webpage)
+        query = {}
 
         # check for MovieFap-style config
         cfg_url = self._proto_relative_url(self._html_search_regex(
             self._CONFIG_REGEX, webpage, 'flashvars.config', default=None,
             group='url'), 'http:')
-        query = {}
+
+        if not cfg_url:
+            cfg_url = inputs.get('config')
 
         # check for TNAFlix-style config
-        if not cfg_url:
-            inputs = self._hidden_inputs(webpage)
-            if inputs.get('vkey') and inputs.get('nkey'):
-                cfg_url = f'https://www.{host}.com/cdn/cdn.php'
-                query.update({
-                    'file': inputs['vkey'],
-                    'key': inputs['nkey'],
-                    'VID': video_id,
-                    'premium': '1',
-                    'vip': '1',
-                    'alpha': '',
-                })
+        if not cfg_url and inputs.get('vkey') and inputs.get('nkey'):
+            cfg_url = f'http://cdn-fck.{host}.com/{host}/{inputs["vkey"]}.fid'
+            query.update({
+                'key': inputs['nkey'],
+                'VID': video_id,
+                'premium': '1',
+                'vip': '1',
+                'alpha': '',
+            })
 
         formats, json_ld = [], {}
 

From cc13293c2819b5461be211a9729fd02bb1e2f476 Mon Sep 17 00:00:00 2001
From: qulaz <32022811+qulaz@users.noreply.github.com>
Date: Sun, 12 Feb 2023 10:28:15 +0600
Subject: [PATCH 041/871] [extractor/clyp] Support `wav` (#6102)

Authored by: qulaz
---
 yt_dlp/extractor/clyp.py | 43 ++++++++++++++++++++++++++++++----------
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/yt_dlp/extractor/clyp.py b/yt_dlp/extractor/clyp.py
index 0aaf73d18..273d0025f 100644
--- a/yt_dlp/extractor/clyp.py
+++ b/yt_dlp/extractor/clyp.py
@@ -9,22 +9,22 @@
 class ClypIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
     _TESTS = [{
-        'url': 'https://clyp.it/ojz2wfah',
-        'md5': '1d4961036c41247ecfdcc439c0cddcbb',
+        'url': 'https://clyp.it/iynkjk4b',
+        'md5': '4bc6371c65210e7b372097fce4d92441',
         'info_dict': {
-            'id': 'ojz2wfah',
-            'ext': 'mp3',
-            'title': 'Krisson80 - bits wip wip',
-            'description': '#Krisson80BitsWipWip #chiptune\n#wip',
-            'duration': 263.21,
-            'timestamp': 1443515251,
-            'upload_date': '20150929',
+            'id': 'iynkjk4b',
+            'ext': 'ogg',
+            'title': 'research',
+            'description': '#Research',
+            'duration': 51.278,
+            'timestamp': 1435524981,
+            'upload_date': '20150628',
         },
     }, {
         'url': 'https://clyp.it/b04p1odi?token=b0078e077e15835845c528a44417719d',
         'info_dict': {
             'id': 'b04p1odi',
-            'ext': 'mp3',
+            'ext': 'ogg',
             'title': 'GJ! (Reward Edit)',
             'description': 'Metal Resistance (THE ONE edition)',
             'duration': 177.789,
@@ -34,6 +34,17 @@ class ClypIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://clyp.it/v42214lc',
+        'md5': '4aca4dfc3236fb6d6ddc4ea08314f33f',
+        'info_dict': {
+            'id': 'v42214lc',
+            'ext': 'wav',
+            'title': 'i dont wanna go (old version)',
+            'duration': 113.528,
+            'timestamp': 1607348505,
+            'upload_date': '20201207',
+        },
     }]
 
     def _real_extract(self, url):
@@ -59,8 +70,20 @@ def _real_extract(self, url):
                         'url': format_url,
                         'format_id': format_id,
                         'vcodec': 'none',
+                        'acodec': ext.lower(),
                     })
 
+        page = self._download_webpage(url, video_id=audio_id)
+        wav_url = self._html_search_regex(
+            r'var\s*wavStreamUrl\s*=\s*["\'](?P<url>https?://[^\'"]+)', page, 'url', default=None)
+        if wav_url:
+            formats.append({
+                'url': wav_url,
+                'format_id': 'wavStreamUrl',
+                'vcodec': 'none',
+                'acodec': 'wav',
+            })
+
         title = metadata['Title']
         description = metadata.get('Description')
         duration = float_or_none(metadata.get('Duration'))

From 20266508dd6247dd3cf0e97b9b9f14c3afc046db Mon Sep 17 00:00:00 2001
From: Chris Caruso <carusochrisr@gmail.com>
Date: Sat, 11 Feb 2023 20:29:41 -0800
Subject: [PATCH 042/871] [extractor/bfmtv] Support `rmc` prefix (#6025)

Authored by: carusocr
Closes #6021
---
 yt_dlp/extractor/bfmtv.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/bfmtv.py b/yt_dlp/extractor/bfmtv.py
index d86d283fa..a7be0e67d 100644
--- a/yt_dlp/extractor/bfmtv.py
+++ b/yt_dlp/extractor/bfmtv.py
@@ -5,7 +5,7 @@
 
 
 class BFMTVBaseIE(InfoExtractor):
-    _VALID_URL_BASE = r'https?://(?:www\.)?bfmtv\.com/'
+    _VALID_URL_BASE = r'https?://(?:www\.|rmc\.)?bfmtv\.com/'
     _VALID_URL_TMPL = _VALID_URL_BASE + r'(?:[^/]+/)*[^/?&#]+_%s[A-Z]-(?P<id>\d{12})\.html'
     _VIDEO_BLOCK_REGEX = r'(<div[^>]+class="video_block"[^>]*>)'
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
@@ -31,6 +31,9 @@ class BFMTVIE(BFMTVBaseIE):
             'uploader_id': '876450610001',
             'upload_date': '20201002',
             'timestamp': 1601629620,
+            'duration': 44.757,
+            'tags': ['bfmactu', 'politique'],
+            'thumbnail': 'https://cf-images.eu-west-1.prod.boltdns.net/v1/static/876450610001/5041f4c1-bc48-4af8-a256-1b8300ad8ef0/cf2f9114-e8e2-4494-82b4-ab794ea4bc7d/1920x1080/match/image.jpg',
         },
     }]
 
@@ -81,6 +84,20 @@ class BFMTVArticleIE(BFMTVBaseIE):
     }, {
         'url': 'https://www.bfmtv.com/sante/covid-19-oui-le-vaccin-de-pfizer-distribue-en-france-a-bien-ete-teste-sur-des-personnes-agees_AN-202101060275.html',
         'only_matching': True,
+    }, {
+        'url': 'https://rmc.bfmtv.com/actualites/societe/transports/ce-n-est-plus-tout-rentable-le-bioethanol-e85-depasse-1eu-le-litre-des-automobilistes-regrettent_AV-202301100268.html',
+        'info_dict': {
+            'id': '6318445464112',
+            'ext': 'mp4',
+            'title': 'Le plein de bioéthanol fait de plus en plus mal à la pompe',
+            'description': None,
+            'uploader_id': '876630703001',
+            'upload_date': '20230110',
+            'timestamp': 1673341692,
+            'duration': 109.269,
+            'tags': ['rmc', 'show', 'apolline de malherbe', 'info', 'talk', 'matinale', 'radio'],
+            'thumbnail': 'https://cf-images.eu-west-1.prod.boltdns.net/v1/static/876630703001/5bef74b8-9d5e-4480-a21f-60c2e2480c46/96c88b74-f9db-45e1-8040-e199c5da216c/1920x1080/match/image.jpg'
+        }
     }]
 
     def _real_extract(self, url):

From cc2389c8ac72a514d4e002a0f6ca5a7d65c7eff0 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sun, 12 Feb 2023 05:35:24 +0100
Subject: [PATCH 043/871] [extractor/npo] Fix extractor and add HD support
 (#6155)

Authored by: seproDev
---
 yt_dlp/extractor/npo.py | 314 +++++++++++-----------------------------
 1 file changed, 83 insertions(+), 231 deletions(-)

diff --git a/yt_dlp/extractor/npo.py b/yt_dlp/extractor/npo.py
index f18cb9e28..40fee24d0 100644
--- a/yt_dlp/extractor/npo.py
+++ b/yt_dlp/extractor/npo.py
@@ -1,36 +1,22 @@
+import random
 import re
+import urllib.parse
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
 from ..utils import (
     determine_ext,
-    ExtractorError,
-    fix_xml_ampersands,
     int_or_none,
     merge_dicts,
     orderedSet,
-    parse_duration,
-    qualities,
     str_or_none,
-    strip_jsonp,
-    unified_strdate,
+    try_call,
     unified_timestamp,
     url_or_none,
     urlencode_postdata,
 )
 
 
-class NPOBaseIE(InfoExtractor):
-    def _get_token(self, video_id):
-        return self._download_json(
-            'http://ida.omroep.nl/app.php/auth', video_id,
-            note='Downloading token')['token']
-
-
-class NPOIE(NPOBaseIE):
+class NPOIE(InfoExtractor):
     IE_NAME = 'npo'
     IE_DESC = 'npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl'
     _VALID_URL = r'''(?x)
@@ -58,6 +44,7 @@ class NPOIE(NPOBaseIE):
             'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.',
             'upload_date': '20140622',
         },
+        'skip': 'Video was removed',
     }, {
         'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800',
         'md5': 'da50a5787dbfc1603c4ad80f31c5120b',
@@ -69,29 +56,41 @@ class NPOIE(NPOBaseIE):
             'upload_date': '20090227',
             'duration': 2400,
         },
+        'skip': 'Video was removed',
     }, {
         'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289',
-        'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
+        'md5': '1b279c0547f6b270e014c576415268c5',
         'info_dict': {
             'id': 'VPWON_1169289',
-            'ext': 'm4v',
-            'title': 'Tegenlicht: Zwart geld. De toekomst komt uit Afrika',
-            'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
+            'ext': 'mp4',
+            'title': 'Zwart geld: de toekomst komt uit Afrika',
+            'description': 'md5:dffaf3d628a9c36f78ca48d834246261',
             'upload_date': '20130225',
             'duration': 3000,
+            'creator': 'NED2',
+            'series': 'Tegenlicht',
+            'timestamp': 1361822340,
+            'thumbnail': 'https://images.npo.nl/tile/1280x720/142854.jpg',
+            'episode': 'Zwart geld: de toekomst komt uit Afrika',
+            'episode_number': 18,
         },
     }, {
         'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706',
         'info_dict': {
             'id': 'WO_VPRO_043706',
-            'ext': 'm4v',
+            'ext': 'mp4',
             'title': 'De nieuwe mens - Deel 1',
             'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b',
             'duration': 4680,
+            'episode': 'De nieuwe mens - Deel 1',
+            'thumbnail': 'https://images.npo.nl/tile/1280x720/6289.jpg',
+            'timestamp': 1279716057,
+            'series': 'De nieuwe mens - Deel 1',
+            'upload_date': '20100721',
         },
         'params': {
             'skip_download': True,
-        }
+        },
     }, {
         # non asf in streams
         'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771',
@@ -102,20 +101,25 @@ class NPOIE(NPOBaseIE):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Video was removed',
     }, {
         'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content',
         'info_dict': {
             'id': 'VPWON_1233944',
-            'ext': 'm4v',
+            'ext': 'mp4',
             'title': 'Aap, poot, pies',
-            'description': 'md5:c9c8005d1869ae65b858e82c01a91fde',
+            'description': 'md5:4b46b1b9553b4c036a04d2a532a137e6',
             'upload_date': '20150508',
             'duration': 599,
+            'episode': 'Aap, poot, pies',
+            'thumbnail': 'https://images.poms.omroep.nl/image/s1280/c1280x720/608118.jpg',
+            'timestamp': 1431064200,
+            'series': 'Aap, poot, pies',
         },
         'params': {
             'skip_download': True,
-        }
+        },
     }, {
         'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698',
         'info_dict': {
@@ -128,7 +132,8 @@ class NPOIE(NPOBaseIE):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Video was removed',
     }, {
         # audio
         'url': 'http://www.npo.nl/jouw-stad-rotterdam/29-01-2017/RBX_FUNX_6683215/RBX_FUNX_7601437',
@@ -140,7 +145,8 @@ class NPOIE(NPOBaseIE):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Video was removed',
     }, {
         'url': 'http://www.zapp.nl/de-bzt-show/gemist/KN_1687547',
         'only_matching': True,
@@ -169,6 +175,25 @@ class NPOIE(NPOBaseIE):
     }, {
         'url': 'https://npo.nl/KN_1698996',
         'only_matching': True,
+    }, {
+        'url': 'https://www.npo3.nl/the-genius/21-11-2022/VPWON_1341105',
+        'info_dict': {
+            'id': 'VPWON_1341105',
+            'ext': 'mp4',
+            'duration': 2658,
+            'series': 'The Genius',
+            'description': 'md5:db02f1456939ca63f7c408f858044e94',
+            'title': 'The Genius',
+            'timestamp': 1669062000,
+            'creator': 'NED3',
+            'episode': 'The Genius',
+            'thumbnail': 'https://images.npo.nl/tile/1280x720/1827650.jpg',
+            'episode_number': 8,
+            'upload_date': '20221121',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     @classmethod
@@ -179,25 +204,32 @@ def suitable(cls, url):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        return self._get_info(url, video_id) or self._get_old_info(video_id)
-
-    def _get_info(self, url, video_id):
-        token = self._download_json(
-            'https://www.npostart.nl/api/token', video_id,
-            'Downloading token', headers={
-                'Referer': url,
-                'X-Requested-With': 'XMLHttpRequest',
-            })['token']
-
-        player = self._download_json(
-            'https://www.npostart.nl/player/%s' % video_id, video_id,
-            'Downloading player JSON', data=urlencode_postdata({
-                'autoplay': 0,
-                'share': 1,
-                'pageUrl': url,
-                'hasAdConsent': 0,
-                '_token': token,
-            }))
+        if urllib.parse.urlparse(url).netloc in ['www.ntr.nl', 'ntr.nl']:
+            player = self._download_json(
+                f'https://www.ntr.nl/ajax/player/embed/{video_id}', video_id,
+                'Downloading player JSON', query={
+                    'parameters[elementId]': f'npo{random.randint(0, 999)}',
+                    'parameters[sterReferralUrl]': url,
+                    'parameters[autoplay]': 0,
+                })
+        else:
+            self._request_webpage(
+                'https://www.npostart.nl/api/token', video_id,
+                'Downloading token', headers={
+                    'Referer': url,
+                    'X-Requested-With': 'XMLHttpRequest',
+                })
+            player = self._download_json(
+                f'https://www.npostart.nl/player/{video_id}', video_id,
+                'Downloading player JSON', data=urlencode_postdata({
+                    'autoplay': 0,
+                    'share': 1,
+                    'pageUrl': url,
+                    'hasAdConsent': 0,
+                }), headers={
+                    'x-xsrf-token': try_call(lambda: urllib.parse.unquote(
+                        self._get_cookies('https://www.npostart.nl')['XSRF-TOKEN'].value))
+                })
 
         player_token = player['token']
 
@@ -210,7 +242,7 @@ def _get_info(self, url, video_id):
                 video_id, 'Downloading %s profile JSON' % profile, fatal=False,
                 query={
                     'profile': profile,
-                    'quality': 'npo',
+                    'quality': 'npoplus',
                     'tokenId': player_token,
                     'streamType': 'broadcast',
                 })
@@ -291,188 +323,8 @@ def _get_info(self, url, video_id):
 
         return info
 
-    def _get_old_info(self, video_id):
-        metadata = self._download_json(
-            'http://e.omroep.nl/metadata/%s' % video_id,
-            video_id,
-            # We have to remove the javascript callback
-            transform_source=strip_jsonp,
-        )
 
-        error = metadata.get('error')
-        if error:
-            raise ExtractorError(error, expected=True)
-
-        # For some videos actual video id (prid) is different (e.g. for
-        # http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698
-        # video id is POMS_WNL_853698 but prid is POW_00996502)
-        video_id = metadata.get('prid') or video_id
-
-        # titel is too generic in some cases so utilize aflevering_titel as well
-        # when available (e.g. http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html)
-        title = metadata['titel']
-        sub_title = metadata.get('aflevering_titel')
-        if sub_title and sub_title != title:
-            title += ': %s' % sub_title
-
-        token = self._get_token(video_id)
-
-        formats = []
-        urls = set()
-
-        def is_legal_url(format_url):
-            return format_url and format_url not in urls and re.match(
-                r'^(?:https?:)?//', format_url)
-
-        QUALITY_LABELS = ('Laag', 'Normaal', 'Hoog')
-        QUALITY_FORMATS = ('adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std')
-
-        quality_from_label = qualities(QUALITY_LABELS)
-        quality_from_format_id = qualities(QUALITY_FORMATS)
-        items = self._download_json(
-            'http://ida.omroep.nl/app.php/%s' % video_id, video_id,
-            'Downloading formats JSON', query={
-                'adaptive': 'yes',
-                'token': token,
-            })['items'][0]
-        for num, item in enumerate(items):
-            item_url = item.get('url')
-            if not is_legal_url(item_url):
-                continue
-            urls.add(item_url)
-            format_id = self._search_regex(
-                r'video/ida/([^/]+)', item_url, 'format id',
-                default=None)
-
-            item_label = item.get('label')
-
-            def add_format_url(format_url):
-                width = int_or_none(self._search_regex(
-                    r'(\d+)[xX]\d+', format_url, 'width', default=None))
-                height = int_or_none(self._search_regex(
-                    r'\d+[xX](\d+)', format_url, 'height', default=None))
-                if item_label in QUALITY_LABELS:
-                    quality = quality_from_label(item_label)
-                    f_id = item_label
-                elif item_label in QUALITY_FORMATS:
-                    quality = quality_from_format_id(format_id)
-                    f_id = format_id
-                else:
-                    quality, f_id = [None] * 2
-                formats.append({
-                    'url': format_url,
-                    'format_id': f_id,
-                    'width': width,
-                    'height': height,
-                    'quality': quality,
-                })
-
-            # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706
-            if item.get('contentType') in ('url', 'audio'):
-                add_format_url(item_url)
-                continue
-
-            try:
-                stream_info = self._download_json(
-                    item_url + '&type=json', video_id,
-                    'Downloading %s stream JSON'
-                    % item_label or item.get('format') or format_id or num)
-            except ExtractorError as ee:
-                if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
-                    error = (self._parse_json(
-                        ee.cause.read().decode(), video_id,
-                        fatal=False) or {}).get('errorstring')
-                    if error:
-                        raise ExtractorError(error, expected=True)
-                raise
-            # Stream URL instead of JSON, example: npo:LI_NL1_4188102
-            if isinstance(stream_info, compat_str):
-                if not stream_info.startswith('http'):
-                    continue
-                video_url = stream_info
-            # JSON
-            else:
-                video_url = stream_info.get('url')
-            if not video_url or 'vodnotavailable.' in video_url or video_url in urls:
-                continue
-            urls.add(video_url)
-            if determine_ext(video_url) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    video_url, video_id, ext='mp4',
-                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
-            else:
-                add_format_url(video_url)
-
-        is_live = metadata.get('medium') == 'live'
-
-        if not is_live:
-            for num, stream in enumerate(metadata.get('streams', [])):
-                stream_url = stream.get('url')
-                if not is_legal_url(stream_url):
-                    continue
-                urls.add(stream_url)
-                # smooth streaming is not supported
-                stream_type = stream.get('type', '').lower()
-                if stream_type in ['ss', 'ms']:
-                    continue
-                if stream_type == 'hds':
-                    f4m_formats = self._extract_f4m_formats(
-                        stream_url, video_id, fatal=False)
-                    # f4m downloader downloads only piece of live stream
-                    for f4m_format in f4m_formats:
-                        f4m_format['preference'] = -5
-                    formats.extend(f4m_formats)
-                elif stream_type == 'hls':
-                    formats.extend(self._extract_m3u8_formats(
-                        stream_url, video_id, ext='mp4', fatal=False))
-                # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706
-                elif '.asf' in stream_url:
-                    asx = self._download_xml(
-                        stream_url, video_id,
-                        'Downloading stream %d ASX playlist' % num,
-                        transform_source=fix_xml_ampersands, fatal=False)
-                    if not asx:
-                        continue
-                    ref = asx.find('./ENTRY/Ref')
-                    if ref is None:
-                        continue
-                    video_url = ref.get('href')
-                    if not video_url or video_url in urls:
-                        continue
-                    urls.add(video_url)
-                    formats.append({
-                        'url': video_url,
-                        'ext': stream.get('formaat', 'asf'),
-                        'quality': stream.get('kwaliteit'),
-                        'preference': -10,
-                    })
-                else:
-                    formats.append({
-                        'url': stream_url,
-                        'quality': stream.get('kwaliteit'),
-                    })
-
-        subtitles = {}
-        if metadata.get('tt888') == 'ja':
-            subtitles['nl'] = [{
-                'ext': 'vtt',
-                'url': 'http://tt888.omroep.nl/tt888/%s' % video_id,
-            }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': metadata.get('info'),
-            'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
-            'upload_date': unified_strdate(metadata.get('gidsdatum')),
-            'duration': parse_duration(metadata.get('tijdsduur')),
-            'formats': formats,
-            'subtitles': subtitles,
-            'is_live': is_live,
-        }
-
-
-class NPOLiveIE(NPOBaseIE):
+class NPOLiveIE(InfoExtractor):
     IE_NAME = 'npo.nl:live'
     _VALID_URL = r'https?://(?:www\.)?npo(?:start)?\.nl/live(?:/(?P<id>[^/?#&]+))?'
 

From a31d0fa6c315b1145d682361149003d98f1e3782 Mon Sep 17 00:00:00 2001
From: "lauren n. liberda" <lauren@selfisekai.rocks>
Date: Sun, 12 Feb 2023 05:43:10 +0100
Subject: [PATCH 044/871] [extractor/tvp] Support `stream.tvp.pl` (#6139)

Authored by: selfisekai
---
 yt_dlp/extractor/tvp.py | 34 +++++++++++++++-------------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/yt_dlp/extractor/tvp.py b/yt_dlp/extractor/tvp.py
index 8483564f7..f8ded2646 100644
--- a/yt_dlp/extractor/tvp.py
+++ b/yt_dlp/extractor/tvp.py
@@ -268,8 +268,11 @@ def _real_extract(self, url):
 
 class TVPStreamIE(InfoExtractor):
     IE_NAME = 'tvp:stream'
-    _VALID_URL = r'(?:tvpstream:|https?://tvpstream\.vod\.tvp\.pl/(?:\?(?:[^&]+[&;])*channel_id=)?)(?P<id>\d*)'
+    _VALID_URL = r'(?:tvpstream:|https?://(?:tvpstream\.vod|stream)\.tvp\.pl/(?:\?(?:[^&]+[&;])*channel_id=)?)(?P<id>\d*)'
     _TESTS = [{
+        'url': 'https://stream.tvp.pl/?channel_id=56969941',
+        'only_matching': True,
+    }, {
         # untestable as "video" id changes many times across a day
         'url': 'https://tvpstream.vod.tvp.pl/?channel_id=1455',
         'only_matching': True,
@@ -285,28 +288,21 @@ class TVPStreamIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    _PLAYER_BOX_RE = r'<div\s[^>]*id\s*=\s*["\']?tvp_player_box["\']?[^>]+data-%s-id\s*=\s*["\']?(\d+)'
-    _BUTTON_RE = r'<div\s[^>]*data-channel-id=["\']?%s["\']?[^>]*\sdata-title=(?:"([^"]*)"|\'([^\']*)\')[^>]*\sdata-stationname=(?:"([^"]*)"|\'([^\']*)\')'
-
     def _real_extract(self, url):
         channel_id = self._match_id(url)
-        channel_url = self._proto_relative_url('//tvpstream.vod.tvp.pl/?channel_id=%s' % channel_id or 'default')
-        webpage = self._download_webpage(channel_url, channel_id, 'Downloading channel webpage')
-        if not channel_id:
-            channel_id = self._search_regex(self._PLAYER_BOX_RE % 'channel',
-                                            webpage, 'default channel id')
-        video_id = self._search_regex(self._PLAYER_BOX_RE % 'video',
-                                      webpage, 'video id')
-        audition_title, station_name = self._search_regex(
-            self._BUTTON_RE % (re.escape(channel_id)), webpage,
-            'audition title and station name',
-            group=(1, 2))
+        channel_url = self._proto_relative_url('//stream.tvp.pl/?channel_id=%s' % channel_id or 'default')
+        webpage = self._download_webpage(channel_url, channel_id or 'default', 'Downloading channel webpage')
+        channels = self._search_json(
+            r'window\.__channels\s*=', webpage, 'channel list', channel_id,
+            contains_pattern=r'\[\s*{(?s:.+)}\s*]')
+        channel = traverse_obj(channels, (lambda _, v: channel_id == str(v['id'])), get_all=False) if channel_id else channels[0]
+        audition = traverse_obj(channel, ('items', lambda _, v: v['is_live'] is True), get_all=False)
         return {
             '_type': 'url_transparent',
-            'id': channel_id,
-            'url': 'tvp:%s' % video_id,
-            'title': audition_title,
-            'alt_title': station_name,
+            'id': channel_id or channel['id'],
+            'url': 'tvp:%s' % audition['video_id'],
+            'title': audition.get('title'),
+            'alt_title': channel.get('title'),
             'is_live': True,
             'ie_key': 'TVPEmbed',
         }

From b3eaab7ca2e118d4db73dcb44afd9c8717db8b67 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sun, 12 Feb 2023 05:47:03 +0100
Subject: [PATCH 045/871] [extractor/vlive] Replace with `VLiveWebArchiveIE`
 (#6196)

vlive has shut down: https://web.archive.org/web/20221031171019/https://www.vlive.tv/notice/4749

Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py |   6 +-
 yt_dlp/extractor/archiveorg.py  | 236 ++++++++++++++++++++
 yt_dlp/extractor/naver.py       |  35 +--
 yt_dlp/extractor/vlive.py       | 372 --------------------------------
 4 files changed, 255 insertions(+), 394 deletions(-)
 delete mode 100644 yt_dlp/extractor/vlive.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 50dfe2eb1..12ef50cc6 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -122,6 +122,7 @@
 from .archiveorg import (
     ArchiveOrgIE,
     YoutubeWebArchiveIE,
+    VLiveWebArchiveIE,
 )
 from .arcpublishing import ArcPublishingIE
 from .arkena import ArkenaIE
@@ -2183,11 +2184,6 @@
     VKUserVideosIE,
     VKWallPostIE,
 )
-from .vlive import (
-    VLiveIE,
-    VLivePostIE,
-    VLiveChannelIE,
-)
 from .vodlocker import VodlockerIE
 from .vodpl import VODPlIE
 from .vodplatform import VODPlatformIE
diff --git a/yt_dlp/extractor/archiveorg.py b/yt_dlp/extractor/archiveorg.py
index 90dda9f53..4ccd39825 100644
--- a/yt_dlp/extractor/archiveorg.py
+++ b/yt_dlp/extractor/archiveorg.py
@@ -1,8 +1,10 @@
 import json
 import re
+import urllib.error
 import urllib.parse
 
 from .common import InfoExtractor
+from .naver import NaverBaseIE
 from .youtube import YoutubeBaseInfoExtractor, YoutubeIE
 from ..compat import compat_HTTPError, compat_urllib_parse_unquote
 from ..utils import (
@@ -945,3 +947,237 @@ def _real_extract(self, url):
         if not info.get('title'):
             info['title'] = video_id
         return info
+
+
+class VLiveWebArchiveIE(InfoExtractor):
+    IE_NAME = 'web.archive:vlive'
+    IE_DESC = 'web.archive.org saved vlive videos'
+    _VALID_URL = r'''(?x)
+            (?:https?://)?web\.archive\.org/
+            (?:web/)?(?:(?P<date>[0-9]{14})?[0-9A-Za-z_*]*/)?  # /web and the version index is optional
+            (?:https?(?::|%3[Aa])//)?(?:
+                (?:(?:www|m)\.)?vlive\.tv(?::(?:80|443))?/(?:video|embed)/(?P<id>[0-9]+)  # VLive URL
+            )
+        '''
+    _TESTS = [{
+        'url': 'https://web.archive.org/web/20221221144331/http://www.vlive.tv/video/1326',
+        'md5': 'cc7314812855ce56de70a06a27314983',
+        'info_dict': {
+            'id': '1326',
+            'ext': 'mp4',
+            'title': "Girl's Day's Broadcast",
+            'creator': "Girl's Day",
+            'view_count': int,
+            'uploader_id': 'muploader_a',
+            'uploader_url': None,
+            'uploader': None,
+            'upload_date': '20150817',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
+            'timestamp': 1439816449,
+            'like_count': int,
+            'channel': 'Girl\'s Day',
+            'channel_id': 'FDF27',
+            'comment_count': int,
+            'release_timestamp': 1439818140,
+            'release_date': '20150817',
+            'duration': 1014,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://web.archive.org/web/20221221182103/http://www.vlive.tv/video/16937',
+        'info_dict': {
+            'id': '16937',
+            'ext': 'mp4',
+            'title': '첸백시 걍방',
+            'creator': 'EXO',
+            'view_count': int,
+            'subtitles': 'mincount:12',
+            'uploader_id': 'muploader_j',
+            'uploader_url': 'http://vlive.tv',
+            'uploader': None,
+            'upload_date': '20161112',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
+            'timestamp': 1478923074,
+            'like_count': int,
+            'channel': 'EXO',
+            'channel_id': 'F94BD',
+            'comment_count': int,
+            'release_timestamp': 1478924280,
+            'release_date': '20161112',
+            'duration': 906,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://web.archive.org/web/20221127190050/http://www.vlive.tv/video/101870',
+        'info_dict': {
+            'id': '101870',
+            'ext': 'mp4',
+            'title': '[ⓓ xV] “레벨이들 매력에 반해? 안 반해?” 움직이는 HD 포토 (레드벨벳:Red Velvet)',
+            'creator': 'Dispatch',
+            'view_count': int,
+            'subtitles': 'mincount:6',
+            'uploader_id': 'V__FRA08071',
+            'uploader_url': 'http://vlive.tv',
+            'uploader': None,
+            'upload_date': '20181130',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
+            'timestamp': 1543601327,
+            'like_count': int,
+            'channel': 'Dispatch',
+            'channel_id': 'C796F3',
+            'comment_count': int,
+            'release_timestamp': 1543601040,
+            'release_date': '20181130',
+            'duration': 279,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    # The wayback machine has special timestamp and "mode" values:
+    # timestamp:
+    #   1 = the first capture
+    #   2 = the last capture
+    # mode:
+    #   id_ = Identity - perform no alterations of the original resource, return it as it was archived.
+    _WAYBACK_BASE_URL = 'https://web.archive.org/web/2id_/'
+
+    def _download_archived_page(self, url, video_id, *, timestamp='2', **kwargs):
+        for retry in self.RetryManager():
+            try:
+                return self._download_webpage(f'https://web.archive.org/web/{timestamp}id_/{url}', video_id, **kwargs)
+            except ExtractorError as e:
+                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+                    raise ExtractorError('Page was not archived', expected=True)
+                retry.error = e
+                continue
+
+    def _download_archived_json(self, url, video_id, **kwargs):
+        page = self._download_archived_page(url, video_id, **kwargs)
+        if not page:
+            raise ExtractorError('Page was not archived', expected=True)
+        else:
+            return self._parse_json(page, video_id)
+
+    def _extract_formats_from_m3u8(self, m3u8_url, params, video_id):
+        m3u8_doc = self._download_archived_page(m3u8_url, video_id, note='Downloading m3u8', query=params, fatal=False)
+        if not m3u8_doc:
+            return
+
+        # M3U8 document should be changed to archive domain
+        m3u8_doc = m3u8_doc.splitlines()
+        url_base = m3u8_url.rsplit('/', 1)[0]
+        first_segment = None
+        for i, line in enumerate(m3u8_doc):
+            if not line.startswith('#'):
+                m3u8_doc[i] = f'{self._WAYBACK_BASE_URL}{url_base}/{line}?{urllib.parse.urlencode(params)}'
+                first_segment = first_segment or m3u8_doc[i]
+
+        # Segments may not have been archived. See https://web.archive.org/web/20221127190050/http://www.vlive.tv/video/101870
+        urlh = self._request_webpage(HEADRequest(first_segment), video_id, errnote=False,
+                                     fatal=False, note='Check first segment availablity')
+        if urlh:
+            formats, subtitles = self._parse_m3u8_formats_and_subtitles('\n'.join(m3u8_doc), ext='mp4', video_id=video_id)
+            if subtitles:
+                self._report_ignoring_subs('m3u8')
+            return formats
+
+    # Closely follows the logic of the ArchiveTeam grab script
+    # See: https://github.com/ArchiveTeam/vlive-grab/blob/master/vlive.lua
+    def _real_extract(self, url):
+        video_id, url_date = self._match_valid_url(url).group('id', 'date')
+
+        webpage = self._download_archived_page(f'https://www.vlive.tv/video/{video_id}', video_id, timestamp=url_date)
+
+        player_info = self._search_json(r'__PRELOADED_STATE__\s*=', webpage, 'player info', video_id)
+        user_country = traverse_obj(player_info, ('common', 'userCountry'))
+
+        main_script_url = self._search_regex(r'<script\s+src="([^"]+/js/main\.[^"]+\.js)"', webpage, 'main script url')
+        main_script = self._download_archived_page(main_script_url, video_id, note='Downloading main script')
+        app_id = self._search_regex(r'appId\s*=\s*"([^"]+)"', main_script, 'app id')
+
+        inkey = self._download_archived_json(
+            f'https://www.vlive.tv/globalv-web/vam-web/video/v1.0/vod/{video_id}/inkey', video_id, note='Fetching inkey', query={
+                'appId': app_id,
+                'platformType': 'PC',
+                'gcc': user_country,
+                'locale': 'en_US',
+            }, fatal=False)
+
+        vod_id = traverse_obj(player_info, ('postDetail', 'post', 'officialVideo', 'vodId'))
+
+        vod_data = self._download_archived_json(
+            f'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{vod_id}', video_id, note='Fetching vod data', query={
+                'key': inkey.get('inkey'),
+                'pid': 'rmcPlayer_16692457559726800',  # partially unix time and partially random. Fixed value used by archiveteam project
+                'sid': '2024',
+                'ver': '2.0',
+                'devt': 'html5_pc',
+                'doct': 'json',
+                'ptc': 'https',
+                'sptc': 'https',
+                'cpt': 'vtt',
+                'ctls': '%7B%22visible%22%3A%7B%22fullscreen%22%3Atrue%2C%22logo%22%3Afalse%2C%22playbackRate%22%3Afalse%2C%22scrap%22%3Afalse%2C%22playCount%22%3Atrue%2C%22commentCount%22%3Atrue%2C%22title%22%3Atrue%2C%22writer%22%3Atrue%2C%22expand%22%3Afalse%2C%22subtitles%22%3Atrue%2C%22thumbnails%22%3Atrue%2C%22quality%22%3Atrue%2C%22setting%22%3Atrue%2C%22script%22%3Afalse%2C%22logoDimmed%22%3Atrue%2C%22badge%22%3Atrue%2C%22seekingTime%22%3Atrue%2C%22muted%22%3Atrue%2C%22muteButton%22%3Afalse%2C%22viewerNotice%22%3Afalse%2C%22linkCount%22%3Afalse%2C%22createTime%22%3Afalse%2C%22thumbnail%22%3Atrue%7D%2C%22clicked%22%3A%7B%22expand%22%3Afalse%2C%22subtitles%22%3Afalse%7D%7D',
+                'pv': '4.26.9',
+                'dr': '1920x1080',
+                'cpl': 'en_US',
+                'lc': 'en_US',
+                'adi': '%5B%7B%22type%22%3A%22pre%22%2C%22exposure%22%3Afalse%2C%22replayExposure%22%3Afalse%7D%5D',
+                'adu': '%2F',
+                'videoId': vod_id,
+                'cc': user_country,
+            })
+
+        formats = []
+
+        streams = traverse_obj(vod_data, ('streams', ...))
+        if len(streams) > 1:
+            self.report_warning('Multiple streams found. Only the first stream will be downloaded.')
+        stream = streams[0]
+
+        max_stream = max(
+            stream.get('videos') or [],
+            key=lambda v: traverse_obj(v, ('bitrate', 'video'), default=0), default=None)
+        if max_stream is not None:
+            params = {arg.get('name'): arg.get('value') for arg in stream.get('keys', []) if arg.get('type') == 'param'}
+            formats = self._extract_formats_from_m3u8(max_stream.get('source'), params, video_id) or []
+
+        # For parts of the project MP4 files were archived
+        max_video = max(
+            traverse_obj(vod_data, ('videos', 'list', ...)),
+            key=lambda v: traverse_obj(v, ('bitrate', 'video'), default=0), default=None)
+        if max_video is not None:
+            video_url = self._WAYBACK_BASE_URL + max_video.get('source')
+            urlh = self._request_webpage(HEADRequest(video_url), video_id, errnote=False,
+                                         fatal=False, note='Check video availablity')
+            if urlh:
+                formats.append({'url': video_url})
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            **traverse_obj(player_info, ('postDetail', 'post', {
+                'title': ('officialVideo', 'title', {str}),
+                'creator': ('author', 'nickname', {str}),
+                'channel': ('channel', 'channelName', {str}),
+                'channel_id': ('channel', 'channelCode', {str}),
+                'duration': ('officialVideo', 'playTime', {int_or_none}),
+                'view_count': ('officialVideo', 'playCount', {int_or_none}),
+                'like_count': ('officialVideo', 'likeCount', {int_or_none}),
+                'comment_count': ('officialVideo', 'commentCount', {int_or_none}),
+                'timestamp': ('officialVideo', 'createdAt', {lambda x: int_or_none(x, scale=1000)}),
+                'release_timestamp': ('officialVideo', 'willStartAt', {lambda x: int_or_none(x, scale=1000)}),
+            })),
+            **traverse_obj(vod_data, ('meta', {
+                'uploader_id': ('user', 'id', {str}),
+                'uploader': ('user', 'name', {str}),
+                'uploader_url': ('user', 'url', {url_or_none}),
+                'thumbnail': ('cover', 'source', {url_or_none}),
+            }), expected_type=lambda x: x or None),
+            **NaverBaseIE.process_subtitles(vod_data, lambda x: [self._WAYBACK_BASE_URL + x]),
+        }
diff --git a/yt_dlp/extractor/naver.py b/yt_dlp/extractor/naver.py
index e2e6e9728..7a1890a61 100644
--- a/yt_dlp/extractor/naver.py
+++ b/yt_dlp/extractor/naver.py
@@ -21,6 +21,23 @@
 class NaverBaseIE(InfoExtractor):
     _CAPTION_EXT_RE = r'\.(?:ttml|vtt)'
 
+    @staticmethod  # NB: Used in VLiveWebArchiveIE
+    def process_subtitles(vod_data, process_url):
+        ret = {'subtitles': {}, 'automatic_captions': {}}
+        for caption in traverse_obj(vod_data, ('captions', 'list', ...)):
+            caption_url = caption.get('source')
+            if not caption_url:
+                continue
+            type_ = 'automatic_captions' if caption.get('type') == 'auto' else 'subtitles'
+            lang = caption.get('locale') or join_nonempty('language', 'country', from_dict=caption) or 'und'
+            if caption.get('type') == 'fan':
+                lang += '_fan%d' % next(i for i in itertools.count(1) if f'{lang}_fan{i}' not in ret[type_])
+            ret[type_].setdefault(lang, []).extend({
+                'url': sub_url,
+                'name': join_nonempty('label', 'fanName', from_dict=caption, delim=' - '),
+            } for sub_url in process_url(caption_url))
+        return ret
+
     def _extract_video_info(self, video_id, vid, key):
         video_data = self._download_json(
             'http://play.rmcnmv.naver.com/vod/play/v2.0/' + vid,
@@ -79,34 +96,18 @@ def get_subs(caption_url):
                 ]
             return [caption_url]
 
-        automatic_captions = {}
-        subtitles = {}
-        for caption in get_list('caption'):
-            caption_url = caption.get('source')
-            if not caption_url:
-                continue
-            sub_dict = automatic_captions if caption.get('type') == 'auto' else subtitles
-            lang = caption.get('locale') or join_nonempty('language', 'country', from_dict=caption) or 'und'
-            if caption.get('type') == 'fan':
-                lang += '_fan%d' % next(i for i in itertools.count(1) if f'{lang}_fan{i}' not in sub_dict)
-            sub_dict.setdefault(lang, []).extend({
-                'url': sub_url,
-                'name': join_nonempty('label', 'fanName', from_dict=caption, delim=' - '),
-            } for sub_url in get_subs(caption_url))
-
         user = meta.get('user', {})
 
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'subtitles': subtitles,
-            'automatic_captions': automatic_captions,
             'thumbnail': try_get(meta, lambda x: x['cover']['source']),
             'view_count': int_or_none(meta.get('count')),
             'uploader_id': user.get('id'),
             'uploader': user.get('name'),
             'uploader_url': user.get('url'),
+            **self.process_subtitles(video_data, get_subs),
         }
 
 
diff --git a/yt_dlp/extractor/vlive.py b/yt_dlp/extractor/vlive.py
deleted file mode 100644
index e2fd39315..000000000
--- a/yt_dlp/extractor/vlive.py
+++ /dev/null
@@ -1,372 +0,0 @@
-import itertools
-import json
-
-from .naver import NaverBaseIE
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    LazyList,
-    merge_dicts,
-    str_or_none,
-    strip_or_none,
-    traverse_obj,
-    try_get,
-    urlencode_postdata,
-    url_or_none,
-)
-
-
-class VLiveBaseIE(NaverBaseIE):
-    _NETRC_MACHINE = 'vlive'
-    _logged_in = False
-
-    def _perform_login(self, username, password):
-        if self._logged_in:
-            return
-        LOGIN_URL = 'https://www.vlive.tv/auth/email/login'
-        self._request_webpage(
-            LOGIN_URL, None, note='Downloading login cookies')
-
-        self._download_webpage(
-            LOGIN_URL, None, note='Logging in',
-            data=urlencode_postdata({'email': username, 'pwd': password}),
-            headers={
-                'Referer': LOGIN_URL,
-                'Content-Type': 'application/x-www-form-urlencoded'
-            })
-
-        login_info = self._download_json(
-            'https://www.vlive.tv/auth/loginInfo', None,
-            note='Checking login status',
-            headers={'Referer': 'https://www.vlive.tv/home'})
-
-        if not try_get(login_info, lambda x: x['message']['login'], bool):
-            raise ExtractorError('Unable to log in', expected=True)
-        VLiveBaseIE._logged_in = True
-
-    def _call_api(self, path_template, video_id, fields=None, query_add={}, note=None):
-        if note is None:
-            note = 'Downloading %s JSON metadata' % path_template.split('/')[-1].split('-')[0]
-        query = {'appId': '8c6cc7b45d2568fb668be6e05b6e5a3b', 'gcc': 'KR', 'platformType': 'PC'}
-        if fields:
-            query['fields'] = fields
-        if query_add:
-            query.update(query_add)
-        try:
-            return self._download_json(
-                'https://www.vlive.tv/globalv-web/vam-web/' + path_template % video_id, video_id,
-                note, headers={'Referer': 'https://www.vlive.tv/'}, query=query)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                self.raise_login_required(json.loads(e.cause.read().decode('utf-8'))['message'])
-            raise
-
-
-class VLiveIE(VLiveBaseIE):
-    IE_NAME = 'vlive'
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/(?:video|embed)/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.vlive.tv/video/1326',
-        'md5': 'cc7314812855ce56de70a06a27314983',
-        'info_dict': {
-            'id': '1326',
-            'ext': 'mp4',
-            'title': "Girl's Day's Broadcast",
-            'creator': "Girl's Day",
-            'view_count': int,
-            'uploader_id': 'muploader_a',
-            'upload_date': '20150817',
-            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
-            'timestamp': 1439816449,
-            'like_count': int,
-            'channel': 'Girl\'s Day',
-            'channel_id': 'FDF27',
-            'comment_count': int,
-            'release_timestamp': 1439818140,
-            'release_date': '20150817',
-            'duration': 1014,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://www.vlive.tv/video/16937',
-        'info_dict': {
-            'id': '16937',
-            'ext': 'mp4',
-            'title': '첸백시 걍방',
-            'creator': 'EXO',
-            'view_count': int,
-            'subtitles': 'mincount:12',
-            'uploader_id': 'muploader_j',
-            'upload_date': '20161112',
-            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
-            'timestamp': 1478923074,
-            'like_count': int,
-            'channel': 'EXO',
-            'channel_id': 'F94BD',
-            'comment_count': int,
-            'release_timestamp': 1478924280,
-            'release_date': '20161112',
-            'duration': 906,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://www.vlive.tv/video/129100',
-        'md5': 'ca2569453b79d66e5b919e5d308bff6b',
-        'info_dict': {
-            'id': '129100',
-            'ext': 'mp4',
-            'title': '[V LIVE] [BTS+] Run BTS! 2019 - EP.71 :: Behind the scene',
-            'creator': 'BTS+',
-            'view_count': int,
-            'subtitles': 'mincount:10',
-        },
-        'skip': 'This video is only available for CH+ subscribers',
-    }, {
-        'url': 'https://www.vlive.tv/embed/1326',
-        'only_matching': True,
-    }, {
-        # works only with gcc=KR
-        'url': 'https://www.vlive.tv/video/225019',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.vlive.tv/video/223906',
-        'info_dict': {
-            'id': '58',
-            'title': 'RUN BTS!'
-        },
-        'playlist_mincount': 120
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        post = self._call_api(
-            'post/v1.0/officialVideoPost-%s', video_id,
-            'author{nickname},channel{channelCode,channelName},officialVideo{commentCount,exposeStatus,likeCount,playCount,playTime,status,title,type,vodId},playlist{playlistSeq,totalCount,name}')
-
-        playlist_id = str_or_none(try_get(post, lambda x: x['playlist']['playlistSeq']))
-        if not self._yes_playlist(playlist_id, video_id):
-            video = post['officialVideo']
-            return self._get_vlive_info(post, video, video_id)
-
-        playlist_name = str_or_none(try_get(post, lambda x: x['playlist']['name']))
-        playlist_count = str_or_none(try_get(post, lambda x: x['playlist']['totalCount']))
-
-        playlist = self._call_api(
-            'playlist/v1.0/playlist-%s/posts', playlist_id, 'data', {'limit': playlist_count})
-
-        entries = []
-        for video_data in playlist['data']:
-            video = video_data.get('officialVideo')
-            video_id = str_or_none(video.get('videoSeq'))
-            entries.append(self._get_vlive_info(video_data, video, video_id))
-
-        return self.playlist_result(entries, playlist_id, playlist_name)
-
-    def _get_vlive_info(self, post, video, video_id):
-        def get_common_fields():
-            channel = post.get('channel') or {}
-            return {
-                'title': video.get('title'),
-                'creator': post.get('author', {}).get('nickname'),
-                'channel': channel.get('channelName'),
-                'channel_id': channel.get('channelCode'),
-                'duration': int_or_none(video.get('playTime')),
-                'view_count': int_or_none(video.get('playCount')),
-                'like_count': int_or_none(video.get('likeCount')),
-                'comment_count': int_or_none(video.get('commentCount')),
-                'timestamp': int_or_none(video.get('createdAt'), scale=1000),
-                'release_timestamp': int_or_none(traverse_obj(video, 'onAirStartAt', 'willStartAt'), scale=1000),
-                'thumbnail': video.get('thumb'),
-            }
-
-        video_type = video.get('type')
-        if video_type == 'VOD':
-            inkey = self._call_api('video/v1.0/vod/%s/inkey', video_id)['inkey']
-            vod_id = video['vodId']
-            info_dict = merge_dicts(
-                get_common_fields(),
-                self._extract_video_info(video_id, vod_id, inkey))
-            thumbnail = video.get('thumb')
-            if thumbnail:
-                if not info_dict.get('thumbnails') and info_dict.get('thumbnail'):
-                    info_dict['thumbnails'] = [{'url': info_dict.pop('thumbnail')}]
-                info_dict.setdefault('thumbnails', []).append({'url': thumbnail, 'preference': 1})
-            return info_dict
-        elif video_type == 'LIVE':
-            status = video.get('status')
-            if status == 'ON_AIR':
-                stream_url = self._call_api(
-                    'old/v3/live/%s/playInfo',
-                    video_id)['result']['adaptiveStreamUrl']
-                formats = self._extract_m3u8_formats(stream_url, video_id, 'mp4')
-                info = get_common_fields()
-                info.update({
-                    'title': video['title'],
-                    'id': video_id,
-                    'formats': formats,
-                    'is_live': True,
-                })
-                return info
-            elif status == 'ENDED':
-                raise ExtractorError(
-                    'Uploading for replay. Please wait...', expected=True)
-            elif status == 'RESERVED':
-                raise ExtractorError('Coming soon!', expected=True)
-            elif video.get('exposeStatus') == 'CANCEL':
-                raise ExtractorError(
-                    'We are sorry, but the live broadcast has been canceled.',
-                    expected=True)
-            else:
-                raise ExtractorError('Unknown status ' + status)
-
-
-class VLivePostIE(VLiveBaseIE):
-    IE_NAME = 'vlive:post'
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/post/(?P<id>\d-\d+)'
-    _TESTS = [{
-        # uploadType = SOS
-        'url': 'https://www.vlive.tv/post/1-20088044',
-        'info_dict': {
-            'id': '1-20088044',
-            'title': 'Hola estrellitas la tierra les dice hola (si era así no?) Ha...',
-            'description': 'md5:fab8a1e50e6e51608907f46c7fa4b407',
-        },
-        'playlist_count': 3,
-    }, {
-        # uploadType = V
-        'url': 'https://www.vlive.tv/post/1-20087926',
-        'info_dict': {
-            'id': '1-20087926',
-            'title': 'James Corden: And so, the baby becamos the Papa💜😭💪😭',
-        },
-        'playlist_count': 1,
-    }]
-    _FVIDEO_TMPL = 'fvideo/v1.0/fvideo-%%s/%s'
-
-    def _real_extract(self, url):
-        post_id = self._match_id(url)
-
-        post = self._call_api(
-            'post/v1.0/post-%s', post_id,
-            'attachments{video},officialVideo{videoSeq},plainBody,title')
-
-        video_seq = str_or_none(try_get(
-            post, lambda x: x['officialVideo']['videoSeq']))
-        if video_seq:
-            return self.url_result(
-                'http://www.vlive.tv/video/' + video_seq,
-                VLiveIE.ie_key(), video_seq)
-
-        title = post['title']
-        entries = []
-        for idx, video in enumerate(post['attachments']['video'].values()):
-            video_id = video.get('videoId')
-            if not video_id:
-                continue
-            upload_type = video.get('uploadType')
-            upload_info = video.get('uploadInfo') or {}
-            entry = None
-            if upload_type == 'SOS':
-                download = self._call_api(
-                    self._FVIDEO_TMPL % 'sosPlayInfo', video_id)['videoUrl']['download']
-                formats = []
-                for f_id, f_url in download.items():
-                    formats.append({
-                        'format_id': f_id,
-                        'url': f_url,
-                        'height': int_or_none(f_id[:-1]),
-                    })
-                entry = {
-                    'formats': formats,
-                    'id': video_id,
-                    'thumbnail': upload_info.get('imageUrl'),
-                }
-            elif upload_type == 'V':
-                vod_id = upload_info.get('videoId')
-                if not vod_id:
-                    continue
-                inkey = self._call_api(self._FVIDEO_TMPL % 'inKey', video_id)['inKey']
-                entry = self._extract_video_info(video_id, vod_id, inkey)
-            if entry:
-                entry['title'] = '%s_part%s' % (title, idx)
-                entries.append(entry)
-        return self.playlist_result(
-            entries, post_id, title, strip_or_none(post.get('plainBody')))
-
-
-class VLiveChannelIE(VLiveBaseIE):
-    IE_NAME = 'vlive:channel'
-    _VALID_URL = r'https?://(?:channels\.vlive\.tv|(?:(?:www|m)\.)?vlive\.tv/channel)/(?P<channel_id>[0-9A-Z]+)(?:/board/(?P<posts_id>\d+))?'
-    _TESTS = [{
-        'url': 'http://channels.vlive.tv/FCD4B',
-        'info_dict': {
-            'id': 'FCD4B',
-            'title': 'MAMAMOO',
-        },
-        'playlist_mincount': 110
-    }, {
-        'url': 'https://www.vlive.tv/channel/FCD4B',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.vlive.tv/channel/FCD4B/board/3546',
-        'info_dict': {
-            'id': 'FCD4B-3546',
-            'title': 'MAMAMOO - Star Board',
-        },
-        'playlist_mincount': 880
-    }]
-
-    def _entries(self, posts_id, board_name):
-        if board_name:
-            posts_path = 'post/v1.0/board-%s/posts'
-            query_add = {'limit': 100, 'sortType': 'LATEST'}
-        else:
-            posts_path = 'post/v1.0/channel-%s/starPosts'
-            query_add = {'limit': 100}
-
-        for page_num in itertools.count(1):
-            video_list = self._call_api(
-                posts_path, posts_id, 'channel{channelName},contentType,postId,title,url', query_add,
-                note=f'Downloading playlist page {page_num}')
-
-            for video in try_get(video_list, lambda x: x['data'], list) or []:
-                video_id = str(video.get('postId'))
-                video_title = str_or_none(video.get('title'))
-                video_url = url_or_none(video.get('url'))
-                if not all((video_id, video_title, video_url)) or video.get('contentType') != 'VIDEO':
-                    continue
-                channel_name = try_get(video, lambda x: x['channel']['channelName'], compat_str)
-                yield self.url_result(video_url, VLivePostIE.ie_key(), video_id, video_title, channel=channel_name)
-
-            after = try_get(video_list, lambda x: x['paging']['nextParams']['after'], compat_str)
-            if not after:
-                break
-            query_add['after'] = after
-
-    def _real_extract(self, url):
-        channel_id, posts_id = self._match_valid_url(url).groups()
-
-        board_name = None
-        if posts_id:
-            board = self._call_api(
-                'board/v1.0/board-%s', posts_id, 'title,boardType')
-            board_name = board.get('title') or 'Unknown'
-            if board.get('boardType') not in ('STAR', 'VLIVE_PLUS'):
-                raise ExtractorError(f'Board {board_name!r} is not supported', expected=True)
-
-        entries = LazyList(self._entries(posts_id or channel_id, board_name))
-        channel_name = entries[0]['channel']
-
-        return self.playlist_result(
-            entries,
-            f'{channel_id}-{posts_id}' if posts_id else channel_id,
-            f'{channel_name} - {board_name}' if channel_name and board_name else channel_name)

From 417cdaae08fc447c9d15c53a88e2e9a027cdbf0a Mon Sep 17 00:00:00 2001
From: Chris Caruso <carusochrisr@gmail.com>
Date: Sat, 11 Feb 2023 20:53:24 -0800
Subject: [PATCH 046/871] [extractor/ximalaya] Update album `_VALID_URL`
 (#6110)

Authored by: carusocr
Closes #6059
---
 yt_dlp/extractor/ximalaya.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/ximalaya.py b/yt_dlp/extractor/ximalaya.py
index b25be772e..ff18ba697 100644
--- a/yt_dlp/extractor/ximalaya.py
+++ b/yt_dlp/extractor/ximalaya.py
@@ -36,7 +36,7 @@ class XimalayaIE(XimalayaBaseIE):
                         'height': 180
                     }
                 ],
-                'categories': ['人文'],
+                'categories': ['其他'],
                 'duration': 93,
                 'view_count': int,
                 'like_count': int,
@@ -123,7 +123,7 @@ def _real_extract(self, url):
 class XimalayaAlbumIE(XimalayaBaseIE):
     IE_NAME = 'ximalaya:album'
     IE_DESC = '喜马拉雅FM 专辑'
-    _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/\d+/album/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(?:\d+/)?album/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.ximalaya.com/61425525/album/5534601/',
         'info_dict': {
@@ -131,6 +131,13 @@ class XimalayaAlbumIE(XimalayaBaseIE):
             'id': '5534601',
         },
         'playlist_mincount': 323,
+    }, {
+        'url': 'https://www.ximalaya.com/album/6912905',
+        'info_dict': {
+            'title': '埃克哈特《修炼当下的力量》',
+            'id': '6912905',
+        },
+        'playlist_mincount': 41,
     }]
 
     def _real_extract(self, url):

From b23167e7542c177f32b22b29857b637dc4aede69 Mon Sep 17 00:00:00 2001
From: LowSuggestion912 <13346421+LowSuggestion912@users.noreply.github.com>
Date: Sun, 12 Feb 2023 18:25:24 +1100
Subject: [PATCH 047/871] [extractor/common] Fix `_search_nuxt_data` (#6062)

Authored by: LowSuggestion912
---
 yt_dlp/extractor/common.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 09b03e69a..b7c687bc3 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -1670,11 +1670,8 @@ def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', *, fatal
         if js is None:
             return {}
 
-        args = dict(zip(arg_keys.split(','), arg_vals.split(',')))
-
-        for key, val in args.items():
-            if val in ('undefined', 'void 0'):
-                args[key] = 'null'
+        args = dict(zip(arg_keys.split(','), map(json.dumps, self._parse_json(
+            f'[{arg_vals}]', video_id, transform_source=js_to_json, fatal=fatal) or ())))
 
         ret = self._parse_json(js, video_id, transform_source=functools.partial(js_to_json, vars=args), fatal=fatal)
         return traverse_obj(ret, traverse) or {}

From 93abb7406b95793f6872d12979b91d5f336b4f43 Mon Sep 17 00:00:00 2001
From: ByteDream <63594396+ByteDream@users.noreply.github.com>
Date: Sun, 12 Feb 2023 08:47:12 +0100
Subject: [PATCH 048/871] [extractor/crunchyroll] Add intro chapter (#6023)

Authored by: ByteDream
---
 yt_dlp/extractor/crunchyroll.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py
index 836bcb622..3c9f8bbf0 100644
--- a/yt_dlp/extractor/crunchyroll.py
+++ b/yt_dlp/extractor/crunchyroll.py
@@ -116,6 +116,7 @@ class CrunchyrollBetaIE(CrunchyrollBaseIE):
             'episode': 'To the Future',
             'episode_number': 73,
             'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$',
+            'chapters': 'count:2',
         },
         'params': {'skip_download': 'm3u8', 'format': 'all[format_id~=hardsub]'},
     }, {
@@ -136,6 +137,7 @@ class CrunchyrollBetaIE(CrunchyrollBaseIE):
             'episode': 'Porter Robinson presents Shelter the Animation',
             'episode_number': 0,
             'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$',
+            'chapters': 'count:0',
         },
         'params': {'skip_download': True},
         'skip': 'Video is Premium only',
@@ -209,6 +211,17 @@ def _real_extract(self, url):
                 f['quality'] = hardsub_preference(hardsub_lang.lower())
             formats.extend(adaptive_formats)
 
+        chapters = None
+        # if no intro chapter is available, a 403 without usable data is returned
+        intro_chapter = self._download_json(f'https://static.crunchyroll.com/datalab-intro-v2/{internal_id}.json',
+                                            display_id, fatal=False, errnote=False)
+        if isinstance(intro_chapter, dict):
+            chapters = [{
+                'title': 'Intro',
+                'start_time': float_or_none(intro_chapter.get('startTime')),
+                'end_time': float_or_none(intro_chapter.get('endTime'))
+            }]
+
         return {
             'id': internal_id,
             'title': '%s Episode %s – %s' % (
@@ -235,6 +248,7 @@ def _real_extract(self, url):
                     'ext': subtitle_data.get('format')
                 }] for lang, subtitle_data in get_streams('subtitles')
             },
+            'chapters': chapters
         }
 
 

From 05799a48c7dec12b34c8bf951c8d2eceedda59f8 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 12 Feb 2023 13:21:15 +0530
Subject: [PATCH 049/871] [extractor/youtube] Update invidious and piped
 instances (#6030)

Authored by: rohieb
---
 yt_dlp/extractor/youtube.py | 40 +++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index aff89f8ac..01e2e3793 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -312,6 +312,40 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         r'(?:www\.)?invidious\.pussthecat\.org',
         r'(?:www\.)?invidious\.zee\.li',
         r'(?:www\.)?invidious\.ethibox\.fr',
+        r'(?:www\.)?iv\.ggtyler\.dev',
+        r'(?:www\.)?inv\.vern\.i2p',
+        r'(?:www\.)?am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd\.onion',
+        r'(?:www\.)?inv\.riverside\.rocks',
+        r'(?:www\.)?invidious\.silur\.me',
+        r'(?:www\.)?inv\.bp\.projectsegfau\.lt',
+        r'(?:www\.)?invidious\.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid\.onion',
+        r'(?:www\.)?invidious\.slipfox\.xyz',
+        r'(?:www\.)?invidious\.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd\.onion',
+        r'(?:www\.)?inv\.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad\.onion',
+        r'(?:www\.)?invidious\.tiekoetter\.com',
+        r'(?:www\.)?iv\.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd\.onion',
+        r'(?:www\.)?invidious\.nerdvpn\.de',
+        r'(?:www\.)?invidious\.weblibre\.org',
+        r'(?:www\.)?inv\.odyssey346\.dev',
+        r'(?:www\.)?invidious\.dhusch\.de',
+        r'(?:www\.)?iv\.melmac\.space',
+        r'(?:www\.)?watch\.thekitty\.zone',
+        r'(?:www\.)?invidious\.privacydev\.net',
+        r'(?:www\.)?ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid\.onion',
+        r'(?:www\.)?invidious\.drivet\.xyz',
+        r'(?:www\.)?vid\.priv\.au',
+        r'(?:www\.)?euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd\.onion',
+        r'(?:www\.)?inv\.vern\.cc',
+        r'(?:www\.)?invidious\.esmailelbob\.xyz',
+        r'(?:www\.)?invidious\.sethforprivacy\.com',
+        r'(?:www\.)?yt\.oelrichsgarcia\.de',
+        r'(?:www\.)?yt\.artemislena\.eu',
+        r'(?:www\.)?invidious\.flokinet\.to',
+        r'(?:www\.)?invidious\.baczek\.me',
+        r'(?:www\.)?y\.com\.sb',
+        r'(?:www\.)?invidious\.epicsite\.xyz',
+        r'(?:www\.)?invidious\.lidarshield\.cloud',
+        r'(?:www\.)?yt\.funami\.tech',
         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
@@ -390,6 +424,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         r'(?:www\.)?piped\.qdi\.fi',
         r'(?:www\.)?piped\.video',
         r'(?:www\.)?piped\.aeong\.one',
+        r'(?:www\.)?piped\.moomoo\.me',
+        r'(?:www\.)?piped\.chauvet\.pro',
+        r'(?:www\.)?watch\.leptons\.xyz',
+        r'(?:www\.)?pd\.vern\.cc',
+        r'(?:www\.)?piped\.hostux\.net',
+        r'(?:www\.)?piped\.lunar\.icu',
     )
 
     # extracted from account/account_menu ep

From 0ba87dd279d3565ed93c559cf7880ad61eb83af8 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Sun, 12 Feb 2023 16:54:36 +0900
Subject: [PATCH 050/871] [extractor/biliintl] Add intro and ending chapters
 (#6018)

Authored by: HobbyistDev
---
 yt_dlp/extractor/bilibili.py | 59 ++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 266d57871..f4180633a 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -933,6 +933,19 @@ class BiliIntlIE(BiliIntlBaseIE):
             'episode': 'Episode 2',
             'timestamp': 1602259500,
             'description': 'md5:297b5a17155eb645e14a14b385ab547e',
+            'chapters': [{
+                'start_time': 0,
+                'end_time': 76.242,
+                'title': '<Untitled Chapter 1>'
+            }, {
+                'start_time': 76.242,
+                'end_time': 161.161,
+                'title': 'Intro'
+            }, {
+                'start_time': 1325.742,
+                'end_time': 1403.903,
+                'title': 'Outro'
+            }],
         }
     }, {
         # Non-Bstation page
@@ -947,6 +960,19 @@ class BiliIntlIE(BiliIntlBaseIE):
             'episode': 'Episode 3',
             'upload_date': '20211219',
             'timestamp': 1639928700,
+            'chapters': [{
+                'start_time': 0,
+                'end_time': 88.0,
+                'title': '<Untitled Chapter 1>'
+            }, {
+                'start_time': 88.0,
+                'end_time': 156.0,
+                'title': 'Intro'
+            }, {
+                'start_time': 1173.0,
+                'end_time': 1259.535,
+                'title': 'Outro'
+            }],
         }
     }, {
         # Subtitle with empty content
@@ -970,6 +996,20 @@ class BiliIntlIE(BiliIntlBaseIE):
             'upload_date': '20221212',
             'title': 'Kimetsu no Yaiba Season 3 Official Trailer - Bstation',
         }
+    }, {
+        # episode id without intro and outro
+        'url': 'https://www.bilibili.tv/en/play/1048837/11246489',
+        'info_dict': {
+            'id': '11246489',
+            'ext': 'mp4',
+            'title': 'E1 - Operation \'Strix\' <Owl>',
+            'description': 'md5:b4434eb1a9a97ad2bccb779514b89f17',
+            'timestamp': 1649516400,
+            'thumbnail': 'https://pic.bstarstatic.com/ogv/62cb1de23ada17fb70fbe7bdd6ff29c29da02a64.png',
+            'episode': 'Episode 1',
+            'episode_number': 1,
+            'upload_date': '20220409',
+        },
     }, {
         'url': 'https://www.biliintl.com/en/play/34613/341736',
         'only_matching': True,
@@ -1022,12 +1062,31 @@ def _extract_video_metadata(self, url, video_id, season_id):
     def _real_extract(self, url):
         season_id, ep_id, aid = self._match_valid_url(url).group('season_id', 'ep_id', 'aid')
         video_id = ep_id or aid
+        chapters = None
+
+        if ep_id:
+            intro_ending_json = self._call_api(
+                f'/web/v2/ogv/play/episode?episode_id={ep_id}&platform=web',
+                video_id, fatal=False) or {}
+            if intro_ending_json.get('skip'):
+                # FIXME: start time and end time seems a bit off a few second even it corrext based on ogv.*.js
+                # ref: https://p.bstarstatic.com/fe-static/bstar-web-new/assets/ogv.2b147442.js
+                chapters = [{
+                    'start_time': float_or_none(traverse_obj(intro_ending_json, ('skip', 'opening_start_time')), 1000),
+                    'end_time': float_or_none(traverse_obj(intro_ending_json, ('skip', 'opening_end_time')), 1000),
+                    'title': 'Intro'
+                }, {
+                    'start_time': float_or_none(traverse_obj(intro_ending_json, ('skip', 'ending_start_time')), 1000),
+                    'end_time': float_or_none(traverse_obj(intro_ending_json, ('skip', 'ending_end_time')), 1000),
+                    'title': 'Outro'
+                }]
 
         return {
             'id': video_id,
             **self._extract_video_metadata(url, video_id, season_id),
             'formats': self._get_formats(ep_id=ep_id, aid=aid),
             'subtitles': self.extract_subtitles(ep_id=ep_id, aid=aid),
+            'chapters': chapters
         }
 
 

From 78a78fa74dbc888d20f1b65e1382bf99131597d5 Mon Sep 17 00:00:00 2001
From: Bruno Guerreiro <Generator@users.noreply.github.com>
Date: Sun, 12 Feb 2023 08:33:45 +0000
Subject: [PATCH 051/871] [extractor/youtube] Add hyperpipe instances (#6020)

Authored by: Generator
---
 yt_dlp/extractor/youtube.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 01e2e3793..95ca52b3a 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -430,6 +430,11 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         r'(?:www\.)?pd\.vern\.cc',
         r'(?:www\.)?piped\.hostux\.net',
         r'(?:www\.)?piped\.lunar\.icu',
+        # Hyperpipe instances from https://hyperpipe.codeberg.page/
+        r'(?:www\.)?hyperpipe\.surge\.sh',
+        r'(?:www\.)?hyperpipe\.esmailelbob\.xyz',
+        r'(?:www\.)?listen\.whatever\.social',
+        r'(?:www\.)?music\.adminforge\.de',
     )
 
     # extracted from account/account_menu ep

From 2e269bd998c61efaf7500907d114a56e5e83e65e Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 12 Feb 2023 18:43:07 +0530
Subject: [PATCH 052/871] [pyinst] Fix for pyinstaller 5.8

Fixes comment https://github.com/yt-dlp/yt-dlp/issues/1839#issuecomment-1427002271
---
 pyinst.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pyinst.py b/pyinst.py
index 22df672c9..c36f6acd4 100644
--- a/pyinst.py
+++ b/pyinst.py
@@ -85,7 +85,6 @@ def set_version_info(exe, version):
 def windows_set_version(exe, version):
     from PyInstaller.utils.win32.versioninfo import (
         FixedFileInfo,
-        SetVersion,
         StringFileInfo,
         StringStruct,
         StringTable,
@@ -94,6 +93,11 @@ def windows_set_version(exe, version):
         VSVersionInfo,
     )
 
+    try:
+        from PyInstaller.utils.win32.versioninfo import SetVersion
+    except ImportError:  # Pyinstaller >= 5.8
+        from PyInstaller.utils.win32.versioninfo import write_version_info_to_executable as SetVersion
+
     version_list = version_to_list(version)
     suffix = MACHINE and f'_{MACHINE}'
     SetVersion(exe, VSVersionInfo(

From b6795fd310f1dd61dddc9fd08e52fe485bdc8a3e Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 12 Feb 2023 14:43:26 +0100
Subject: [PATCH 053/871] [extractor/twitter] Fix `--no-playlist` and add media
 `view_count` when using GraphQL (#6211)

Authored by: Grub4K
---
 yt_dlp/extractor/twitter.py | 97 ++++++++++++++++++++++++++++---------
 1 file changed, 75 insertions(+), 22 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index d3e52f392..d9d446832 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -293,7 +293,7 @@ def _real_extract(self, url):
 
 class TwitterIE(TwitterBaseIE):
     IE_NAME = 'twitter'
-    _VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)(?:/video/(?P<index>\d+))?'
+    _VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)(?:/(?:video|photo)/(?P<index>\d+))?'
 
     _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
@@ -336,7 +336,7 @@ class TwitterIE(TwitterBaseIE):
             'id': '665052190608723968',
             'display_id': '665052190608723968',
             'ext': 'mp4',
-            'title': 'md5:e99588f17b3dd0503814ffb560e64731',
+            'title': r're:Star Wars.*A new beginning is coming December 18.*',
             'description': 'A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens. https://t.co/OkSqT2fjWJ',
             'uploader_id': 'starwars',
             'uploader': r're:Star Wars.*',
@@ -752,7 +752,7 @@ class TwitterIE(TwitterBaseIE):
         'info_dict': {
             'id': '1600649511827013632',
             'ext': 'mp4',
-            'title': 'md5:be05989b0722e114103ed3851a0ffae2',
+            'title': 'md5:dac4f4d4c591fcc4e88a253eba472dc3',
             'thumbnail': r're:^https?://.+\.jpg',
             'timestamp': 1670459604.0,
             'uploader_id': 'CTVJLaidlaw',
@@ -792,6 +792,52 @@ class TwitterIE(TwitterBaseIE):
             'repost_count': int,
             'comment_count': int,
         },
+    }, {
+        'url': 'https://twitter.com/hlo_again/status/1599108751385972737/video/2',
+        'info_dict': {
+            'id': '1599108643743473680',
+            'display_id': '1599108751385972737',
+            'ext': 'mp4',
+            'title': '\u06ea - \U0001F48B',
+            'uploader_url': 'https://twitter.com/hlo_again',
+            'like_count': int,
+            'uploader_id': 'hlo_again',
+            'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1599108643743473680/pu/img/UG3xjov4rgg5sbYM.jpg?name=orig',
+            'repost_count': int,
+            'duration': 9.531,
+            'comment_count': int,
+            'upload_date': '20221203',
+            'age_limit': 0,
+            'timestamp': 1670092210.0,
+            'tags': [],
+            'uploader': '\u06ea',
+            'description': '\U0001F48B https://t.co/bTj9Qz7vQP',
+        },
+        'params': {'noplaylist': True},
+    }, {
+        # Media view count is GraphQL only, force in test
+        'url': 'https://twitter.com/MunTheShinobi/status/1600009574919962625',
+        'info_dict': {
+            'id': '1600009362759733248',
+            'display_id': '1600009574919962625',
+            'ext': 'mp4',
+            'uploader_url': 'https://twitter.com/MunTheShinobi',
+            'description': 'This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525 https://t.co/cNsA0MoOml',
+            'view_count': int,
+            'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1600009362759733248/pu/img/XVhFQivj75H_YxxV.jpg?name=orig',
+            'age_limit': 0,
+            'uploader': 'Mün The Shinobi | BlaqBoi\'s Therapist',
+            'repost_count': int,
+            'upload_date': '20221206',
+            'title': 'Mün The Shinobi | BlaqBoi\'s Therapist - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
+            'comment_count': int,
+            'like_count': int,
+            'tags': [],
+            'uploader_id': 'MunTheShinobi',
+            'duration': 139.987,
+            'timestamp': 1670306984.0,
+        },
+        'params': {'extractor_args': {'twitter': {'force_graphql': ['']}}},
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -920,13 +966,6 @@ def _real_extract(self, url):
             title = f'{uploader} - {title}'
         uploader_id = user.get('screen_name')
 
-        tags = []
-        for hashtag in (try_get(status, lambda x: x['entities']['hashtags'], list) or []):
-            hashtag_text = hashtag.get('text')
-            if not hashtag_text:
-                continue
-            tags.append(hashtag_text)
-
         info = {
             'id': twid,
             'title': title,
@@ -939,7 +978,7 @@ def _real_extract(self, url):
             'repost_count': int_or_none(status.get('retweet_count')),
             'comment_count': int_or_none(status.get('reply_count')),
             'age_limit': 18 if status.get('possibly_sensitive') else 0,
-            'tags': tags,
+            'tags': traverse_obj(status, ('entities', 'hashtags', ..., 'text')),
         }
 
         def extract_from_video_info(media):
@@ -973,6 +1012,7 @@ def add_thumbnail(name, size):
                 'formats': formats,
                 'subtitles': subtitles,
                 'thumbnails': thumbnails,
+                'view_count': traverse_obj(media, ('mediaStats', 'viewCount', {int_or_none})),
                 'duration': float_or_none(video_info.get('duration_millis'), 1000),
                 # The codec of http formats are unknown
                 '_format_sort_fields': ('res', 'br', 'size', 'proto'),
@@ -1052,11 +1092,31 @@ def get_binding_value(k):
                         'content_duration_seconds')),
                 }
 
-        media_path = ((None, 'quoted_status'), 'extended_entities', 'media', lambda _, m: m['type'] != 'photo')
-        videos = map(extract_from_video_info, traverse_obj(status, media_path, expected_type=dict))
-        cards = extract_from_card_info(status.get('card'))
-        entries = [{**info, **data, 'display_id': twid} for data in (*videos, *cards)]
+        videos = traverse_obj(status, (
+            (None, 'quoted_status'), 'extended_entities', 'media', lambda _, m: m['type'] != 'photo', {dict}))
 
+        if self._yes_playlist(twid, selected_index, video_label='URL-specified video number'):
+            selected_entries = (*map(extract_from_video_info, videos), *extract_from_card_info(status.get('card')))
+        else:
+            desired_obj = traverse_obj(status, ('extended_entities', 'media', int(selected_index) - 1, {dict}))
+            if not desired_obj:
+                raise ExtractorError(f'Video #{selected_index} is unavailable', expected=True)
+            elif desired_obj.get('type') != 'video':
+                raise ExtractorError(f'Media #{selected_index} is not a video', expected=True)
+
+            # Restore original archive id and video index in title
+            for index, entry in enumerate(videos, 1):
+                if entry.get('id') != desired_obj.get('id'):
+                    continue
+                if index == 1:
+                    info['_old_archive_ids'] = [make_archive_id(self, twid)]
+                if len(videos) != 1:
+                    info['title'] += f' #{index}'
+                break
+
+            return {**info, **extract_from_video_info(desired_obj), 'display_id': twid}
+
+        entries = [{**info, **data, 'display_id': twid} for data in selected_entries]
         if not entries:
             expanded_url = traverse_obj(status, ('entities', 'urls', 0, 'expanded_url'), expected_type=url_or_none)
             if not expanded_url or expanded_url == url:
@@ -1066,13 +1126,6 @@ def get_binding_value(k):
 
         entries[0]['_old_archive_ids'] = [make_archive_id(self, twid)]
 
-        if not self._yes_playlist(twid, selected_index, video_label='URL-specified video number'):
-            index = int(selected_index) - 1
-            if index >= len(entries):
-                raise ExtractorError(f'Video #{selected_index} is unavailable', expected=True)
-
-            return entries[index]
-
         if len(entries) == 1:
             return entries[0]
 

From a4f16832213d9e29beecf685d6cd09a2f0b48c87 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Sun, 12 Feb 2023 23:02:09 +0900
Subject: [PATCH 054/871] [extractor/AbemaTV] Cache user token whenever
 appropriate (#6216)

Authored by: Lesmiscore
---
 yt_dlp/extractor/abematv.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index 7552e3e57..f611c1f2c 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -156,7 +156,7 @@ class AbemaTVBaseIE(InfoExtractor):
     def _generate_aks(cls, deviceid):
         deviceid = deviceid.encode('utf-8')
         # add 1 hour and then drop minute and secs
-        ts_1hour = int((time_seconds(hours=9) // 3600 + 1) * 3600)
+        ts_1hour = int((time_seconds() // 3600 + 1) * 3600)
         time_struct = time.gmtime(ts_1hour)
         ts_1hour_str = str(ts_1hour).encode('utf-8')
 
@@ -190,6 +190,16 @@ def _get_device_token(self):
         if self._USERTOKEN:
             return self._USERTOKEN
 
+        username, _ = self._get_login_info()
+        AbemaTVBaseIE._USERTOKEN = username and self.cache.load(self._NETRC_MACHINE, username)
+        if AbemaTVBaseIE._USERTOKEN:
+            # try authentication with locally stored token
+            try:
+                self._get_media_token(True)
+                return
+            except ExtractorError as e:
+                self.report_warning(f'Failed to login with cached user token; obtaining a fresh one ({e})')
+
         AbemaTVBaseIE._DEVICE_ID = str(uuid.uuid4())
         aks = self._generate_aks(self._DEVICE_ID)
         user_data = self._download_json(
@@ -300,6 +310,11 @@ class AbemaTVIE(AbemaTVBaseIE):
     _TIMETABLE = None
 
     def _perform_login(self, username, password):
+        self._get_device_token()
+        if self.cache.load(self._NETRC_MACHINE, username) and self._get_media_token():
+            self.write_debug('Skipping logging in')
+            return
+
         if '@' in username:  # don't strictly check if it's email address or not
             ep, method = 'user/email', 'email'
         else:
@@ -319,6 +334,7 @@ def _perform_login(self, username, password):
 
         AbemaTVBaseIE._USERTOKEN = login_response['token']
         self._get_media_token(True)
+        self.cache.store(self._NETRC_MACHINE, username, AbemaTVBaseIE._USERTOKEN)
 
     def _real_extract(self, url):
         # starting download using infojson from this extractor is undefined behavior,

From c6b657867ad68af6b930ed0aa11ec5d93ee187b7 Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Sun, 12 Feb 2023 15:43:20 +0100
Subject: [PATCH 055/871] [extractor/rcs] Fix extractors (#5700)

Authored by: nixxo, pukkandan
Closes #5683
---
 yt_dlp/extractor/rcs.py | 408 ++++++++++++++++++----------------------
 1 file changed, 182 insertions(+), 226 deletions(-)

diff --git a/yt_dlp/extractor/rcs.py b/yt_dlp/extractor/rcs.py
index b905f8d2e..2440858ca 100644
--- a/yt_dlp/extractor/rcs.py
+++ b/yt_dlp/extractor/rcs.py
@@ -3,9 +3,18 @@
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    HEADRequest,
     base_url,
     clean_html,
+    extract_attributes,
+    get_element_html_by_class,
+    get_element_html_by_id,
+    int_or_none,
     js_to_json,
+    mimetype2ext,
+    sanitize_url,
+    traverse_obj,
+    try_call,
     url_basename,
     urljoin,
 )
@@ -15,41 +24,8 @@ class RCSBaseIE(InfoExtractor):
     # based on VideoPlayerLoader.prototype.getVideoSrc
     # and VideoPlayerLoader.prototype.transformSrc from
     # https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs
-    _ALL_REPLACE = {
-        'media2vam.corriere.it.edgesuite.net':
-            'media2vam-corriere-it.akamaized.net',
-        'media.youreporter.it.edgesuite.net':
-            'media-youreporter-it.akamaized.net',
-        'corrierepmd.corriere.it.edgesuite.net':
-            'corrierepmd-corriere-it.akamaized.net',
-        'media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/':
-            'video.corriere.it/vr360/videos/',
-        '.net//': '.net/',
-    }
-    _MP4_REPLACE = {
-        'media2vam.corbologna.corriere.it.edgesuite.net':
-            'media2vam-bologna-corriere-it.akamaized.net',
-        'media2vam.corfiorentino.corriere.it.edgesuite.net':
-            'media2vam-fiorentino-corriere-it.akamaized.net',
-        'media2vam.cormezzogiorno.corriere.it.edgesuite.net':
-            'media2vam-mezzogiorno-corriere-it.akamaized.net',
-        'media2vam.corveneto.corriere.it.edgesuite.net':
-            'media2vam-veneto-corriere-it.akamaized.net',
-        'media2.oggi.it.edgesuite.net':
-            'media2-oggi-it.akamaized.net',
-        'media2.quimamme.it.edgesuite.net':
-            'media2-quimamme-it.akamaized.net',
-        'media2.amica.it.edgesuite.net':
-            'media2-amica-it.akamaized.net',
-        'media2.living.corriere.it.edgesuite.net':
-            'media2-living-corriere-it.akamaized.net',
-        'media2.style.corriere.it.edgesuite.net':
-            'media2-style-corriere-it.akamaized.net',
-        'media2.iodonna.it.edgesuite.net':
-            'media2-iodonna-it.akamaized.net',
-        'media2.leitv.it.edgesuite.net':
-            'media2-leitv-it.akamaized.net',
-    }
+    _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
+    _RCS_ID_RE = r'[\w-]+-\d{10}'
     _MIGRATION_MAP = {
         'videoamica-vh.akamaihd': 'amica',
         'media2-amica-it.akamaized': 'amica',
@@ -90,183 +66,140 @@ class RCSBaseIE(InfoExtractor):
         'vivimilano-vh.akamaihd': 'vivimilano',
         'media2-youreporter-it.akamaized': 'youreporter'
     }
-    _MIGRATION_MEDIA = {
-        'advrcs-vh.akamaihd': '',
-        'corriere-f.akamaihd': '',
-        'corrierepmd-corriere-it.akamaized': '',
-        'corrprotetto-vh.akamaihd': '',
-        'gazzetta-f.akamaihd': '',
-        'gazzettapmd-gazzetta-it.akamaized': '',
-        'gazzprotetto-vh.akamaihd': '',
-        'periodici-f.akamaihd': '',
-        'periodicisecure-vh.akamaihd': '',
-        'videocoracademy-vh.akamaihd': ''
-    }
 
     def _get_video_src(self, video):
-        mediaFiles = video.get('mediaProfile').get('mediaFile')
-        src = {}
-        # audio
-        if video.get('mediaType') == 'AUDIO':
-            for aud in mediaFiles:
-                # todo: check
-                src['mp3'] = aud.get('value')
-        # video
-        else:
-            for vid in mediaFiles:
-                if vid.get('mimeType') == 'application/vnd.apple.mpegurl':
-                    src['m3u8'] = vid.get('value')
-                if vid.get('mimeType') == 'video/mp4':
-                    src['mp4'] = vid.get('value')
+        for source in traverse_obj(video, (
+                'mediaProfile', 'mediaFile', lambda _, v: v.get('mimeType'))):
+            url = source['value']
+            for s, r in (
+                ('media2vam.corriere.it.edgesuite.net', 'media2vam-corriere-it.akamaized.net'),
+                ('media.youreporter.it.edgesuite.net', 'media-youreporter-it.akamaized.net'),
+                ('corrierepmd.corriere.it.edgesuite.net', 'corrierepmd-corriere-it.akamaized.net'),
+                ('media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/', 'video.corriere.it/vr360/videos/'),
+                ('http://', 'https://'),
+            ):
+                url = url.replace(s, r)
 
-        # replace host
-        for t in src:
-            for s, r in self._ALL_REPLACE.items():
-                src[t] = src[t].replace(s, r)
-            for s, r in self._MP4_REPLACE.items():
-                src[t] = src[t].replace(s, r)
+            type_ = mimetype2ext(source['mimeType'])
+            if type_ == 'm3u8' and '-vh.akamaihd' in url:
+                # still needed for some old content: see _TESTS #3
+                matches = re.search(r'(?:https?:)?//(?P<host>[\w\.\-]+)\.net/i(?P<path>.+)$', url)
+                if matches:
+                    url = f'https://vod.rcsobjects.it/hls/{self._MIGRATION_MAP[matches.group("host")]}{matches.group("path")}'
+            if traverse_obj(video, ('mediaProfile', 'geoblocking')) or (
+                    type_ == 'm3u8' and 'fcs.quotidiani_!' in url):
+                url = url.replace('vod.rcsobjects', 'vod-it.rcsobjects')
+            if type_ == 'm3u8' and 'vod' in url:
+                url = url.replace('.csmil', '.urlset')
+            if type_ == 'mp3':
+                url = url.replace('media2vam-corriere-it.akamaized.net', 'vod.rcsobjects.it/corriere')
 
-        # switch cdn
-        if 'mp4' in src and 'm3u8' in src:
-            if ('-lh.akamaihd' not in src.get('m3u8')
-                    and 'akamai' in src.get('mp4')):
-                if 'm3u8' in src:
-                    matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('m3u8'))
-                    src['m3u8'] = 'https://vod.rcsobjects.it/hls/%s%s' % (
-                        self._MIGRATION_MAP[matches.group('host')],
-                        matches.group('path').replace(
-                            '///', '/').replace(
-                            '//', '/').replace(
-                            '.csmil', '.urlset'
-                        )
-                    )
-                if 'mp4' in src:
-                    matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('mp4'))
-                    if matches:
-                        if matches.group('host') in self._MIGRATION_MEDIA:
-                            vh_stream = 'https://media2.corriereobjects.it'
-                            if src.get('mp4').find('fcs.quotidiani_!'):
-                                vh_stream = 'https://media2-it.corriereobjects.it'
-                            src['mp4'] = '%s%s' % (
-                                vh_stream,
-                                matches.group('path').replace(
-                                    '///', '/').replace(
-                                    '//', '/').replace(
-                                    '/fcs.quotidiani/mediacenter', '').replace(
-                                    '/fcs.quotidiani_!/mediacenter', '').replace(
-                                    'corriere/content/mediacenter/', '').replace(
-                                    'gazzetta/content/mediacenter/', '')
-                            )
-                        else:
-                            src['mp4'] = 'https://vod.rcsobjects.it/%s%s' % (
-                                self._MIGRATION_MAP[matches.group('host')],
-                                matches.group('path').replace('///', '/').replace('//', '/')
-                            )
+            yield {
+                'type': type_,
+                'url': url,
+                'bitrate': source.get('bitrate')
+            }
 
-        if 'mp3' in src:
-            src['mp3'] = src.get('mp3').replace(
-                'media2vam-corriere-it.akamaized.net',
-                'vod.rcsobjects.it/corriere')
-        if 'mp4' in src:
-            if src.get('mp4').find('fcs.quotidiani_!'):
-                src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
-        if 'm3u8' in src:
-            if src.get('m3u8').find('fcs.quotidiani_!'):
-                src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')
+    def _create_http_formats(self, m3u8_formats, video_id):
+        for f in m3u8_formats:
+            if f['vcodec'] == 'none':
+                continue
+            http_url = re.sub(r'(https?://[^/]+)/hls/([^?#]+?\.mp4).+', r'\g<1>/\g<2>', f['url'])
+            if http_url == f['url']:
+                continue
 
-        if 'geoblocking' in video.get('mediaProfile'):
-            if 'm3u8' in src:
-                src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')
-            if 'mp4' in src:
-                src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
-        if 'm3u8' in src:
-            if src.get('m3u8').find('csmil') and src.get('m3u8').find('vod'):
-                src['m3u8'] = src.get('m3u8').replace('.csmil', '.urlset')
+            http_f = f.copy()
+            del http_f['manifest_url']
+            format_id = try_call(lambda: http_f['format_id'].replace('hls-', 'https-'))
+            urlh = self._request_webpage(HEADRequest(http_url), video_id, fatal=False,
+                                         note=f'Check filesize for {format_id}')
+            if not urlh:
+                continue
 
-        return src
-
-    def _create_formats(self, urls, video_id):
-        formats = []
-        formats = self._extract_m3u8_formats(
-            urls.get('m3u8'), video_id, 'mp4', entry_protocol='m3u8_native',
-            m3u8_id='hls', fatal=False)
-
-        if urls.get('mp4'):
-            formats.append({
-                'format_id': 'http-mp4',
-                'url': urls['mp4']
+            http_f.update({
+                'format_id': format_id,
+                'url': http_url,
+                'protocol': 'https',
+                'filesize_approx': int_or_none(urlh.headers.get('Content-Length', None)),
             })
-        return formats
+            yield http_f
+
+    def _create_formats(self, sources, video_id):
+        for source in sources:
+            if source['type'] == 'm3u8':
+                m3u8_formats = self._extract_m3u8_formats(
+                    source['url'], video_id, 'mp4', m3u8_id='hls', fatal=False)
+                yield from m3u8_formats
+                yield from self._create_http_formats(m3u8_formats, video_id)
+            elif source['type'] == 'mp3':
+                yield {
+                    'format_id': 'https-mp3',
+                    'ext': 'mp3',
+                    'acodec': 'mp3',
+                    'vcodec': 'none',
+                    'abr': source.get('bitrate'),
+                    'url': source['url'],
+                }
 
     def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
+        cdn, video_id = self._match_valid_url(url).group('cdn', 'id')
+        display_id, video_data = None, None
 
-        if 'cdn' not in mobj.groupdict():
-            raise ExtractorError('CDN not found in url: %s' % url)
-
-        # for leitv/youreporter/viaggi don't use the embed page
-        if ((mobj.group('cdn') not in ['leitv.it', 'youreporter.it'])
-                and (mobj.group('vid') == 'video')):
-            url = 'https://video.%s/video-embed/%s' % (mobj.group('cdn'), video_id)
-
-        page = self._download_webpage(url, video_id)
-
-        video_data = None
-        # look for json video data url
-        json = self._search_regex(
-            r'''(?x)url\s*=\s*(["'])
-            (?P<url>
-                (?:https?:)?//video\.rcs\.it
-                /fragment-includes/video-includes/.+?\.json
-            )\1;''',
-            page, video_id, group='url', default=None)
-        if json:
-            if json.startswith('//'):
-                json = 'https:%s' % json
-            video_data = self._download_json(json, video_id)
-
-        # if json url not found, look for json video data directly in the page
+        if re.match(self._UUID_RE, video_id) or re.match(self._RCS_ID_RE, video_id):
+            url = f'https://video.{cdn}/video-json/{video_id}'
         else:
-            # RCS normal pages and most of the embeds
-            json = self._search_regex(
-                r'[\s;]video\s*=\s*({[\s\S]+?})(?:;|,playlist=)',
-                page, video_id, default=None)
-            if not json and 'video-embed' in url:
-                page = self._download_webpage(url.replace('video-embed', 'video-json'), video_id)
-                json = self._search_regex(
-                    r'##start-video##({[\s\S]+?})##end-video##',
-                    page, video_id, default=None)
-            if not json:
-                # if no video data found try search for iframes
-                emb = RCSEmbedsIE._extract_url(page)
+            webpage = self._download_webpage(url, video_id)
+            data_config = get_element_html_by_id('divVideoPlayer', webpage) or get_element_html_by_class('divVideoPlayer', webpage)
+
+            if data_config:
+                data_config = self._parse_json(
+                    extract_attributes(data_config).get('data-config'),
+                    video_id, fatal=False) or {}
+                if data_config.get('newspaper'):
+                    cdn = f'{data_config["newspaper"]}.it'
+                display_id, video_id = video_id, data_config.get('uuid') or video_id
+                url = f'https://video.{cdn}/video-json/{video_id}'
+            else:
+                json_url = self._search_regex(
+                    r'''(?x)url\s*=\s*(["'])
+                    (?P<url>
+                        (?:https?:)?//video\.rcs\.it
+                        /fragment-includes/video-includes/[^"']+?\.json
+                    )\1;''',
+                    webpage, video_id, group='url', default=None)
+                if json_url:
+                    video_data = self._download_json(sanitize_url(json_url, scheme='https'), video_id)
+                    display_id, video_id = video_id, video_data.get('id') or video_id
+
+        if not video_data:
+            webpage = self._download_webpage(url, video_id)
+
+            video_data = self._search_json(
+                '##start-video##', webpage, 'video data', video_id, default=None,
+                end_pattern='##end-video##', transform_source=js_to_json)
+
+            if not video_data:
+                # try search for iframes
+                emb = RCSEmbedsIE._extract_url(webpage)
                 if emb:
                     return {
                         '_type': 'url_transparent',
                         'url': emb,
                         'ie_key': RCSEmbedsIE.ie_key()
                     }
-            if json:
-                video_data = self._parse_json(
-                    json, video_id, transform_source=js_to_json)
 
         if not video_data:
             raise ExtractorError('Video data not found in the page')
 
-        formats = self._create_formats(
-            self._get_video_src(video_data), video_id)
-
-        description = (video_data.get('description')
-                       or clean_html(video_data.get('htmlDescription'))
-                       or self._html_search_meta('description', page))
-        uploader = video_data.get('provider') or mobj.group('cdn')
-
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': video_data.get('title'),
-            'description': description,
-            'uploader': uploader,
-            'formats': formats
+            'description': (clean_html(video_data.get('description'))
+                            or clean_html(video_data.get('htmlDescription'))
+                            or self._html_search_meta('description', webpage)),
+            'uploader': video_data.get('provider') or cdn,
+            'formats': list(self._create_formats(self._get_video_src(video_data), video_id)),
         }
 
 
@@ -296,7 +229,7 @@ class RCSEmbedsIE(RCSBaseIE):
             \1''']
     _TESTS = [{
         'url': 'https://video.rcs.it/video-embed/iodonna-0001585037',
-        'md5': '623ecc8ffe7299b2d0c1046d8331a9df',
+        'md5': '0faca97df525032bb9847f690bc3720c',
         'info_dict': {
             'id': 'iodonna-0001585037',
             'ext': 'mp4',
@@ -305,38 +238,31 @@ class RCSEmbedsIE(RCSBaseIE):
             'uploader': 'rcs.it',
         }
     }, {
-        # redownload the page changing 'video-embed' in 'video-json'
         'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789',
-        'md5': 'a043e3fecbe4d9ed7fc5d888652a5440',
-        'info_dict': {
-            'id': 'gazzanet-mo05-0000260789',
-            'ext': 'mp4',
-            'title': 'Valentino Rossi e papà Graziano si divertono col drifting',
-            'description': 'md5:a8bf90d6adafd9815f70fc74c0fc370a',
-            'uploader': 'rcd',
-        }
-    }, {
-        'url': 'https://video.corriere.it/video-embed/b727632a-f9d0-11ea-91b0-38d50a849abb?player',
         'match_only': True
     }, {
         'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140',
         'match_only': True
     }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://www.iodonna.it/video-iodonna/personaggi-video/monica-bellucci-piu-del-lavoro-oggi-per-me-sono-importanti-lamicizia-e-la-famiglia/',
+        'info_dict': {
+            'id': 'iodonna-0002033648',
+            'ext': 'mp4',
+            'title': 'Monica Bellucci: «Più del lavoro, oggi per me sono importanti l\'amicizia e la famiglia»',
+            'description': 'md5:daea6d9837351e56b1ab615c06bebac1',
+            'uploader': 'rcs.it',
+        }
+    }]
 
     @staticmethod
-    def _sanitize_urls(urls):
-        # add protocol if missing
-        for i, e in enumerate(urls):
-            if e.startswith('//'):
-                urls[i] = 'https:%s' % e
-        # clean iframes urls
-        for i, e in enumerate(urls):
-            urls[i] = urljoin(base_url(e), url_basename(e))
-        return urls
+    def _sanitize_url(url):
+        url = sanitize_url(url, scheme='https')
+        return urljoin(base_url(url), url_basename(url))
 
     @classmethod
     def _extract_embed_urls(cls, url, webpage):
-        return cls._sanitize_urls(list(super()._extract_embed_urls(url, webpage)))
+        return map(cls._sanitize_url, super()._extract_embed_urls(url, webpage))
 
 
 class RCSIE(RCSBaseIE):
@@ -349,37 +275,53 @@ class RCSIE(RCSBaseIE):
                         |corrierefiorentino\.
                     )?corriere\.it
                     |(?:gazzanet\.)?gazzetta\.it)
-                    /(?!video-embed/).+?/(?P<id>[^/\?]+)(?=\?|/$|$)'''
+                    /(?!video-embed/)[^?#]+?/(?P<id>[^/\?]+)(?=\?|/$|$)'''
     _TESTS = [{
+        # json iframe directly from id
         'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb',
-        'md5': '0f4ededc202b0f00b6e509d831e2dcda',
+        'md5': '14946840dec46ecfddf66ba4eea7d2b2',
         'info_dict': {
             'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb',
             'ext': 'mp4',
             'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante',
-            'description': 'md5:93b51c9161ac8a64fb2f997b054d0152',
+            'description': 'md5:3915ce5ebb3d2571deb69a5eb85ac9b5',
             'uploader': 'Corriere Tv',
         }
     }, {
-        # video data inside iframe
+        # search for video id inside the page
         'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/',
-        'md5': 'da378e4918d2afbf7d61c35abb948d4c',
+        'md5': 'f22a92d9e666e80f2fffbf2825359c81',
         'info_dict': {
             'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2',
+            'display_id': 'norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen',
             'ext': 'mp4',
             'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen',
             'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8',
             'uploader': 'DOVE Viaggi',
         }
     }, {
-        'url': 'https://video.gazzetta.it/video-motogp-catalogna-cadute-dovizioso-vale-rossi/49612410-00ca-11eb-bcd8-30d4253e0140?vclk=Videobar',
-        'md5': 'eedc1b5defd18e67383afef51ff7bdf9',
+        # only audio format https://github.com/yt-dlp/yt-dlp/issues/5683
+        'url': 'https://video.corriere.it/cronaca/audio-telefonata-il-papa-becciu-santita-lettera-che-mi-ha-inviato-condanna/b94c0d20-70c2-11ed-9572-e4b947a0ebd2',
+        'md5': 'aaffb08d02f2ce4292a4654694c78150',
         'info_dict': {
-            'id': '49612410-00ca-11eb-bcd8-30d4253e0140',
+            'id': 'b94c0d20-70c2-11ed-9572-e4b947a0ebd2',
+            'ext': 'mp3',
+            'title': 'L\'audio della telefonata tra il Papa e Becciu: «Santità, la lettera che mi ha inviato è una condanna»',
+            'description': 'md5:c0ddb61bd94a8d4e0d4bb9cda50a689b',
+            'uploader': 'Corriere Tv',
+            'formats': [{'format_id': 'https-mp3', 'ext': 'mp3'}],
+        }
+    }, {
+        # old content still needs cdn migration
+        'url': 'https://viaggi.corriere.it/video/milano-varallo-sesia-sul-treno-a-vapore/',
+        'md5': '2dfdce7af249654ad27eeba03fe1e08d',
+        'info_dict': {
+            'id': 'd8f6c8d0-f7d7-11e8-bfca-f74cf4634191',
+            'display_id': 'milano-varallo-sesia-sul-treno-a-vapore',
             'ext': 'mp4',
-            'title': 'Dovizioso, il contatto con Zarco e la caduta. E anche Vale finisce a terra',
-            'description': 'md5:8c6e905dc3b9413218beca11ebd69778',
-            'uploader': 'AMorici',
+            'title': 'Milano-Varallo Sesia sul treno a vapore',
+            'description': 'md5:6348f47aac230397fe341a74f7678d53',
+            'uploader': 'DOVE Viaggi',
         }
     }, {
         'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945',
@@ -391,13 +333,15 @@ class RCSVariousIE(RCSBaseIE):
     _VALID_URL = r'''(?x)https?://www\.
                     (?P<cdn>
                         leitv\.it|
-                        youreporter\.it
+                        youreporter\.it|
+                        amica\.it
                     )/(?:[^/]+/)?(?P<id>[^/]+?)(?:$|\?|/)'''
     _TESTS = [{
-        'url': 'https://www.leitv.it/benessere/mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa/',
-        'md5': '92b4e63667b8f95acb0a04da25ae28a1',
+        'url': 'https://www.leitv.it/benessere/mal-di-testa/',
+        'md5': '3b7a683d105a7313ec7513b014443631',
         'info_dict': {
-            'id': 'mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa',
+            'id': 'leitv-0000125151',
+            'display_id': 'mal-di-testa',
             'ext': 'mp4',
             'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto',
             'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5',
@@ -405,12 +349,24 @@ class RCSVariousIE(RCSBaseIE):
         }
     }, {
         'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/',
-        'md5': '8dccd436b47a830bab5b4a88232f391a',
+        'md5': '3989b6d603482611a2abd2f32b79f739',
         'info_dict': {
-            'id': 'fiume-sesia-3-ottobre-2020',
+            'id': 'youreporter-0000332574',
+            'display_id': 'fiume-sesia-3-ottobre-2020',
             'ext': 'mp4',
             'title': 'Fiume Sesia 3 ottobre 2020',
             'description': 'md5:0070eef1cc884d13c970a4125063de55',
             'uploader': 'youreporter.it',
         }
+    }, {
+        'url': 'https://www.amica.it/video-post/saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi/',
+        'md5': '187cce524dfd0343c95646c047375fc4',
+        'info_dict': {
+            'id': 'amica-0001225365',
+            'display_id': 'saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi',
+            'ext': 'mp4',
+            'title': '"Saint Omer": al cinema il film Leone d\'argento che ribalta gli stereotipi',
+            'description': 'md5:b1c8869c2dcfd6073a2a311ba0008aa8',
+            'uploader': 'rcs.it',
+        }
     }]

From 365b9006051ac7d735c20bb63c4907b758233048 Mon Sep 17 00:00:00 2001
From: shirt <danbaerwalde@gmail.com>
Date: Sun, 12 Feb 2023 10:57:57 -0500
Subject: [PATCH 056/871] [Build] Update pyinstaller

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 49b9411fd..6041376a4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -255,7 +255,7 @@ jobs:
     - name: Install Requirements
       run: |  # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
           python -m pip install -U pip setuptools wheel py2exe
-          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.3-py3-none-any.whl" -r requirements.txt
+          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
 
     - name: Prepare
       run: |
@@ -291,7 +291,7 @@ jobs:
     - name: Install Requirements
       run: |
           python -m pip install -U pip setuptools wheel
-          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.3-py3-none-any.whl" -r requirements.txt
+          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
 
     - name: Prepare
       run: |

From 39f32f1715c0dffb7626dda7307db6388bb7abaa Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 13 Feb 2023 01:14:43 +0530
Subject: [PATCH 057/871] Sanitize formats before sorting

Closes #4501
---
 yt_dlp/YoutubeDL.py | 78 +++++++++++++++++++++------------------------
 1 file changed, 36 insertions(+), 42 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 8f88104ef..4b652d172 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2561,7 +2561,6 @@ def sanitize_numeric_fields(info):
         info_dict['requested_subtitles'] = self.process_subtitles(
             info_dict['id'], subtitles, automatic_captions)
 
-        self.sort_formats(info_dict)
         formats = self._get_formats(info_dict)
 
         # or None ensures --clean-infojson removes it
@@ -2601,44 +2600,12 @@ def is_wellformed(f):
         if not formats:
             self.raise_no_formats(info_dict)
 
-        formats_dict = {}
-
-        # We check that all the formats have the format and format_id fields
-        for i, format in enumerate(formats):
+        for format in formats:
             sanitize_string_field(format, 'format_id')
             sanitize_numeric_fields(format)
             format['url'] = sanitize_url(format['url'])
-            if not format.get('format_id'):
-                format['format_id'] = str(i)
-            else:
-                # Sanitize format_id from characters used in format selector expression
-                format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
-            format_id = format['format_id']
-            if format_id not in formats_dict:
-                formats_dict[format_id] = []
-            formats_dict[format_id].append(format)
-
-        # Make sure all formats have unique format_id
-        common_exts = set(itertools.chain(*self._format_selection_exts.values()))
-        for format_id, ambiguous_formats in formats_dict.items():
-            ambigious_id = len(ambiguous_formats) > 1
-            for i, format in enumerate(ambiguous_formats):
-                if ambigious_id:
-                    format['format_id'] = '%s-%d' % (format_id, i)
-                if format.get('ext') is None:
-                    format['ext'] = determine_ext(format['url']).lower()
-                # Ensure there is no conflict between id and ext in format selection
-                # See https://github.com/yt-dlp/yt-dlp/issues/1282
-                if format['format_id'] != format['ext'] and format['format_id'] in common_exts:
-                    format['format_id'] = 'f%s' % format['format_id']
-
-        for i, format in enumerate(formats):
-            if format.get('format') is None:
-                format['format'] = '{id} - {res}{note}'.format(
-                    id=format['format_id'],
-                    res=self.format_resolution(format),
-                    note=format_field(format, 'format_note', ' (%s)'),
-                )
+            if format.get('ext') is None:
+                format['ext'] = determine_ext(format['url']).lower()
             if format.get('protocol') is None:
                 format['protocol'] = determine_protocol(format)
             if format.get('resolution') is None:
@@ -2650,16 +2617,43 @@ def is_wellformed(f):
             if (info_dict.get('duration') and format.get('tbr')
                     and not format.get('filesize') and not format.get('filesize_approx')):
                 format['filesize_approx'] = int(info_dict['duration'] * format['tbr'] * (1024 / 8))
+            format['http_headers'] = self._calc_headers(collections.ChainMap(format, info_dict))
 
-            # Add HTTP headers, so that external programs can use them from the
-            # json output
-            full_format_info = info_dict.copy()
-            full_format_info.update(format)
-            format['http_headers'] = self._calc_headers(full_format_info)
-        # Remove private housekeeping stuff
+        # This is copied to http_headers by the above _calc_headers and can now be removed
         if '__x_forwarded_for_ip' in info_dict:
             del info_dict['__x_forwarded_for_ip']
 
+        self.sort_formats({'formats': formats})
+
+        # Sanitize and group by format_id
+        formats_dict = {}
+        for i, format in enumerate(formats):
+            if not format.get('format_id'):
+                format['format_id'] = str(i)
+            else:
+                # Sanitize format_id from characters used in format selector expression
+                format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
+            formats_dict.setdefault(format['format_id'], []).append(format)
+
+        # Make sure all formats have unique format_id
+        common_exts = set(itertools.chain(*self._format_selection_exts.values()))
+        for format_id, ambiguous_formats in formats_dict.items():
+            ambigious_id = len(ambiguous_formats) > 1
+            for i, format in enumerate(ambiguous_formats):
+                if ambigious_id:
+                    format['format_id'] = '%s-%d' % (format_id, i)
+                # Ensure there is no conflict between id and ext in format selection
+                # See https://github.com/yt-dlp/yt-dlp/issues/1282
+                if format['format_id'] != format['ext'] and format['format_id'] in common_exts:
+                    format['format_id'] = 'f%s' % format['format_id']
+
+                if format.get('format') is None:
+                    format['format'] = '{id} - {res}{note}'.format(
+                        id=format['format_id'],
+                        res=self.format_resolution(format),
+                        note=format_field(format, 'format_note', ' (%s)'),
+                    )
+
         if self.params.get('check_formats') is True:
             formats = LazyList(self._check_formats(formats[::-1]), reverse=True)
 

From 5712943b764ba819ef479524c32700228603817a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 13 Feb 2023 01:19:46 +0530
Subject: [PATCH 058/871] Imply `--no-progress` when `--print`

---
 yt_dlp/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index df1a54138..255b31735 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -708,6 +708,7 @@ def parse_options(argv=None):
         'dumpjson', 'dump_single_json', 'getdescription', 'getduration', 'getfilename',
         'getformat', 'getid', 'getthumbnail', 'gettitle', 'geturl'
     ))
+    opts.quiet = opts.quiet or any_getting or opts.print_json or bool(opts.forceprint)
 
     playlist_pps = [pp for pp in postprocessors if pp.get('when') == 'playlist']
     write_playlist_infojson = (opts.writeinfojson and not opts.clean_infojson
@@ -743,7 +744,7 @@ def parse_options(argv=None):
         'client_certificate': opts.client_certificate,
         'client_certificate_key': opts.client_certificate_key,
         'client_certificate_password': opts.client_certificate_password,
-        'quiet': opts.quiet or any_getting or opts.print_json or bool(opts.forceprint),
+        'quiet': opts.quiet,
         'no_warnings': opts.no_warnings,
         'forceurl': opts.geturl,
         'forcetitle': opts.gettitle,

From c154302c588c3d4362cec4fc5545e7e5d2bcf7a3 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 13 Feb 2023 01:34:27 +0530
Subject: [PATCH 059/871] Bugfix for 39f32f1715c0dffb7626dda7307db6388bb7abaa

---
 yt_dlp/YoutubeDL.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 4b652d172..d214a6449 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2466,15 +2466,8 @@ def _raise_pending_errors(self, info):
 
     def sort_formats(self, info_dict):
         formats = self._get_formats(info_dict)
-        if not formats:
-            return
-        # Backward compatibility with InfoExtractor._sort_formats
-        field_preference = formats[0].pop('__sort_fields', None)
-        if field_preference:
-            info_dict['_format_sort_fields'] = field_preference
-
         formats.sort(key=FormatSorter(
-            self, info_dict.get('_format_sort_fields', [])).calculate_preference)
+            self, info_dict.get('_format_sort_fields') or []).calculate_preference)
 
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'
@@ -2563,6 +2556,11 @@ def sanitize_numeric_fields(info):
 
         formats = self._get_formats(info_dict)
 
+        # Backward compatibility with InfoExtractor._sort_formats
+        field_preference = formats[0].pop('__sort_fields', None)
+        if field_preference:
+            info_dict['_format_sort_fields'] = field_preference
+
         # or None ensures --clean-infojson removes it
         info_dict['_has_drm'] = any(f.get('has_drm') for f in formats) or None
         if not self.params.get('allow_unplayable_formats'):
@@ -2623,7 +2621,10 @@ def is_wellformed(f):
         if '__x_forwarded_for_ip' in info_dict:
             del info_dict['__x_forwarded_for_ip']
 
-        self.sort_formats({'formats': formats})
+        self.sort_formats({
+            'formats': formats,
+            '_format_sort_fields': info_dict.get('_format_sort_fields')
+        })
 
         # Sanitize and group by format_id
         formats_dict = {}

From a9c685453f7019bee94170f936619c6db76c964e Mon Sep 17 00:00:00 2001
From: Marenga <107524538+the-marenga@users.noreply.github.com>
Date: Mon, 13 Feb 2023 07:07:47 +0100
Subject: [PATCH 060/871] [extractor/vk] Fix playlists for new API (#6122)

Authored by: the-marenga
Closes #6219
---
 yt_dlp/extractor/vk.py | 184 ++++++++++++++++-------------------------
 1 file changed, 73 insertions(+), 111 deletions(-)

diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py
index 0fb95c863..16ca954f2 100644
--- a/yt_dlp/extractor/vk.py
+++ b/yt_dlp/extractor/vk.py
@@ -9,20 +9,22 @@
 from .sibnet import SibnetEmbedIE
 from .vimeo import VimeoIE
 from .youtube import YoutubeIE
-from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     clean_html,
     get_element_by_class,
+    get_element_html_by_id,
     int_or_none,
-    orderedSet,
+    join_nonempty,
     str_or_none,
     str_to_int,
+    try_call,
     unescapeHTML,
     unified_timestamp,
     update_url_query,
     url_or_none,
     urlencode_postdata,
+    urljoin,
 )
 
 
@@ -117,7 +119,7 @@ class VKIE(VKBaseIE):
                 'upload_date': '20120212',
                 'comment_count': int,
                 'like_count': int,
-                'thumbnail': r're:https?://.+\.jpg$',
+                'thumbnail': r're:https?://.+(?:\.jpg|getVideoPreview.*)$',
             },
             'params': {'skip_download': 'm3u8'},
         },
@@ -134,7 +136,7 @@ class VKIE(VKBaseIE):
                 'upload_date': '20130720',
                 'comment_count': int,
                 'like_count': int,
-                'thumbnail': r're:https?://.+\.jpg$',
+                'thumbnail': r're:https?://.+(?:\.jpg|getVideoPreview.*)$',
             }
         },
         {
@@ -149,55 +151,10 @@ class VKIE(VKBaseIE):
                 'upload_date': '20120212',
                 'timestamp': 1329049880,
                 'uploader_id': '39545378',
-                'thumbnail': r're:https?://.+\.jpg$',
+                'thumbnail': r're:https?://.+(?:\.jpg|getVideoPreview.*)$',
             },
             'params': {'skip_download': 'm3u8'},
         },
-        {
-            # VIDEO NOW REMOVED
-            # please update if you find a video whose URL follows the same pattern
-            'url': 'http://vk.com/video-8871596_164049491',
-            'md5': 'a590bcaf3d543576c9bd162812387666',
-            'note': 'Only available for registered users',
-            'info_dict': {
-                'id': '-8871596_164049491',
-                'ext': 'mp4',
-                'uploader': 'Триллеры',
-                'title': '► Бойцовский клуб / Fight Club 1999 [HD 720]',
-                'duration': 8352,
-                'upload_date': '20121218',
-                'view_count': int,
-            },
-            'skip': 'Removed',
-        },
-        {
-            'url': 'http://vk.com/hd_kino_mania?z=video-43215063_168067957%2F15c66b9b533119788d',
-            'info_dict': {
-                'id': '-43215063_168067957',
-                'ext': 'mp4',
-                'uploader': 'Bro Mazter',
-                'title': ' ',
-                'duration': 7291,
-                'upload_date': '20140328',
-                'uploader_id': '223413403',
-                'timestamp': 1396018030,
-            },
-            'skip': 'Requires vk account credentials',
-        },
-        {
-            'url': 'http://m.vk.com/video-43215063_169084319?list=125c627d1aa1cebb83&from=wall-43215063_2566540',
-            'md5': '0c45586baa71b7cb1d0784ee3f4e00a6',
-            'note': 'ivi.ru embed',
-            'info_dict': {
-                'id': '-43215063_169084319',
-                'ext': 'mp4',
-                'title': 'Книга Илая',
-                'duration': 6771,
-                'upload_date': '20140626',
-                'view_count': int,
-            },
-            'skip': 'Removed',
-        },
         {
             'url': 'https://vk.com/video-93049196_456239755?list=ln-cBjJ7S4jYYx3ADnmDT',
             'info_dict': {
@@ -211,26 +168,11 @@ class VKIE(VKBaseIE):
                 'timestamp': 1640162189,
                 'upload_date': '20211222',
                 'uploader_id': '-93049196',
-                'thumbnail': r're:https?://.+\.jpg$',
+                'thumbnail': r're:https?://.+(?:\.jpg|getVideoPreview.*)$',
             },
         },
         {
-            # video (removed?) only available with list id
-            'url': 'https://vk.com/video30481095_171201961?list=8764ae2d21f14088d4',
-            'md5': '091287af5402239a1051c37ec7b92913',
-            'info_dict': {
-                'id': '30481095_171201961',
-                'ext': 'mp4',
-                'title': 'ТюменцевВВ_09.07.2015',
-                'uploader': 'Anton Ivanov',
-                'duration': 109,
-                'upload_date': '20150709',
-                'view_count': int,
-            },
-            'skip': 'Removed',
-        },
-        {
-            # youtube embed
+            'note': 'youtube embed',
             'url': 'https://vk.com/video276849682_170681728',
             'info_dict': {
                 'id': 'V3K4mi0SYkc',
@@ -254,23 +196,45 @@ class VKIE(VKBaseIE):
                 'start_time': 0.0,
                 'categories': ['Nonprofits & Activism'],
                 'channel_url': 'https://www.youtube.com/channel/UCgzCNQ11TmR9V97ECnhi3gw',
+                'channel_follower_count': int,
                 'age_limit': 0,
             },
         },
         {
-            # dailymotion embed
-            'url': 'https://vk.com/video-37468416_456239855',
+            'note': 'dailymotion embed',
+            'url': 'https://vk.com/video-95168827_456239103?list=cca524a0f0d5557e16',
             'info_dict': {
-                'id': 'k3lz2cmXyRuJQSjGHUv',
+                'id': 'x8gfli0',
                 'ext': 'mp4',
-                'title': 'md5:d52606645c20b0ddbb21655adaa4f56f',
-                'description': 'md5:424b8e88cc873217f520e582ba28bb36',
-                'uploader': 'AniLibria.Tv',
-                'upload_date': '20160914',
-                'uploader_id': 'x1p5vl5',
-                'timestamp': 1473877246,
+                'title': 'md5:45410f60ccd4b2760da98cb5fc777d70',
+                'description': 'md5:2e71c5c9413735cfa06cf1a166f16c84',
+                'uploader': 'Movies and cinema.',
+                'upload_date': '20221218',
+                'uploader_id': 'x1jdavv',
+                'timestamp': 1671387617,
+                'age_limit': 0,
+                'duration': 2918,
+                'like_count': int,
+                'view_count': int,
+                'thumbnail': r're:https?://.+x1080$',
+                'tags': list
+            },
+        },
+        {
+            'url': 'https://vk.com/clips-74006511?z=clip-74006511_456247211',
+            'info_dict': {
+                'id': '-74006511_456247211',
+                'ext': 'mp4',
+                'comment_count': int,
+                'duration': 9,
+                'like_count': int,
+                'thumbnail': r're:https?://.+(?:\.jpg|getVideoPreview.*)$',
+                'timestamp': 1664995597,
+                'title': 'Clip by @madempress',
+                'upload_date': '20221005',
+                'uploader': 'Шальная императрица',
+                'uploader_id': '-74006511',
             },
-            'skip': 'Removed'
         },
         {
             # video key is extra_data not url\d+
@@ -288,7 +252,7 @@ class VKIE(VKBaseIE):
             'skip': 'Removed',
         },
         {
-            # finished live stream, postlive_mp4
+            'note': 'finished live stream, postlive_mp4',
             'url': 'https://vk.com/videos-387766?z=video-387766_456242764%2Fpl_-387766_-2',
             'info_dict': {
                 'id': '-387766_456242764',
@@ -552,7 +516,7 @@ class VKUserVideosIE(VKBaseIE):
     }, {
         'url': 'https://vk.com/video/playlist/-174476437_2',
         'info_dict': {
-            'id': '-174476437_2',
+            'id': '-174476437_playlist_2',
             'title': 'Анонсы'
         },
         'playlist_mincount': 108,
@@ -595,6 +559,7 @@ def _real_extract(self, url):
             page_id = self._search_regex(r'data-owner-id\s?=\s?"([^"]+)"', webpage, 'page_id')
         elif '_' in u_id:
             page_id, section = u_id.split('_', 1)
+            section = f'playlist_{section}'
         else:
             raise ExtractorError('Invalid URL', expected=True)
 
@@ -614,13 +579,13 @@ class VKWallPostIE(VKBaseIE):
         'info_dict': {
             'id': '-23538238_35',
             'title': 'Black Shadow - Wall post -23538238_35',
-            'description': 'md5:3f84b9c4f9ef499731cf1ced9998cc0c',
+            'description': 'md5:190c78f905a53e0de793d83933c6e67f',
         },
         'playlist': [{
             'md5': '5ba93864ec5b85f7ce19a9af4af080f6',
             'info_dict': {
                 'id': '135220665_111806521',
-                'ext': 'mp4',
+                'ext': 'm4a',
                 'title': 'Black Shadow - Слепое Верование',
                 'duration': 370,
                 'uploader': 'Black Shadow',
@@ -631,7 +596,7 @@ class VKWallPostIE(VKBaseIE):
             'md5': '4cc7e804579122b17ea95af7834c9233',
             'info_dict': {
                 'id': '135220665_111802303',
-                'ext': 'mp4',
+                'ext': 'm4a',
                 'title': 'Black Shadow - Война - Негасимое Бездны Пламя!',
                 'duration': 423,
                 'uploader': 'Black Shadow',
@@ -642,16 +607,15 @@ class VKWallPostIE(VKBaseIE):
         'params': {
             'skip_download': True,
         },
-        'skip': 'Requires vk account credentials',
     }, {
-        # single YouTube embed, no leading -
-        'url': 'https://vk.com/wall85155021_6319',
+        # single YouTube embed with irrelevant reaction videos
+        'url': 'https://vk.com/wall-32370614_7173954',
         'info_dict': {
-            'id': '85155021_6319',
-            'title': 'Сергей Горбунов - Wall post 85155021_6319',
+            'id': '-32370614_7173954',
+            'title': 'md5:9f93c405bbc00061d34007d78c75e3bc',
+            'description': 'md5:953b811f26fa9f21ee5856e2ea8e68fc',
         },
         'playlist_count': 1,
-        'skip': 'Requires vk account credentials',
     }, {
         # wall page URL
         'url': 'https://vk.com/wall-23538238_35',
@@ -703,39 +667,37 @@ def _real_extract(self, url):
             'w': 'wall' + post_id,
         })[1]
 
-        description = clean_html(get_element_by_class('wall_post_text', webpage))
-        uploader = clean_html(get_element_by_class('author', webpage))
+        uploader = clean_html(get_element_by_class('PostHeaderTitle__authorName', webpage))
 
         entries = []
 
         for audio in re.findall(r'data-audio="([^"]+)', webpage):
             audio = self._parse_json(unescapeHTML(audio), post_id)
-            a = self._AUDIO._make(audio[:16])
-            if not a.url:
+            if not audio['url']:
                 continue
-            title = unescapeHTML(a.title)
-            performer = unescapeHTML(a.performer)
+            title = unescapeHTML(audio.get('title'))
+            artist = unescapeHTML(audio.get('artist'))
             entries.append({
-                'id': '%s_%s' % (a.owner_id, a.id),
-                'url': self._unmask_url(a.url, a.ads['vk_id']),
-                'title': '%s - %s' % (performer, title) if performer else title,
-                'thumbnails': [{'url': c_url} for c_url in a.cover_url.split(',')] if a.cover_url else None,
-                'duration': int_or_none(a.duration),
+                'id': f'{audio["owner_id"]}_{audio["id"]}',
+                'title': join_nonempty(artist, title, delim=' - '),
+                'thumbnails': try_call(lambda: [{'url': u} for u in audio['coverUrl'].split(',')]),
+                'duration': int_or_none(audio.get('duration')),
                 'uploader': uploader,
-                'artist': performer,
+                'artist': artist,
                 'track': title,
-                'ext': 'mp4',
-                'protocol': 'm3u8_native',
+                'formats': [{
+                    'url': audio['url'],
+                    'ext': 'm4a',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'container': 'm4a_dash',
+                }],
             })
 
-        for video in re.finditer(
-                r'<a[^>]+href=(["\'])(?P<url>/video(?:-?[\d_]+).*?)\1', webpage):
-            entries.append(self.url_result(
-                compat_urlparse.urljoin(url, video.group('url')), VKIE.ie_key()))
-
-        title = 'Wall post %s' % post_id
+        entries.extend(self.url_result(urljoin(url, entry), VKIE) for entry in set(re.findall(
+            r'<a[^>]+href=(?:["\'])(/video(?:-?[\d_]+)[^"\']*)',
+            get_element_html_by_id('wl_post_body', webpage))))
 
         return self.playlist_result(
-            orderedSet(entries), post_id,
-            '%s - %s' % (uploader, title) if uploader else title,
-            description)
+            entries, post_id, join_nonempty(uploader, f'Wall post {post_id}', delim=' - '),
+            clean_html(get_element_by_class('wall_post_text', webpage)))

From 44699d10dc8de9c6a338f4a8e5c63506ec4d2118 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 14 Feb 2023 01:06:38 +0530
Subject: [PATCH 061/871] [extractor/crunchyroll] Better message for premium
 videos

Closes #6227
---
 yt_dlp/extractor/crunchyroll.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py
index 3c9f8bbf0..7d356d673 100644
--- a/yt_dlp/extractor/crunchyroll.py
+++ b/yt_dlp/extractor/crunchyroll.py
@@ -20,8 +20,12 @@ class CrunchyrollBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'crunchyroll'
     params = None
 
+    @property
+    def is_logged_in(self):
+        return self._get_cookies(self._LOGIN_URL).get('etp_rt')
+
     def _perform_login(self, username, password):
-        if self._get_cookies(self._LOGIN_URL).get('etp_rt'):
+        if self.is_logged_in:
             return
 
         upsell_response = self._download_json(
@@ -46,7 +50,7 @@ def _perform_login(self, username, password):
             }).encode('ascii'))
         if login_response['code'] != 'ok':
             raise ExtractorError('Login failed. Server message: %s' % login_response['message'], expected=True)
-        if not self._get_cookies(self._LOGIN_URL).get('etp_rt'):
+        if not self.is_logged_in:
             raise ExtractorError('Login succeeded but did not set etp_rt cookie')
 
     def _get_embedded_json(self, webpage, display_id):
@@ -157,7 +161,10 @@ def _real_extract(self, url):
             f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
             note='Retrieving episode metadata', query=params)
         if episode_response.get('is_premium_only') and not episode_response.get('playback'):
-            raise ExtractorError('This video is for premium members only.', expected=True)
+            if self.is_logged_in:
+                raise ExtractorError('This video is for premium members only', expected=True)
+            else:
+                self.raise_login_required('This video is for premium members only')
 
         stream_response = self._download_json(
             f'{api_domain}{episode_response["__links__"]["streams"]["href"]}', display_id,

From d3bb187f01e1e30db05e639fc23a2e1935d777fe Mon Sep 17 00:00:00 2001
From: Greg Sadetsky <lepetitg@gmail.com>
Date: Mon, 13 Feb 2023 21:52:27 -0500
Subject: [PATCH 062/871] [extractor/NZOnScreen] Add extractor (#6208)

Authored by: gregsadetsky, pukkandan
Closes #6193
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/nzonscreen.py  | 93 +++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100644 yt_dlp/extractor/nzonscreen.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 12ef50cc6..8b3875130 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1287,6 +1287,7 @@
 )
 from .nuvid import NuvidIE
 from .nzherald import NZHeraldIE
+from .nzonscreen import NZOnScreenIE
 from .nzz import NZZIE
 from .odatv import OdaTVIE
 from .odnoklassniki import OdnoklassnikiIE
diff --git a/yt_dlp/extractor/nzonscreen.py b/yt_dlp/extractor/nzonscreen.py
new file mode 100644
index 000000000..6926bc5b2
--- /dev/null
+++ b/yt_dlp/extractor/nzonscreen.py
@@ -0,0 +1,93 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    remove_end,
+    strip_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class NZOnScreenIE(InfoExtractor):
+    _VALID_URL = r'^https://www\.nzonscreen\.com/title/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.nzonscreen.com/title/shoop-shoop-diddy-wop-cumma-cumma-wang-dang-1982',
+        'info_dict': {
+            'id': '726ed6585c6bfb30',
+            'ext': 'mp4',
+            'format_id': 'hi',
+            'display_id': 'shoop-shoop-diddy-wop-cumma-cumma-wang-dang-1982',
+            'title': 'Monte Video - "Shoop Shoop, Diddy Wop"',
+            'description': 'Monte Video - "Shoop Shoop, Diddy Wop"',
+            'alt_title': 'Shoop Shoop Diddy Wop Cumma Cumma Wang Dang | Music Video',
+            'thumbnail': r're:https://www\.nzonscreen\.com/content/images/.+\.jpg',
+            'duration': 158,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.nzonscreen.com/title/shes-a-mod-1964?collection=best-of-the-60s',
+        'info_dict': {
+            'id': '3dbe709ff03c36f1',
+            'ext': 'mp4',
+            'format_id': 'hi',
+            'display_id': 'shes-a-mod-1964',
+            'title': 'Ray Columbus - \'She\'s A Mod\'',
+            'description': 'Ray Columbus - \'She\'s A Mod\'',
+            'alt_title': 'She\'s a Mod | Music Video',
+            'thumbnail': r're:https://www\.nzonscreen\.com/content/images/.+\.jpg',
+            'duration': 130,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.nzonscreen.com/title/puha-and-pakeha-1968/overview',
+        'info_dict': {
+            'id': 'f86342544385ad8a',
+            'ext': 'mp4',
+            'format_id': 'hi',
+            'display_id': 'puha-and-pakeha-1968',
+            'title': 'Looking At New Zealand - Puha and Pakeha',
+            'alt_title': 'Looking at New Zealand - \'Pūhā and Pākehā\' | Television',
+            'description': 'An excerpt from this television programme.',
+            'duration': 212,
+            'thumbnail': r're:https://www\.nzonscreen\.com/content/images/.+\.jpg',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _extract_formats(self, playlist):
+        for quality, (id_, url) in enumerate(traverse_obj(
+                playlist, ('h264', {'lo': 'lo_res', 'hi': 'hi_res'}), expected_type=url_or_none).items()):
+            yield {
+                'url': url,
+                'format_id': id_,
+                'ext': 'mp4',
+                'quality': quality,
+                'height': int_or_none(playlist.get('height')) if id_ == 'hi' else None,
+                'width': int_or_none(playlist.get('width')) if id_ == 'hi' else None,
+                'filesize_approx': float_or_none(traverse_obj(playlist, ('h264', f'{id_}_res_mb')), invscale=1024**2),
+            }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        playlist = self._parse_json(self._html_search_regex(
+            r'data-video-config=\'([^\']+)\'', webpage, 'media data'), video_id)
+
+        return {
+            'id': playlist['uuid'],
+            'display_id': video_id,
+            'title': strip_or_none(playlist.get('label')),
+            'description': strip_or_none(playlist.get('description')),
+            'alt_title': strip_or_none(remove_end(
+                self._html_extract_title(webpage, default=None) or self._og_search_title(webpage),
+                ' | NZ On Screen')),
+            'thumbnail': traverse_obj(playlist, ('thumbnail', 'path')),
+            'duration': float_or_none(playlist.get('duration')),
+            'formats': list(self._extract_formats(playlist)),
+            'http_headers': {
+                'Referer': 'https://www.nzonscreen.com/',
+                'Origin': 'https://www.nzonscreen.com/',
+            }
+        }

From 8b37c58f8b5494504acdb5ebe3f8bbd26230f725 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Mon, 13 Feb 2023 20:57:24 -0600
Subject: [PATCH 063/871] [extractor/nfl] Add `NFLPlus` extractors (#6222)

Closes #6165
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   2 +
 yt_dlp/extractor/nfl.py         | 148 +++++++++++++++++++++++++++++++-
 2 files changed, 147 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 8b3875130..061a25a4e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1197,6 +1197,8 @@
 from .nfl import (
     NFLIE,
     NFLArticleIE,
+    NFLPlusEpisodeIE,
+    NFLPlusReplayIE,
 )
 from .nhk import (
     NhkVodIE,
diff --git a/yt_dlp/extractor/nfl.py b/yt_dlp/extractor/nfl.py
index 29c53d5a5..cc3f4495c 100644
--- a/yt_dlp/extractor/nfl.py
+++ b/yt_dlp/extractor/nfl.py
@@ -1,10 +1,18 @@
+import base64
+import json
 import re
+import time
+import uuid
 
+from .anvato import AnvatoIE
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     clean_html,
     determine_ext,
     get_element_by_class,
+    traverse_obj,
+    urlencode_postdata,
 )
 
 
@@ -54,15 +62,14 @@ class NFLBaseIE(InfoExtractor):
                         )/
                     '''
     _VIDEO_CONFIG_REGEX = r'<script[^>]+id="[^"]*video-config-[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}[^"]*"[^>]*>\s*({.+});?\s*</script>'
+    _ANVATO_PREFIX = 'anvato:GXvEgwyJeWem8KCYXfeoHWknwP48Mboj:'
 
     def _parse_video_config(self, video_config, display_id):
         video_config = self._parse_json(video_config, display_id)
         item = video_config['playlist'][0]
         mcp_id = item.get('mcpID')
         if mcp_id:
-            info = self.url_result(
-                'anvato:GXvEgwyJeWem8KCYXfeoHWknwP48Mboj:' + mcp_id,
-                'Anvato', mcp_id)
+            info = self.url_result(f'{self._ANVATO_PREFIX}{mcp_id}', AnvatoIE, mcp_id)
         else:
             media_id = item.get('id') or item['entityId']
             title = item.get('title')
@@ -157,3 +164,138 @@ def _real_extract(self, url):
             'nfl-c-article__title', webpage)) or self._html_search_meta(
             ['og:title', 'twitter:title'], webpage)
         return self.playlist_result(entries, display_id, title)
+
+
+class NFLPlusReplayIE(NFLBaseIE):
+    IE_NAME = 'nfl.com:plus:replay'
+    _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/games/[\w-]+/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.nfl.com/plus/games/giants-at-vikings-2022-post-1/1572108',
+        'info_dict': {
+            'id': '1572108',
+            'ext': 'mp4',
+            'title': 'New York Giants at Minnesota Vikings',
+            'description': 'New York Giants play the Minnesota Vikings at U.S. Bank Stadium on January 15, 2023',
+            'uploader': 'NFL',
+            'upload_date': '20230116',
+            'timestamp': 1673864520,
+            'duration': 7157,
+            'categories': ['Game Highlights'],
+            'tags': ['Minnesota Vikings', 'New York Giants', 'Minnesota Vikings vs. New York Giants'],
+            'thumbnail': r're:^https?://.*\.jpg',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id)
+
+
+class NFLPlusEpisodeIE(NFLBaseIE):
+    IE_NAME = 'nfl.com:plus:episode'
+    _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/episodes/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'note': 'premium content',
+        'url': 'https://www.nfl.com/plus/episodes/kurt-s-qb-insider-conference-championships',
+        'info_dict': {
+            'id': '1576832',
+            'ext': 'mp4',
+            'title': 'Kurt\'s QB Insider: Conference Championships',
+            'description': 'md5:944f7fab56f7a37430bf8473f5473857',
+            'uploader': 'NFL',
+            'upload_date': '20230127',
+            'timestamp': 1674782760,
+            'duration': 730,
+            'categories': ['Analysis'],
+            'tags': ['Cincinnati Bengals at Kansas City Chiefs (2022-POST-3)'],
+            'thumbnail': r're:^https?://.*\.jpg',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    _CLIENT_DATA = {
+        'clientKey': '4cFUW6DmwJpzT9L7LrG3qRAcABG5s04g',
+        'clientSecret': 'CZuvCL49d9OwfGsR',
+        'deviceId': str(uuid.uuid4()),
+        'deviceInfo': base64.b64encode(json.dumps({
+            'model': 'desktop',
+            'version': 'Chrome',
+            'osName': 'Windows',
+            'osVersion': '10.0',
+        }, separators=(',', ':')).encode()).decode(),
+        'networkType': 'other',
+        'nflClaimGroupsToAdd': [],
+        'nflClaimGroupsToRemove': [],
+    }
+    _ACCOUNT_INFO = {}
+    _API_KEY = None
+
+    _TOKEN = None
+    _TOKEN_EXPIRY = 0
+
+    def _get_account_info(self, url, video_id):
+        cookies = self._get_cookies('https://www.nfl.com/')
+        login_token = traverse_obj(cookies, (
+            (f'glt_{self._API_KEY}', f'gig_loginToken_{self._API_KEY}',
+             lambda k, _: k.startswith('glt_') or k.startswith('gig_loginToken_')),
+            {lambda x: x.value}), get_all=False)
+        if not login_token:
+            self.raise_login_required()
+
+        account = self._download_json(
+            'https://auth-id.nfl.com/accounts.getAccountInfo', video_id,
+            note='Downloading account info', data=urlencode_postdata({
+                'include': 'profile,data',
+                'lang': 'en',
+                'APIKey': self._API_KEY,
+                'sdk': 'js_latest',
+                'login_token': login_token,
+                'authMode': 'cookie',
+                'pageURL': url,
+                'sdkBuild': traverse_obj(cookies, (
+                    'gig_canary_ver', {lambda x: x.value.partition('-')[0]}), default='13642'),
+                'format': 'json',
+            }), headers={'Content-Type': 'application/x-www-form-urlencoded'})
+
+        self._ACCOUNT_INFO = traverse_obj(account, {
+            'signatureTimestamp': 'signatureTimestamp',
+            'uid': 'UID',
+            'uidSignature': 'UIDSignature',
+        })
+
+        if len(self._ACCOUNT_INFO) != 3:
+            raise ExtractorError('Failed to retrieve account info with provided cookies', expected=True)
+
+    def _get_auth_token(self, url, video_id):
+        if not self._ACCOUNT_INFO:
+            self._get_account_info(url, video_id)
+
+        token = self._download_json(
+            'https://api.nfl.com/identity/v3/token%s' % (
+                '/refresh' if self._ACCOUNT_INFO.get('refreshToken') else ''),
+            video_id, headers={'Content-Type': 'application/json'}, note='Downloading access token',
+            data=json.dumps({**self._CLIENT_DATA, **self._ACCOUNT_INFO}, separators=(',', ':')).encode())
+
+        self._TOKEN = token['accessToken']
+        self._TOKEN_EXPIRY = token['expiresIn']
+        self._ACCOUNT_INFO['refreshToken'] = token['refreshToken']
+
+    def _real_extract(self, url):
+        slug = self._match_id(url)
+
+        if not self._API_KEY:
+            webpage = self._download_webpage(url, slug, fatal=False) or ''
+            self._API_KEY = self._search_regex(
+                r'window\.gigyaApiKey=["\'](\w+)["\'];', webpage, 'API key',
+                default='3_Qa8TkWpIB8ESCBT8tY2TukbVKgO5F6BJVc7N1oComdwFzI7H2L9NOWdm11i_BY9f')
+
+        if not self._TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
+            self._get_auth_token(url, slug)
+
+        video_id = self._download_json(
+            f'https://api.nfl.com/content/v1/videos/episodes/{slug}', slug, headers={
+                'Authorization': f'Bearer {self._TOKEN}',
+            })['mcpPlaybackId']
+
+        return self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id)

From 9ebac35577e61c3d25fafc959655fa3ab04ca7ef Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 16 Feb 2023 17:06:48 +0530
Subject: [PATCH 064/871] Bugfix for 39f32f1715c0dffb7626dda7307db6388bb7abaa

when `--ignore-no-formats-error`
---
 yt_dlp/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index d214a6449..33b4fb3ca 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2557,7 +2557,7 @@ def sanitize_numeric_fields(info):
         formats = self._get_formats(info_dict)
 
         # Backward compatibility with InfoExtractor._sort_formats
-        field_preference = formats[0].pop('__sort_fields', None)
+        field_preference = (formats or [{}])[0].pop('__sort_fields', None)
         if field_preference:
             info_dict['_format_sort_fields'] = field_preference
 

From 149eb0bbf34fa8fdf8d1e2aa28e17479d099e26b Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Thu, 16 Feb 2023 08:51:45 -0600
Subject: [PATCH 065/871] [extractor/youtube] Fix `uploader_id` extraction

Closes #6247
Authored by: bashonly
---
 yt_dlp/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 95ca52b3a..4dde4bbaa 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4120,7 +4120,7 @@ def is_bad_format(fmt):
             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
             'description': video_description,
             'uploader': get_first(video_details, 'author'),
-            'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
+            'uploader_id': self._search_regex(r'/(?:channel/|user/|@)([^/?&#]+)', owner_profile_url, 'uploader id', default=None),
             'uploader_url': owner_profile_url,
             'channel_id': channel_id,
             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),

From c9d14bd22ab31e2a41f9f8061843668a06db583b Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Thu, 16 Feb 2023 15:54:11 +0100
Subject: [PATCH 066/871] [extractor/crunchyroll] Fix incorrect premium-only
 error

Closes #6234

Authored by: Grub4K
---
 yt_dlp/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py
index 7d356d673..1abffcd74 100644
--- a/yt_dlp/extractor/crunchyroll.py
+++ b/yt_dlp/extractor/crunchyroll.py
@@ -160,7 +160,7 @@ def _real_extract(self, url):
         episode_response = self._download_json(
             f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
             note='Retrieving episode metadata', query=params)
-        if episode_response.get('is_premium_only') and not episode_response.get('playback'):
+        if episode_response.get('is_premium_only') and not bucket.endswith('crunchyroll'):
             if self.is_logged_in:
                 raise ExtractorError('This video is for premium members only', expected=True)
             else:

From 376aa24b1541e2bfb23337c0ae9bafa5bb3787f1 Mon Sep 17 00:00:00 2001
From: Siddhartha Sahu <github12@sdht.in>
Date: Thu, 16 Feb 2023 14:55:01 -0500
Subject: [PATCH 067/871] Improve default subtitle language selection (#6240)

Authored by: sdht0
---
 yt_dlp/YoutubeDL.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 33b4fb3ca..4e5c40b58 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2810,10 +2810,14 @@ def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
                     self.params.get('subtitleslangs'), {'all': all_sub_langs}, use_regex=True)
             except re.error as e:
                 raise ValueError(f'Wrong regex for subtitlelangs: {e.pattern}')
-        elif normal_sub_langs:
-            requested_langs = ['en'] if 'en' in normal_sub_langs else normal_sub_langs[:1]
         else:
-            requested_langs = ['en'] if 'en' in all_sub_langs else all_sub_langs[:1]
+            requested_langs = LazyList(itertools.chain(
+                ['en'] if 'en' in normal_sub_langs else [],
+                filter(lambda f: f.startswith('en'), normal_sub_langs),
+                ['en'] if 'en' in all_sub_langs else [],
+                filter(lambda f: f.startswith('en'), all_sub_langs),
+                normal_sub_langs, all_sub_langs,
+            ))[:1]
         if requested_langs:
             self.to_screen(f'[info] {video_id}: Downloading subtitles: {", ".join(requested_langs)}')
 

From 72671a212d7c939329cb5d34335fa089dd3acbd3 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Fri, 17 Feb 2023 11:57:52 +0900
Subject: [PATCH 068/871] [extractor/viu] Add `ViuOTTIndonesiaIE` extractor
 (#6099)

Authored by: HobbyistDev
Closes #1757
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/viu.py         | 146 ++++++++++++++++++++++++++++++++
 2 files changed, 147 insertions(+)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 061a25a4e..081696855 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2181,6 +2181,7 @@
     ViuIE,
     ViuPlaylistIE,
     ViuOTTIE,
+    ViuOTTIndonesiaIE,
 )
 from .vk import (
     VKIE,
diff --git a/yt_dlp/extractor/viu.py b/yt_dlp/extractor/viu.py
index dd4cad7ba..6f9af9f64 100644
--- a/yt_dlp/extractor/viu.py
+++ b/yt_dlp/extractor/viu.py
@@ -9,9 +9,12 @@
 from ..utils import (
     ExtractorError,
     int_or_none,
+    remove_end,
     strip_or_none,
+    traverse_obj,
     try_get,
     smuggle_url,
+    unified_timestamp,
     unsmuggle_url,
     url_or_none,
 )
@@ -394,3 +397,146 @@ def download_playback():
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class ViuOTTIndonesiaBaseIE(InfoExtractor):
+    _BASE_QUERY = {
+        'ver': 1.0,
+        'fmt': 'json',
+        'aver': 5.0,
+        'appver': 2.0,
+        'appid': 'viu_desktop',
+        'platform': 'desktop',
+    }
+
+    _DEVICE_ID = str(uuid.uuid4())
+    _SESSION_ID = str(uuid.uuid4())
+    _TOKEN = None
+
+    _HEADERS = {
+        'x-session-id': _SESSION_ID,
+        'x-client': 'browser'
+    }
+
+    _AGE_RATINGS_MAPPER = {
+        'ADULTS': 18,
+        'teens': 13
+    }
+
+    def _real_initialize(self):
+        ViuOTTIndonesiaBaseIE._TOKEN = self._download_json(
+            'https://um.viuapi.io/user/identity', None,
+            headers={'Content-type': 'application/json', **self._HEADERS},
+            query={**self._BASE_QUERY, 'iid': self._DEVICE_ID},
+            data=json.dumps({'deviceId': self._DEVICE_ID}).encode(),
+            note='Downloading token information')['token']
+
+
+class ViuOTTIndonesiaIE(ViuOTTIndonesiaBaseIE):
+    _VALID_URL = r'https?://www\.viu\.com/ott/\w+/\w+/all/video-[\w-]+-(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.viu.com/ott/id/id/all/video-japanese-drama-tv_shows-detective_conan_episode_793-1165863142?containerId=playlist-26271226',
+        'info_dict': {
+            'id': '1165863142',
+            'ext': 'mp4',
+            'episode_number': 793,
+            'episode': 'Episode 793',
+            'title': 'Detective Conan - Episode 793',
+            'duration': 1476,
+            'description': 'md5:b79d55345bc1e0217ece22616267c9a5',
+            'thumbnail': 'https://vuclipi-a.akamaihd.net/p/cloudinary/h_171,w_304,dpr_1.5,f_auto,c_thumb,q_auto:low/1165863189/d-1',
+            'upload_date': '20210101',
+            'timestamp': 1609459200,
+        }
+    }, {
+        'url': 'https://www.viu.com/ott/id/id/all/video-korean-reality-tv_shows-entertainment_weekly_episode_1622-1118617054',
+        'info_dict': {
+            'id': '1118617054',
+            'ext': 'mp4',
+            'episode_number': 1622,
+            'episode': 'Episode 1622',
+            'description': 'md5:6d68ca450004020113e9bf27ad99f0f8',
+            'title': 'Entertainment Weekly - Episode 1622',
+            'duration': 4729,
+            'thumbnail': 'https://vuclipi-a.akamaihd.net/p/cloudinary/h_171,w_304,dpr_1.5,f_auto,c_thumb,q_auto:low/1120187848/d-1',
+            'timestamp': 1420070400,
+            'upload_date': '20150101',
+            'cast': ['Shin Hyun-joon', 'Lee Da-Hee']
+        }
+    }, {
+        # age-limit test
+        'url': 'https://www.viu.com/ott/id/id/all/video-japanese-trailer-tv_shows-trailer_jujutsu_kaisen_ver_01-1166044219?containerId=playlist-26273140',
+        'info_dict': {
+            'id': '1166044219',
+            'ext': 'mp4',
+            'upload_date': '20200101',
+            'timestamp': 1577836800,
+            'title': 'Trailer \'Jujutsu Kaisen\' Ver.01',
+            'duration': 92,
+            'thumbnail': 'https://vuclipi-a.akamaihd.net/p/cloudinary/h_171,w_304,dpr_1.5,f_auto,c_thumb,q_auto:low/1166044240/d-1',
+            'description': 'Trailer \'Jujutsu Kaisen\' Ver.01',
+            'cast': ['Junya Enoki', ' Yûichi Nakamura', ' Yuma Uchida', 'Asami Seto'],
+            'age_limit': 13,
+        }
+    }, {
+        # json ld metadata type equal to Movie instead of TVEpisodes
+        'url': 'https://www.viu.com/ott/id/id/all/video-japanese-animation-movies-demon_slayer_kimetsu_no_yaiba_the_movie_mugen_train-1165892707?containerId=1675060691786',
+        'info_dict': {
+            'id': '1165892707',
+            'ext': 'mp4',
+            'timestamp': 1577836800,
+            'upload_date': '20200101',
+            'title': 'Demon Slayer - Kimetsu no Yaiba - The Movie: Mugen Train',
+            'age_limit': 13,
+            'cast': 'count:9',
+            'thumbnail': 'https://vuclipi-a.akamaihd.net/p/cloudinary/h_171,w_304,dpr_1.5,f_auto,c_thumb,q_auto:low/1165895279/d-1',
+            'description': 'md5:1ce9c35a3aeab384085533f746c87469',
+            'duration': 7021,
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_data = self._download_json(
+            f'https://um.viuapi.io/drm/v1/content/{display_id}', display_id, data=b'',
+            headers={'Authorization': ViuOTTIndonesiaBaseIE._TOKEN, **self._HEADERS, 'ccode': 'ID'})
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_data['playUrl'], display_id)
+
+        initial_state = self._search_json(
+            r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state',
+            display_id)['content']['clipDetails']
+        for key, url in initial_state.items():
+            lang, ext = self._search_regex(
+                r'^subtitle_(?P<lang>[\w-]+)_(?P<ext>\w+)$', key, 'subtitle metadata',
+                default=(None, None), group=('lang', 'ext'))
+            if lang and ext:
+                subtitles.setdefault(lang, []).append({
+                    'ext': ext,
+                    'url': url,
+                })
+
+                if ext == 'vtt':
+                    subtitles[lang].append({
+                        'ext': 'srt',
+                        'url': f'{remove_end(initial_state[key], "vtt")}srt',
+                    })
+
+        episode = traverse_obj(list(filter(
+            lambda x: x.get('@type') in ('TVEpisode', 'Movie'), self._yield_json_ld(webpage, display_id))), 0) or {}
+        return {
+            'id': display_id,
+            'title': (traverse_obj(initial_state, 'title', 'display_title')
+                      or episode.get('name')),
+            'description': initial_state.get('description') or episode.get('description'),
+            'duration': initial_state.get('duration'),
+            'thumbnail': traverse_obj(episode, ('image', 'url')),
+            'timestamp': unified_timestamp(episode.get('dateCreated')),
+            'formats': formats,
+            'subtitles': subtitles,
+            'episode_number': (traverse_obj(initial_state, 'episode_no', 'episodeno', expected_type=int_or_none)
+                               or int_or_none(episode.get('episodeNumber'))),
+            'cast': traverse_obj(episode, ('actor', ..., 'name'), default=None),
+            'age_limit': self._AGE_RATINGS_MAPPER.get(initial_state.get('internal_age_rating'))
+        }

From 10fd9e6ee833c88edf6c633f864f42843a708d32 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Fri, 17 Feb 2023 12:00:07 +0900
Subject: [PATCH 069/871] [extractor/odkmedia] Add `OnDemandChinaEpisodeIE`
 (#6116)

Authored by: HobbyistDev, pukkandan
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/odkmedia.py    | 105 ++++++++++++++++++++++++++++++++
 2 files changed, 106 insertions(+)
 create mode 100644 yt_dlp/extractor/odkmedia.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 081696855..86fa117b7 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1292,6 +1292,7 @@
 from .nzonscreen import NZOnScreenIE
 from .nzz import NZZIE
 from .odatv import OdaTVIE
+from .odkmedia import OnDemandChinaEpisodeIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oftv import (
     OfTVIE,
diff --git a/yt_dlp/extractor/odkmedia.py b/yt_dlp/extractor/odkmedia.py
new file mode 100644
index 000000000..2960860d6
--- /dev/null
+++ b/yt_dlp/extractor/odkmedia.py
@@ -0,0 +1,105 @@
+import json
+import urllib.error
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    GeoRestrictedError,
+    float_or_none,
+    traverse_obj,
+    try_call
+)
+
+
+class OnDemandChinaEpisodeIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.ondemandchina\.com/\w+/watch/(?P<series>[\w-]+)/(?P<id>ep-(?P<ep>\d+))'
+    _TESTS = [{
+        'url': 'https://www.ondemandchina.com/en/watch/together-against-covid-19/ep-1',
+        'info_dict': {
+            'id': '264394',
+            'ext': 'mp4',
+            'duration': 3256.88,
+            'title': 'EP 1 The Calling',
+            'alt_title': '第1集 令出如山',
+            'thumbnail': 'https://d2y2efdi5wgkcl.cloudfront.net/fit-in/256x256/media-io/2020/9/11/image.d9816e81.jpg',
+            'description': '疫情严峻，党政军民学、东西南北中协同应考',
+            'tags': ['Social Humanities', 'Documentary', 'Medical', 'Social'],
+        }
+    }]
+
+    _QUERY = '''
+        query Episode($programSlug: String!, $episodeNumber: Int!) {
+            episode(
+                programSlug: $programSlug
+                episodeNumber: $episodeNumber
+                kind: "series"
+                part: null
+            ) {
+                id
+                title
+                titleEn
+                titleKo
+                titleZhHans
+                titleZhHant
+                synopsis
+                synopsisEn
+                synopsisKo
+                synopsisZhHans
+                synopsisZhHant
+                videoDuration
+                images {
+                    thumbnail
+                }
+            }
+        }'''
+
+    def _real_extract(self, url):
+        program_slug, display_id, ep_number = self._match_valid_url(url).group('series', 'id', 'ep')
+        webpage = self._download_webpage(url, display_id)
+
+        video_info = self._download_json(
+            'https://odc-graphql.odkmedia.io/graphql', display_id,
+            headers={'Content-type': 'application/json'},
+            data=json.dumps({
+                'operationName': 'Episode',
+                'query': self._QUERY,
+                'variables': {
+                    'programSlug': program_slug,
+                    'episodeNumber': int(ep_number),
+                },
+            }).encode())['data']['episode']
+
+        try:
+            source_json = self._download_json(
+                f'https://odkmedia.io/odc/api/v2/playback/{video_info["id"]}/', display_id,
+                headers={'Authorization': '', 'service-name': 'odc'})
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError):
+                error_data = self._parse_json(e.cause.read(), display_id)['detail']
+                raise GeoRestrictedError(error_data)
+
+        formats, subtitles = [], {}
+        for source in traverse_obj(source_json, ('sources', ...)):
+            if source.get('type') == 'hls':
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(source.get('url'), display_id)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            else:
+                self.report_warning(f'Unsupported format {source.get("type")}', display_id)
+
+        return {
+            'id': str(video_info['id']),
+            'duration': float_or_none(video_info.get('videoDuration'), 1000),
+            'thumbnail': (traverse_obj(video_info, ('images', 'thumbnail'))
+                          or self._html_search_meta(['og:image', 'twitter:image'], webpage)),
+            'title': (traverse_obj(video_info, 'title', 'titleEn')
+                      or self._html_search_meta(['og:title', 'twitter:title'], webpage)
+                      or self._html_extract_title(webpage)),
+            'alt_title': traverse_obj(video_info, 'titleKo', 'titleZhHans', 'titleZhHant'),
+            'description': (traverse_obj(
+                video_info, 'synopsisEn', 'synopsisKo', 'synopsisZhHans', 'synopsisZhHant', 'synopisis')
+                or self._html_search_meta(['og:description', 'twitter:description', 'description'], webpage)),
+            'formats': formats,
+            'subtitles': subtitles,
+            'tags': try_call(lambda: self._html_search_meta('keywords', webpage).split(', '))
+        }

From a9189510baadf0dccd2d4d363bc6f3a441128bb0 Mon Sep 17 00:00:00 2001
From: OIRNOIR <70721372+OIRNOIR@users.noreply.github.com>
Date: Thu, 16 Feb 2023 19:06:16 -0800
Subject: [PATCH 070/871] [extractor/nitter] Update instance list (#6236)

Authored by: OIRNOIR
---
 yt_dlp/extractor/nitter.py | 124 ++++++++++++++++++++++++++++---------
 1 file changed, 95 insertions(+), 29 deletions(-)

diff --git a/yt_dlp/extractor/nitter.py b/yt_dlp/extractor/nitter.py
index 251bf444f..5d1ca1f5d 100644
--- a/yt_dlp/extractor/nitter.py
+++ b/yt_dlp/extractor/nitter.py
@@ -39,59 +39,99 @@ class NitterIE(InfoExtractor):
     )
 
     HTTP_INSTANCES = (
-        'nitter.42l.fr',
-        'nitter.pussthecat.org',
-        'nitter.nixnet.services',
+        'nitter.lacontrevoie.fr',
         'nitter.fdn.fr',
         'nitter.1d4.us',
         'nitter.kavin.rocks',
         'nitter.unixfox.eu',
         'nitter.domain.glass',
-        'nitter.eu',
         'nitter.namazso.eu',
-        'nitter.actionsack.com',
         'birdsite.xanny.family',
-        'nitter.hu',
-        'twitr.gq',
         'nitter.moomoo.me',
-        'nittereu.moomoo.me',
-        'bird.from.tf',
+        'bird.trom.tf',
         'nitter.it',
         'twitter.censors.us',
-        'twitter.grimneko.de',
-        'nitter.alefvanoon.xyz',
-        'n.hyperborea.cloud',
-        'nitter.ca',
+        'nitter.grimneko.de',
         'twitter.076.ne.jp',
-        'twitter.mstdn.social',
         'nitter.fly.dev',
         'notabird.site',
         'nitter.weiler.rocks',
-        'nitter.silkky.cloud',
         'nitter.sethforprivacy.com',
-        'nttr.stream',
         'nitter.cutelab.space',
         'nitter.nl',
         'nitter.mint.lgbt',
         'nitter.bus-hit.me',
-        'fuckthesacklers.network',
-        'nitter.govt.land',
-        'nitter.datatunnel.xyz',
         'nitter.esmailelbob.xyz',
         'tw.artemislena.eu',
-        'de.nttr.stream',
         'nitter.winscloud.net',
         'nitter.tiekoetter.com',
         'nitter.spaceint.fr',
-        'twtr.bch.bar',
-        'nitter.exonip.de',
-        'nitter.mastodon.pro',
-        'nitter.notraxx.ch',
-
-
-        # not in the list anymore
-        'nitter.skrep.in',
-        'nitter.snopyta.org',
+        'nitter.privacy.com.de',
+        'nitter.poast.org',
+        'nitter.bird.froth.zone',
+        'nitter.dcs0.hu',
+        'twitter.dr460nf1r3.org',
+        'nitter.garudalinux.org',
+        'twitter.femboy.hu',
+        'nitter.cz',
+        'nitter.privacydev.net',
+        'nitter.evil.site',
+        'tweet.lambda.dance',
+        'nitter.kylrth.com',
+        'nitter.foss.wtf',
+        'nitter.priv.pw',
+        'nitter.tokhmi.xyz',
+        'nitter.catalyst.sx',
+        'unofficialbird.com',
+        'nitter.projectsegfau.lt',
+        'nitter.eu.projectsegfau.lt',
+        'singapore.unofficialbird.com',
+        'canada.unofficialbird.com',
+        'india.unofficialbird.com',
+        'nederland.unofficialbird.com',
+        'uk.unofficialbird.com',
+        'n.l5.ca',
+        'nitter.slipfox.xyz',
+        'nitter.soopy.moe',
+        'nitter.qwik.space',
+        'read.whatever.social',
+        'nitter.rawbit.ninja',
+        'nt.vern.cc',
+        'ntr.odyssey346.dev',
+        'nitter.ir',
+        'nitter.privacytools.io',
+        'nitter.sneed.network',
+        'n.sneed.network',
+        'nitter.manasiwibi.com',
+        'nitter.smnz.de',
+        'nitter.twei.space',
+        'nitter.inpt.fr',
+        'nitter.d420.de',
+        'nitter.caioalonso.com',
+        'nitter.at',
+        'nitter.drivet.xyz',
+        'nitter.pw',
+        'nitter.nicfab.eu',
+        'bird.habedieeh.re',
+        'nitter.hostux.net',
+        'nitter.adminforge.de',
+        'nitter.platypush.tech',
+        'nitter.mask.sh',
+        'nitter.pufe.org',
+        'nitter.us.projectsegfau.lt',
+        'nitter.arcticfoxes.net',
+        't.com.sb',
+        'nitter.kling.gg',
+        'nitter.ktachibana.party',
+        'nitter.riverside.rocks',
+        'nitter.girlboss.ceo',
+        'nitter.lunar.icu',
+        'twitter.moe.ngo',
+        'nitter.freedit.eu',
+        'ntr.frail.duckdns.org',
+        'nitter.librenode.org',
+        'n.opnxng.com',
+        'nitter.plus.st',
     )
 
     DEAD_INSTANCES = (
@@ -117,6 +157,32 @@ class NitterIE(InfoExtractor):
         'nitter.weaponizedhumiliation.com',
         'nitter.vxempire.xyz',
         'tweet.lambda.dance',
+        'nitter.ca',
+        'nitter.42l.fr',
+        'nitter.pussthecat.org',
+        'nitter.nixnet.services',
+        'nitter.eu',
+        'nitter.actionsack.com',
+        'nitter.hu',
+        'twitr.gq',
+        'nittereu.moomoo.me',
+        'bird.from.tf',
+        'twitter.grimneko.de',
+        'nitter.alefvanoon.xyz',
+        'n.hyperborea.cloud',
+        'twitter.mstdn.social',
+        'nitter.silkky.cloud',
+        'nttr.stream',
+        'fuckthesacklers.network',
+        'nitter.govt.land',
+        'nitter.datatunnel.xyz',
+        'de.nttr.stream',
+        'twtr.bch.bar',
+        'nitter.exonip.de',
+        'nitter.mastodon.pro',
+        'nitter.notraxx.ch',
+        'nitter.skrep.in',
+        'nitter.snopyta.org',
     )
 
     INSTANCES = NON_HTTP_INSTANCES + HTTP_INSTANCES + DEAD_INSTANCES

From 65e5c021e7c5f23ecbc6a982b72a02ac6cd6900d Mon Sep 17 00:00:00 2001
From: Felix Yan <felixonmars@archlinux.org>
Date: Fri, 17 Feb 2023 05:08:45 +0200
Subject: [PATCH 071/871] [utils] Don't use Content-length with encoding
 (#6176)

Authored by: felixonmars
Closes #3772, #6178
---
 yt_dlp/downloader/http.py | 7 ++++++-
 yt_dlp/utils.py           | 3 ---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index 95c870ee8..fa72d5722 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -211,7 +211,12 @@ def close_stream():
                 ctx.stream = None
 
         def download():
-            data_len = ctx.data.info().get('Content-length', None)
+            data_len = ctx.data.info().get('Content-length')
+
+            if ctx.data.info().get('Content-encoding'):
+                # Content-encoding is present, Content-length is not reliable anymore as we are
+                # doing auto decompression. (See: https://github.com/yt-dlp/yt-dlp/pull/6176)
+                data_len = None
 
             # Range HTTP header may be ignored/unsupported by a webserver
             # (e.g. extractor/scivee.py, extractor/bambuser.py).
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 7cf151e3a..2d9e61c5b 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -1438,19 +1438,16 @@ def http_response(self, req, resp):
                     raise original_ioerror
             resp = urllib.request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
-            del resp.headers['Content-encoding']
         # deflate
         if resp.headers.get('Content-encoding', '') == 'deflate':
             gz = io.BytesIO(self.deflate(resp.read()))
             resp = urllib.request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
-            del resp.headers['Content-encoding']
         # brotli
         if resp.headers.get('Content-encoding', '') == 'br':
             resp = urllib.request.addinfourl(
                 io.BytesIO(self.brotli(resp.read())), old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
-            del resp.headers['Content-encoding']
         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
         # https://github.com/ytdl-org/youtube-dl/issues/6457).
         if 300 <= resp.code < 400:

From da880559a6ecbbf374cc9f3378e696b55b9599af Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Thu, 16 Feb 2023 23:14:33 -0400
Subject: [PATCH 072/871] [extractor/ebay] Add extractor (#6170)

Closes #6134
Authored by: JChris246
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/ebay.py        | 36 +++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 yt_dlp/extractor/ebay.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 86fa117b7..a9ab66fc7 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -507,6 +507,7 @@
 )
 from .eagleplatform import EaglePlatformIE, ClipYouEmbedIE
 from .ebaumsworld import EbaumsWorldIE
+from .ebay import EbayIE
 from .echomsk import EchoMskIE
 from .egghead import (
     EggheadCourseIE,
diff --git a/yt_dlp/extractor/ebay.py b/yt_dlp/extractor/ebay.py
new file mode 100644
index 000000000..d0eb9fc51
--- /dev/null
+++ b/yt_dlp/extractor/ebay.py
@@ -0,0 +1,36 @@
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class EbayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ebay\.com/itm/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.ebay.com/itm/194509326719',
+        'info_dict': {
+            'id': '194509326719',
+            'ext': 'mp4',
+            'title': 'WiFi internal antenna adhesive for wifi 2.4GHz wifi 5 wifi 6 wifi 6E full bands',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        video_json = self._search_json(r'"video":', webpage, 'video json', video_id)
+
+        formats = []
+        for key, url in video_json['playlistMap'].items():
+            if key == 'HLS':
+                formats.extend(self._extract_m3u8_formats(url, video_id, fatal=False))
+            elif key == 'DASH':
+                formats.extend(self._extract_mpd_formats(url, video_id, fatal=False))
+            else:
+                self.report_warning(f'Unsupported format {key}', video_id)
+
+        return {
+            'id': video_id,
+            'title': remove_end(self._html_extract_title(webpage), ' | eBay'),
+            'formats': formats
+        }

From e4a8b1769e19755acba6d8f212208359905a3159 Mon Sep 17 00:00:00 2001
From: qbnu <93988953+qbnu@users.noreply.github.com>
Date: Fri, 17 Feb 2023 03:18:07 +0000
Subject: [PATCH 073/871] [extractor/vocaroo] Add extractor (#6117)

Authored by: qbnu, SuperSonicHub1
Closes #6152
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/vocaroo.py     | 65 +++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+)
 create mode 100644 yt_dlp/extractor/vocaroo.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index a9ab66fc7..0a36e98de 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2190,6 +2190,7 @@
     VKUserVideosIE,
     VKWallPostIE,
 )
+from .vocaroo import VocarooIE
 from .vodlocker import VodlockerIE
 from .vodpl import VODPlIE
 from .vodplatform import VODPlatformIE
diff --git a/yt_dlp/extractor/vocaroo.py b/yt_dlp/extractor/vocaroo.py
new file mode 100644
index 000000000..704e25c22
--- /dev/null
+++ b/yt_dlp/extractor/vocaroo.py
@@ -0,0 +1,65 @@
+from .common import InfoExtractor
+from ..utils import (
+    HEADRequest,
+    float_or_none,
+)
+
+
+class VocarooIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?:vocaroo\.com|voca\.ro)/(?:embed/)?(?P<id>\w+)'
+    _EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:www\.)?vocaroo\.com/embed/.+?)\1']
+    _TESTS = [
+        {
+            'url': 'https://vocaroo.com/1de8yA3LNe77',
+            'md5': 'c557841d5e50261777a6585648adf439',
+            'info_dict': {
+                'id': '1de8yA3LNe77',
+                'ext': 'mp3',
+                'title': 'Vocaroo video #1de8yA3LNe77',
+                'timestamp': 1675059800.370,
+                'upload_date': '20230130',
+            },
+        },
+        {
+            'url': 'https://vocaroo.com/embed/12WqtjLnpj6g?autoplay=0',
+            'only_matching': True,
+        },
+        {
+            'url': 'https://voca.ro/12D52rgpzkB0',
+            'only_matching': True,
+        },
+    ]
+
+    _WEBPAGE_TESTS = [
+        {
+            'url': 'https://qbnu.github.io/cool.html',
+            'md5': 'f322e529275dd8a47994919eeac404a5',
+            'info_dict': {
+                'id': '19cgWmKO6AmC',
+                'ext': 'mp3',
+                'title': 'Vocaroo video #19cgWmKO6AmC',
+                'timestamp': 1675093841.408,
+                'upload_date': '20230130',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        if len(audio_id) == 10 or (len(audio_id) == 12 and audio_id[0] == '1'):
+            media_subdomain = 'media1'
+        else:
+            media_subdomain = 'media'
+
+        url = f'https://{media_subdomain}.vocaroo.com/mp3/{audio_id}'
+        http_headers = {'Referer': 'https://vocaroo.com/'}
+        resp = self._request_webpage(HEADRequest(url), audio_id, headers=http_headers)
+        return {
+            'id': audio_id,
+            'title': '',
+            'url': url,
+            'ext': 'mp3',
+            'timestamp': float_or_none(resp.getheader('x-bz-upload-timestamp'), scale=1000),
+            'vcodec': 'none',
+            'http_headers': http_headers,
+        }

From 361630015535026712bdb67f804a15b65ff9ee7e Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Fri, 17 Feb 2023 12:19:24 +0900
Subject: [PATCH 074/871] [extractor/yappy] Add extractor (#6111)

Authored by: HobbyistDev
Closes #3522
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/yappy.py       | 99 +++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 yt_dlp/extractor/yappy.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 0a36e98de..4aab6ea78 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2329,6 +2329,7 @@
     ZenYandexChannelIE,
 )
 from .yapfiles import YapFilesIE
+from .yappy import YappyIE
 from .yesjapan import YesJapanIE
 from .yinyuetai import YinYueTaiIE
 from .yle_areena import YleAreenaIE
diff --git a/yt_dlp/extractor/yappy.py b/yt_dlp/extractor/yappy.py
new file mode 100644
index 000000000..f168bdbf9
--- /dev/null
+++ b/yt_dlp/extractor/yappy.py
@@ -0,0 +1,99 @@
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    traverse_obj,
+    unified_timestamp,
+    url_or_none
+)
+
+
+class YappyIE(InfoExtractor):
+    _VALID_URL = r'https?://yappy\.media/video/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://yappy.media/video/47fea6d8586f48d1a0cf96a7342aabd2',
+        'info_dict': {
+            'id': '47fea6d8586f48d1a0cf96a7342aabd2',
+            'ext': 'mp4',
+            'title': 'Куда нажимать? Как снимать? Смотри видос и погнали!🤘🏻',
+            'timestamp': 1661893200,
+            'description': 'Куда нажимать? Как снимать? Смотри видос и погнали!🤘🏻',
+            'thumbnail': 'https://cdn-st.ritm.media/static/pic/thumbnails/0c7c4d73388f47848acaf540d2e2bb8c-thumbnail.jpg',
+            'upload_date': '20220830',
+            'view_count': int,
+            'like_count': int,
+            'uploader_id': '59a0c8c485e5410b9c43474bf4c6a373',
+            'categories': ['Образование и наука', 'Лайфхак', 'Технологии', 'Арт/искусство'],
+            'repost_count': int,
+            'uploader': 'YAPPY',
+        }
+    }, {
+        'url': 'https://yappy.media/video/3862451954ad4bd58ae2ccefddb0bd33',
+        'info_dict': {
+            'id': '3862451954ad4bd58ae2ccefddb0bd33',
+            'ext': 'mp4',
+            'title': 'Опиши свой характер 3 словами🙃\n#психология #дружба #отношения',
+            'timestamp': 1674726985,
+            'like_count': int,
+            'description': 'Опиши свой характер 3 словами🙃\n#психология #дружба #отношения',
+            'uploader_id': '6793ee3581974a3586fc01e157de6c99',
+            'view_count': int,
+            'repost_count': int,
+            'uploader': 'LENA SHTURMAN',
+            'upload_date': '20230126',
+            'thumbnail': 'https://cdn-st.ritm.media/static/pic/user_thumbnails/6e76bb4bbad640b6/9ec84c115b2b1967/1674716171.jpg',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        json_ld = self._search_json_ld(webpage, video_id)
+        nextjs_data = self._search_nextjs_data(webpage, video_id)
+
+        media_data = (
+            traverse_obj(
+                nextjs_data, ('props', 'pageProps', ('data', 'OpenGraphParameters')), get_all=False)
+            or self._download_json(f'https://yappy.media/api/video/{video_id}', video_id))
+
+        media_url = traverse_obj(media_data, ('link', {url_or_none})) or ''
+        has_watermark = media_url.endswith('-wm.mp4')
+
+        formats = [{
+            'url': media_url,
+            'ext': 'mp4',
+            'format_note': 'Watermarked' if has_watermark else None,
+            'preference': -10 if has_watermark else None
+        }] if media_url else []
+
+        if has_watermark:
+            formats.append({
+                'url': media_url.replace('-wm.mp4', '.mp4'),
+                'ext': 'mp4'
+            })
+
+        audio_link = traverse_obj(media_data, ('audio', 'link'))
+        if audio_link:
+            formats.append({
+                'url': audio_link,
+                'ext': 'mp3',
+                'acodec': 'mp3',
+                'vcodec': 'none'
+            })
+
+        return {
+            'id': video_id,
+            'title': (json_ld.get('description') or self._html_search_meta(['og:title'], webpage)
+                      or self._html_extract_title(webpage)),
+            'formats': formats,
+            'thumbnail': (media_data.get('thumbnail')
+                          or self._html_search_meta(['og:image', 'og:image:secure_url'], webpage)),
+            'description': (media_data.get('description') or json_ld.get('description')
+                            or self._html_search_meta(['description', 'og:description'], webpage)),
+            'timestamp': unified_timestamp(media_data.get('publishedAt') or json_ld.get('timestamp')),
+            'view_count': int_or_none(media_data.get('viewsCount') or json_ld.get('view_count')),
+            'like_count': int_or_none(media_data.get('likesCount')),
+            'uploader': traverse_obj(media_data, ('creator', 'firstName')),
+            'uploader_id': traverse_obj(media_data, ('creator', ('uuid', 'nickname')), get_all=False),
+            'categories': traverse_obj(media_data, ('categories', ..., 'name')) or None,
+            'repost_count': int_or_none(media_data.get('sharingCount'))
+        }

From b25d6cb96337d479bdcb41768356da414c3aa835 Mon Sep 17 00:00:00 2001
From: Alex Ionescu <aaionescu@protonmail.com>
Date: Fri, 17 Feb 2023 04:29:32 +0100
Subject: [PATCH 075/871] [utils] Fix race condition in `make_dir` (#6089)

Authored by: aionescu
---
 yt_dlp/cache.py | 6 +-----
 yt_dlp/utils.py | 4 ++--
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/cache.py b/yt_dlp/cache.py
index 7be91eae5..f8344fe77 100644
--- a/yt_dlp/cache.py
+++ b/yt_dlp/cache.py
@@ -39,11 +39,7 @@ def store(self, section, key, data, dtype='json'):
 
         fn = self._get_cache_fn(section, key, dtype)
         try:
-            try:
-                os.makedirs(os.path.dirname(fn))
-            except OSError as ose:
-                if ose.errno != errno.EEXIST:
-                    raise
+            os.makedirs(os.path.dirname(fn), exist_ok=True)
             self._ydl.write_debug(f'Saving {section}.{key} to cache')
             write_json_file({'yt-dlp_version': __version__, 'data': data}, fn)
         except Exception:
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 2d9e61c5b..736468aef 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5370,8 +5370,8 @@ def random_uuidv4():
 def make_dir(path, to_screen=None):
     try:
         dn = os.path.dirname(path)
-        if dn and not os.path.exists(dn):
-            os.makedirs(dn)
+        if dn:
+            os.makedirs(dn, exist_ok=True)
         return True
     except OSError as err:
         if callable(to_screen) is not None:

From a4ad59ff2ded208bf33f6fe07299a3449eadccdc Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Fri, 17 Feb 2023 12:59:04 +0900
Subject: [PATCH 076/871] [extractor/anchorfm] Add episode extractor (#6092)

Authored by: HobbyistDev, bashonly
Closes #6081
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/anchorfm.py    | 98 +++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 yt_dlp/extractor/anchorfm.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 4aab6ea78..6bba25506 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -102,6 +102,7 @@
     AmericasTestKitchenIE,
     AmericasTestKitchenSeasonIE,
 )
+from .anchorfm import AnchorFMEpisodeIE
 from .angel import AngelIE
 from .anvato import AnvatoIE
 from .aol import AolIE
diff --git a/yt_dlp/extractor/anchorfm.py b/yt_dlp/extractor/anchorfm.py
new file mode 100644
index 000000000..52f2ad057
--- /dev/null
+++ b/yt_dlp/extractor/anchorfm.py
@@ -0,0 +1,98 @@
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    float_or_none,
+    int_or_none,
+    str_or_none,
+    traverse_obj,
+    unified_timestamp
+)
+
+
+class AnchorFMEpisodeIE(InfoExtractor):
+    _VALID_URL = r'https?://anchor\.fm/(?P<channel_name>\w+)/(?:embed/)?episodes/[\w-]+-(?P<episode_id>\w+)'
+    _EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL})']
+    _TESTS = [{
+        'url': 'https://anchor.fm/lovelyti/episodes/Chrisean-Rock-takes-to-twitter-to-announce-shes-pregnant--Blueface-denies-he-is-the-father-e1tpt3d',
+        'info_dict': {
+            'id': 'e1tpt3d',
+            'ext': 'mp3',
+            'title': ' Chrisean Rock takes to twitter to announce she\'s pregnant, Blueface denies he is the father!',
+            'description': 'md5:207d167de3e28ceb4ddc1ebf5a30044c',
+            'thumbnail': 'https://s3-us-west-2.amazonaws.com/anchor-generated-image-bank/production/podcast_uploaded_nologo/1034827/1034827-1658438968460-5f3bfdf3601e8.jpg',
+            'duration': 624.718,
+            'uploader': 'Lovelyti ',
+            'uploader_id': '991541',
+            'channel': 'lovelyti',
+            'modified_date': '20230121',
+            'modified_timestamp': 1674285178,
+            'release_date': '20230121',
+            'release_timestamp': 1674285179,
+            'episode_id': 'e1tpt3d',
+        }
+    }, {
+        # embed url
+        'url': 'https://anchor.fm/apakatatempo/embed/episodes/S2E75-Perang-Bintang-di-Balik-Kasus-Ferdy-Sambo-dan-Ismail-Bolong-e1shjqd',
+        'info_dict': {
+            'id': 'e1shjqd',
+            'ext': 'mp3',
+            'title': 'S2E75 Perang Bintang di Balik Kasus Ferdy Sambo dan Ismail Bolong',
+            'description': 'md5:9e95ad9293bf00178bf8d33e9cb92c41',
+            'duration': 1042.008,
+            'thumbnail': 'https://s3-us-west-2.amazonaws.com/anchor-generated-image-bank/production/podcast_uploaded_episode400/2627805/2627805-1671590688729-4db3882ac9e4b.jpg',
+            'release_date': '20221221',
+            'release_timestamp': 1671595916,
+            'modified_date': '20221221',
+            'modified_timestamp': 1671590834,
+            'channel': 'apakatatempo',
+            'uploader': 'Podcast Tempo',
+            'uploader_id': '2585461',
+            'season': 'Season 2',
+            'season_number': 2,
+            'episode_id': 'e1shjqd',
+        }
+    }]
+
+    _WEBPAGE_TESTS = [{
+        'url': 'https://podcast.tempo.co/podcast/192/perang-bintang-di-balik-kasus-ferdy-sambo-dan-ismail-bolong',
+        'info_dict': {
+            'id': 'e1shjqd',
+            'ext': 'mp3',
+            'release_date': '20221221',
+            'duration': 1042.008,
+            'season': 'Season 2',
+            'modified_timestamp': 1671590834,
+            'uploader_id': '2585461',
+            'modified_date': '20221221',
+            'description': 'md5:9e95ad9293bf00178bf8d33e9cb92c41',
+            'season_number': 2,
+            'title': 'S2E75 Perang Bintang di Balik Kasus Ferdy Sambo dan Ismail Bolong',
+            'release_timestamp': 1671595916,
+            'episode_id': 'e1shjqd',
+            'thumbnail': 'https://s3-us-west-2.amazonaws.com/anchor-generated-image-bank/production/podcast_uploaded_episode400/2627805/2627805-1671590688729-4db3882ac9e4b.jpg',
+            'uploader': 'Podcast Tempo',
+            'channel': 'apakatatempo',
+        }
+    }]
+
+    def _real_extract(self, url):
+        channel_name, episode_id = self._match_valid_url(url).group('channel_name', 'episode_id')
+        api_data = self._download_json(f'https://anchor.fm/api/v3/episodes/{episode_id}', episode_id)
+
+        return {
+            'id': episode_id,
+            'title': traverse_obj(api_data, ('episode', 'title')),
+            'url': traverse_obj(api_data, ('episode', 'episodeEnclosureUrl'), ('episodeAudios', 0, 'url')),
+            'ext': 'mp3',
+            'vcodec': 'none',
+            'thumbnail': traverse_obj(api_data, ('episode', 'episodeImage')),
+            'description': clean_html(traverse_obj(api_data, ('episode', ('description', 'descriptionPreview')), get_all=False)),
+            'duration': float_or_none(traverse_obj(api_data, ('episode', 'duration')), 1000),
+            'modified_timestamp': unified_timestamp(traverse_obj(api_data, ('episode', 'modified'))),
+            'release_timestamp': int_or_none(traverse_obj(api_data, ('episode', 'publishOnUnixTimestamp'))),
+            'episode_id': episode_id,
+            'uploader': traverse_obj(api_data, ('creator', 'name')),
+            'uploader_id': str_or_none(traverse_obj(api_data, ('creator', 'userId'))),
+            'season_number': int_or_none(traverse_obj(api_data, ('episode', 'podcastSeasonNumber'))),
+            'channel': channel_name or traverse_obj(api_data, ('creator', 'vanitySlug')),
+        }

From 31c279a2a2c2ef402a9e6dad9992b310d16439a6 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Fri, 17 Feb 2023 13:03:04 +0900
Subject: [PATCH 077/871] [extractor/hypergryph] Add extractor (#6094)

Authored by: HobbyistDev, bashonly
Closes #6052
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/hypergryph.py  | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 yt_dlp/extractor/hypergryph.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 6bba25506..70cb82277 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -747,6 +747,7 @@
     HungamaAlbumPlaylistIE,
 )
 from .hypem import HypemIE
+from .hypergryph import MonsterSirenHypergryphMusicIE
 from .hytale import HytaleIE
 from .icareus import IcareusIE
 from .ichinanalive import (
diff --git a/yt_dlp/extractor/hypergryph.py b/yt_dlp/extractor/hypergryph.py
new file mode 100644
index 000000000..9ca6caebc
--- /dev/null
+++ b/yt_dlp/extractor/hypergryph.py
@@ -0,0 +1,32 @@
+from .common import InfoExtractor
+from ..utils import js_to_json, traverse_obj
+
+
+class MonsterSirenHypergryphMusicIE(InfoExtractor):
+    _VALID_URL = r'https?://monster-siren\.hypergryph\.com/music/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://monster-siren.hypergryph.com/music/514562',
+        'info_dict': {
+            'id': '514562',
+            'ext': 'wav',
+            'artist': ['塞壬唱片-MSR'],
+            'album': 'Flame Shadow',
+            'title': 'Flame Shadow',
+        }
+    }]
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        webpage = self._download_webpage(url, audio_id)
+        json_data = self._search_json(
+            r'window\.g_initialProps\s*=', webpage, 'data', audio_id, transform_source=js_to_json)
+
+        return {
+            'id': audio_id,
+            'title': traverse_obj(json_data, ('player', 'songDetail', 'name')),
+            'url': traverse_obj(json_data, ('player', 'songDetail', 'sourceUrl')),
+            'ext': 'wav',
+            'vcodec': 'none',
+            'artist': traverse_obj(json_data, ('player', 'songDetail', 'artists')),
+            'album': traverse_obj(json_data, ('musicPlay', 'albumDetail', 'name'))
+        }

From 5e1a54f63e393c218a40949012ff0de0ce63cb15 Mon Sep 17 00:00:00 2001
From: Friedrich Rehren <FriedrichRehren@users.noreply.github.com>
Date: Fri, 17 Feb 2023 08:44:26 +0100
Subject: [PATCH 078/871] [extractor/SportDeutschland] Fix extractor (#6041)

Authored by: FriedrichRehren
Closes #3005
---
 yt_dlp/extractor/sportdeutschland.py | 157 +++++++++++++++------------
 1 file changed, 86 insertions(+), 71 deletions(-)

diff --git a/yt_dlp/extractor/sportdeutschland.py b/yt_dlp/extractor/sportdeutschland.py
index 75074b310..6fc3ce9eb 100644
--- a/yt_dlp/extractor/sportdeutschland.py
+++ b/yt_dlp/extractor/sportdeutschland.py
@@ -1,95 +1,110 @@
 from .common import InfoExtractor
+
 from ..utils import (
-    clean_html,
-    float_or_none,
-    int_or_none,
-    parse_iso8601,
-    parse_qs,
-    strip_or_none,
-    try_get,
+    format_field,
+    traverse_obj,
+    unified_timestamp,
+    strip_or_none
 )
 
 
 class SportDeutschlandIE(InfoExtractor):
     _VALID_URL = r'https?://sportdeutschland\.tv/(?P<id>(?:[^/]+/)?[^?#/&]+)'
     _TESTS = [{
-        'url': 'https://sportdeutschland.tv/badminton/re-live-deutsche-meisterschaften-2020-halbfinals?playlistId=0',
+        'url': 'https://sportdeutschland.tv/blauweissbuchholztanzsport/buchholzer-formationswochenende-2023-samstag-1-bundesliga-landesliga',
         'info_dict': {
-            'id': '5318cac0275701382770543d7edaf0a0',
+            'id': '983758e9-5829-454d-a3cf-eb27bccc3c94',
             'ext': 'mp4',
-            'title': 'Re-live: Deutsche Meisterschaften 2020 - Halbfinals - Teil 1',
-            'duration': 16106.36,
-        },
-        'params': {
-            'noplaylist': True,
-            # m3u8 download
-            'skip_download': True,
-        },
+            'title': 'Buchholzer Formationswochenende 2023 - Samstag - 1. Bundesliga / Landesliga',
+            'description': 'md5:a288c794a5ee69e200d8f12982f81a87',
+            'live_status': 'was_live',
+            'channel': 'Blau-Weiss Buchholz Tanzsport',
+            'channel_url': 'https://sportdeutschland.tv/blauweissbuchholztanzsport',
+            'channel_id': '93ec33c9-48be-43b6-b404-e016b64fdfa3',
+            'display_id': '9839a5c7-0dbb-48a8-ab63-3b408adc7b54',
+            'duration': 32447,
+            'upload_date': '20230114',
+            'timestamp': 1673730018.0,
+        }
     }, {
-        'url': 'https://sportdeutschland.tv/badminton/re-live-deutsche-meisterschaften-2020-halbfinals?playlistId=0',
+        'url': 'https://sportdeutschland.tv/deutscherbadmintonverband/bwf-tour-1-runde-feld-1-yonex-gainward-german-open-2022-0',
         'info_dict': {
-            'id': 'c6e2fdd01f63013854c47054d2ab776f',
-            'title': 'Re-live: Deutsche Meisterschaften 2020 - Halbfinals',
-            'description': 'md5:5263ff4c31c04bb780c9f91130b48530',
-            'duration': 31397,
-        },
-        'playlist_count': 2,
-    }, {
-        'url': 'https://sportdeutschland.tv/freeride-world-tour-2021-fieberbrunn-oesterreich',
-        'only_matching': True,
+            'id': '95b97d9a-04f6-4880-9039-182985c33943',
+            'ext': 'mp4',
+            'title': 'BWF Tour: 1. Runde Feld 1 - YONEX GAINWARD German Open 2022',
+            'description': 'md5:2afb5996ceb9ac0b2ac81f563d3a883e',
+            'live_status': 'was_live',
+            'channel': 'Deutscher Badminton Verband',
+            'channel_url': 'https://sportdeutschland.tv/deutscherbadmintonverband',
+            'channel_id': '93ca5866-2551-49fc-8424-6db35af58920',
+            'display_id': '95c80c52-6b9a-4ae9-9197-984145adfced',
+            'duration': 41097,
+            'upload_date': '20220309',
+            'timestamp': 1646860727.0,
+        }
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        data = self._download_json(
-            'https://backend.sportdeutschland.tv/api/permalinks/' + display_id,
+        meta = self._download_json(
+            'https://api.sportdeutschland.tv/api/stateless/frontend/assets/' + display_id,
             display_id, query={'access_token': 'true'})
-        asset = data['asset']
-        title = (asset.get('title') or asset['label']).strip()
-        asset_id = asset.get('id') or asset.get('uuid')
+
+        asset_id = traverse_obj(meta, 'id', 'uuid')
+
         info = {
             'id': asset_id,
-            'title': title,
-            'description': clean_html(asset.get('body') or asset.get('description')) or asset.get('teaser'),
-            'duration': int_or_none(asset.get('seconds')),
+            'channel_url': format_field(meta, ('profile', 'slug'), 'https://sportdeutschland.tv/%s'),
+            **traverse_obj(meta, {
+                'title': (('title', 'name'), {strip_or_none}),
+                'description': 'description',
+                'channel': ('profile', 'name'),
+                'channel_id': ('profile', 'id'),
+                'is_live': 'currently_live',
+                'was_live': 'was_live'
+            }, get_all=False)
         }
-        videos = asset.get('videos') or []
-        if len(videos) > 1:
-            playlist_id = parse_qs(url).get('playlistId', [None])[0]
-            if not self._yes_playlist(playlist_id, asset_id):
-                videos = [videos[int(playlist_id)]]
 
-            def entries():
-                for i, video in enumerate(videos, 1):
-                    video_id = video.get('uuid')
-                    video_url = video.get('url')
-                    if not (video_id and video_url):
-                        continue
-                    formats = self._extract_m3u8_formats(
-                        video_url.replace('.smil', '.m3u8'), video_id, 'mp4', fatal=False)
-                    if not formats and not self.get_param('ignore_no_formats'):
-                        continue
-                    yield {
-                        'id': video_id,
-                        'formats': formats,
-                        'title': title + ' - ' + (video.get('label') or 'Teil %d' % i),
-                        'duration': float_or_none(video.get('duration')),
-                    }
+        videos = meta.get('videos') or []
+
+        if len(videos) > 1:
             info.update({
                 '_type': 'multi_video',
-                'entries': entries(),
-            })
-        else:
-            formats = self._extract_m3u8_formats(
-                videos[0]['url'].replace('.smil', '.m3u8'), asset_id, 'mp4')
-            section_title = strip_or_none(try_get(data, lambda x: x['section']['title']))
-            info.update({
-                'formats': formats,
-                'display_id': asset.get('permalink'),
-                'thumbnail': try_get(asset, lambda x: x['images'][0]),
-                'categories': [section_title] if section_title else None,
-                'view_count': int_or_none(asset.get('views')),
-                'is_live': asset.get('is_live') is True,
-                'timestamp': parse_iso8601(asset.get('date') or asset.get('published_at')),
-            })
+                'entries': self.processVideoOrStream(asset_id, video)
+            } for video in enumerate(videos) if video.get('formats'))
+
+        elif len(videos) == 1:
+            info.update(
+                self.processVideoOrStream(asset_id, videos[0])
+            )
+
+        livestream = meta.get('livestream')
+
+        if livestream is not None:
+            info.update(
+                self.processVideoOrStream(asset_id, livestream)
+            )
+
         return info
+
+    def process_video_or_stream(self, asset_id, video):
+        video_id = video['id']
+        video_src = video['src']
+        video_type = video['type']
+
+        token = self._download_json(
+            f'https://api.sportdeutschland.tv/api/frontend/asset-token/{asset_id}',
+            video_id, query={'type': video_type, 'playback_id': video_src})['token']
+        formats = self._extract_m3u8_formats(f'https://stream.mux.com/{video_src}.m3u8?token={token}', video_id)
+
+        video_data = {
+            'display_id': video_id,
+            'formats': formats,
+        }
+        if video_type == 'mux_vod':
+            video_data.update({
+                'duration': video.get('duration'),
+                'timestamp': unified_timestamp(video.get('created_at'))
+            })
+
+        return video_data

From f737fb16d8234408c85bc189ccc926fea000515b Mon Sep 17 00:00:00 2001
From: Chris Caruso <carusochrisr@gmail.com>
Date: Fri, 17 Feb 2023 00:06:15 -0800
Subject: [PATCH 079/871] [ExtractAudio] Handle outtmpl without ext (#6005)

Authored by: carusocr
Closes #5968
---
 yt_dlp/__init__.py             | 4 ----
 yt_dlp/postprocessor/ffmpeg.py | 3 +--
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 255b31735..fb44303a2 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -318,10 +318,6 @@ def validate_outtmpl(tmpl, msg):
     if outtmpl_default == '':
         opts.skip_download = None
         del opts.outtmpl['default']
-    if outtmpl_default and not os.path.splitext(outtmpl_default)[1] and opts.extractaudio:
-        raise ValueError(
-            'Cannot download a video and extract audio into the same file! '
-            f'Use "{outtmpl_default}.%(ext)s" instead of "{outtmpl_default}" as the output template')
 
     def parse_chapters(name, value):
         chapters, ranges = [], []
diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py
index 5acd75376..123a95a3a 100644
--- a/yt_dlp/postprocessor/ffmpeg.py
+++ b/yt_dlp/postprocessor/ffmpeg.py
@@ -508,8 +508,7 @@ def run(self, information):
         if acodec != 'copy':
             more_opts = self._quality_args(acodec)
 
-        # not os.path.splitext, since the latter does not work on unicode in all setups
-        temp_path = new_path = f'{path.rpartition(".")[0]}.{extension}'
+        temp_path = new_path = replace_extension(path, extension, information['ext'])
 
         if new_path == path:
             if acodec == 'copy':

From c61cf091a54d3aa3c611722035ccde5ecfe981bb Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Fri, 17 Feb 2023 02:14:45 -0600
Subject: [PATCH 080/871] [extractor/youtube] `uploader_id` includes `@` with
 handle

Authored by: bashonly
---
 yt_dlp/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 4dde4bbaa..d891d92a3 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4120,7 +4120,7 @@ def is_bad_format(fmt):
             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
             'description': video_description,
             'uploader': get_first(video_details, 'author'),
-            'uploader_id': self._search_regex(r'/(?:channel/|user/|@)([^/?&#]+)', owner_profile_url, 'uploader id', default=None),
+            'uploader_id': self._search_regex(r'/(?:channel/|user/|(?=@))([^/?&#]+)', owner_profile_url, 'uploader id', default=None),
             'uploader_url': owner_profile_url,
             'channel_id': channel_id,
             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),

From d50ea3ce5abc3b0defc0e5d1e22b22ce9b01b07b Mon Sep 17 00:00:00 2001
From: Henrik Heimbuerger <henrik@heimbuerger.de>
Date: Fri, 17 Feb 2023 09:32:55 +0100
Subject: [PATCH 081/871] [extractor/nebula] Remove broken cookie support
 (#5979)

Authored by: hheimbuerger
Closes #4002
---
 yt_dlp/extractor/nebula.py | 119 +++++++++++--------------------------
 1 file changed, 35 insertions(+), 84 deletions(-)

diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py
index 861fcb164..81e2f56e6 100644
--- a/yt_dlp/extractor/nebula.py
+++ b/yt_dlp/extractor/nebula.py
@@ -1,11 +1,9 @@
 import itertools
 import json
-import time
 import urllib.error
-import urllib.parse
 
 from .common import InfoExtractor
-from ..utils import ExtractorError, parse_iso8601, try_get
+from ..utils import ExtractorError, parse_iso8601
 
 _BASE_URL_RE = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)'
 
@@ -15,11 +13,10 @@ class NebulaBaseIE(InfoExtractor):
 
     _nebula_api_token = None
     _nebula_bearer_token = None
-    _zype_access_token = None
 
     def _perform_nebula_auth(self, username, password):
         if not username or not password:
-            self.raise_login_required()
+            self.raise_login_required(method='password')
 
         data = json.dumps({'email': username, 'password': password}).encode('utf8')
         response = self._download_json(
@@ -33,38 +30,10 @@ def _perform_nebula_auth(self, username, password):
             note='Logging in to Nebula with supplied credentials',
             errnote='Authentication failed or rejected')
         if not response or not response.get('key'):
-            self.raise_login_required()
-
-        # save nebula token as cookie
-        self._set_cookie(
-            'nebula.app', 'nebula-auth',
-            urllib.parse.quote(
-                json.dumps({
-                    "apiToken": response["key"],
-                    "isLoggingIn": False,
-                    "isLoggingOut": False,
-                }, separators=(",", ":"))),
-            expire_time=int(time.time()) + 86400 * 365,
-        )
+            self.raise_login_required(method='password')
 
         return response['key']
 
-    def _retrieve_nebula_api_token(self, username=None, password=None):
-        """
-        Check cookie jar for valid token. Try to authenticate using credentials if no valid token
-        can be found in the cookie jar.
-        """
-        nebula_cookies = self._get_cookies('https://nebula.app')
-        nebula_cookie = nebula_cookies.get('nebula-auth')
-        if nebula_cookie:
-            self.to_screen('Authenticating to Nebula with token from cookie jar')
-            nebula_cookie_value = urllib.parse.unquote(nebula_cookie.value)
-            nebula_api_token = self._parse_json(nebula_cookie_value, None).get('apiToken')
-            if nebula_api_token:
-                return nebula_api_token
-
-        return self._perform_nebula_auth(username, password)
-
     def _call_nebula_api(self, url, video_id=None, method='GET', auth_type='api', note=''):
         assert method in ('GET', 'POST',)
         assert auth_type in ('api', 'bearer',)
@@ -95,35 +64,24 @@ def _fetch_nebula_bearer_token(self):
                                          note='Authorizing to Nebula')
         return response['token']
 
-    def _fetch_zype_access_token(self):
-        """
-        Get a Zype access token, which is required to access video streams -- in our case: to
-        generate video URLs.
-        """
-        user_object = self._call_nebula_api('https://api.watchnebula.com/api/v1/auth/user/', note='Retrieving Zype access token')
-
-        access_token = try_get(user_object, lambda x: x['zype_auth_info']['access_token'], str)
-        if not access_token:
-            if try_get(user_object, lambda x: x['is_subscribed'], bool):
-                # TODO: Reimplement the same Zype token polling the Nebula frontend implements
-                # see https://github.com/ytdl-org/youtube-dl/pull/24805#issuecomment-749231532
-                raise ExtractorError(
-                    'Unable to extract Zype access token from Nebula API authentication endpoint. '
-                    'Open an arbitrary video in a browser with this account to generate a token',
-                    expected=True)
-            raise ExtractorError('Unable to extract Zype access token from Nebula API authentication endpoint')
-        return access_token
+    def _fetch_video_formats(self, slug):
+        stream_info = self._call_nebula_api(f'https://content.watchnebula.com/video/{slug}/stream/',
+                                            video_id=slug,
+                                            auth_type='bearer',
+                                            note='Fetching video stream info')
+        manifest_url = stream_info['manifest']
+        return self._extract_m3u8_formats_and_subtitles(manifest_url, slug)
 
     def _build_video_info(self, episode):
-        zype_id = episode['zype_id']
-        zype_video_url = f'https://player.zype.com/embed/{zype_id}.html?access_token={self._zype_access_token}'
+        fmts, subs = self._fetch_video_formats(episode['slug'])
         channel_slug = episode['channel_slug']
+        channel_title = episode['channel_title']
         return {
             'id': episode['zype_id'],
             'display_id': episode['slug'],
-            '_type': 'url_transparent',
-            'ie_key': 'Zype',
-            'url': zype_video_url,
+            'formats': fmts,
+            'subtitles': subs,
+            'webpage_url': f'https://nebula.tv/{episode["slug"]}',
             'title': episode['title'],
             'description': episode['description'],
             'timestamp': parse_iso8601(episode['published_at']),
@@ -133,27 +91,26 @@ def _build_video_info(self, episode):
                 'height': key,
             } for key, tn in episode['assets']['thumbnail'].items()],
             'duration': episode['duration'],
-            'channel': episode['channel_title'],
+            'channel': channel_title,
             'channel_id': channel_slug,
-            'channel_url': f'https://nebula.app/{channel_slug}',
-            'uploader': episode['channel_title'],
+            'channel_url': f'https://nebula.tv/{channel_slug}',
+            'uploader': channel_title,
             'uploader_id': channel_slug,
-            'uploader_url': f'https://nebula.app/{channel_slug}',
-            'series': episode['channel_title'],
-            'creator': episode['channel_title'],
+            'uploader_url': f'https://nebula.tv/{channel_slug}',
+            'series': channel_title,
+            'creator': channel_title,
         }
 
     def _perform_login(self, username=None, password=None):
-        self._nebula_api_token = self._retrieve_nebula_api_token(username, password)
+        self._nebula_api_token = self._perform_nebula_auth(username, password)
         self._nebula_bearer_token = self._fetch_nebula_bearer_token()
-        self._zype_access_token = self._fetch_zype_access_token()
 
 
 class NebulaIE(NebulaBaseIE):
     _VALID_URL = rf'{_BASE_URL_RE}/videos/(?P<id>[-\w]+)'
     _TESTS = [
         {
-            'url': 'https://nebula.app/videos/that-time-disney-remade-beauty-and-the-beast',
+            'url': 'https://nebula.tv/videos/that-time-disney-remade-beauty-and-the-beast',
             'md5': '14944cfee8c7beeea106320c47560efc',
             'info_dict': {
                 'id': '5c271b40b13fd613090034fd',
@@ -167,19 +124,17 @@ class NebulaIE(NebulaBaseIE):
                 'uploader': 'Lindsay Ellis',
                 'uploader_id': 'lindsayellis',
                 'timestamp': 1533009600,
-                'uploader_url': 'https://nebula.app/lindsayellis',
+                'uploader_url': 'https://nebula.tv/lindsayellis',
                 'series': 'Lindsay Ellis',
-                'average_rating': int,
                 'display_id': 'that-time-disney-remade-beauty-and-the-beast',
-                'channel_url': 'https://nebula.app/lindsayellis',
+                'channel_url': 'https://nebula.tv/lindsayellis',
                 'creator': 'Lindsay Ellis',
                 'duration': 2212,
-                'view_count': int,
                 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*',
             },
         },
         {
-            'url': 'https://nebula.app/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
+            'url': 'https://nebula.tv/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
             'md5': 'd05739cf6c38c09322422f696b569c23',
             'info_dict': {
                 'id': '5e7e78171aaf320001fbd6be',
@@ -192,19 +147,17 @@ class NebulaIE(NebulaBaseIE):
                 'channel_id': 'realengineering',
                 'uploader': 'Real Engineering',
                 'uploader_id': 'realengineering',
-                'view_count': int,
                 'series': 'Real Engineering',
-                'average_rating': int,
                 'display_id': 'the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
                 'creator': 'Real Engineering',
                 'duration': 841,
-                'channel_url': 'https://nebula.app/realengineering',
-                'uploader_url': 'https://nebula.app/realengineering',
+                'channel_url': 'https://nebula.tv/realengineering',
+                'uploader_url': 'https://nebula.tv/realengineering',
                 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*',
             },
         },
         {
-            'url': 'https://nebula.app/videos/money-episode-1-the-draw',
+            'url': 'https://nebula.tv/videos/money-episode-1-the-draw',
             'md5': 'ebe28a7ad822b9ee172387d860487868',
             'info_dict': {
                 'id': '5e779ebdd157bc0001d1c75a',
@@ -217,14 +170,12 @@ class NebulaIE(NebulaBaseIE):
                 'channel_id': 'tom-scott-presents-money',
                 'uploader': 'Tom Scott Presents: Money',
                 'uploader_id': 'tom-scott-presents-money',
-                'uploader_url': 'https://nebula.app/tom-scott-presents-money',
+                'uploader_url': 'https://nebula.tv/tom-scott-presents-money',
                 'duration': 825,
-                'channel_url': 'https://nebula.app/tom-scott-presents-money',
-                'view_count': int,
+                'channel_url': 'https://nebula.tv/tom-scott-presents-money',
                 'series': 'Tom Scott Presents: Money',
                 'display_id': 'money-episode-1-the-draw',
                 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*',
-                'average_rating': int,
                 'creator': 'Tom Scott Presents: Money',
             },
         },
@@ -251,7 +202,7 @@ class NebulaSubscriptionsIE(NebulaBaseIE):
     _VALID_URL = rf'{_BASE_URL_RE}/myshows'
     _TESTS = [
         {
-            'url': 'https://nebula.app/myshows',
+            'url': 'https://nebula.tv/myshows',
             'playlist_mincount': 1,
             'info_dict': {
                 'id': 'myshows',
@@ -279,7 +230,7 @@ class NebulaChannelIE(NebulaBaseIE):
     _VALID_URL = rf'{_BASE_URL_RE}/(?!myshows|videos/)(?P<id>[-\w]+)'
     _TESTS = [
         {
-            'url': 'https://nebula.app/tom-scott-presents-money',
+            'url': 'https://nebula.tv/tom-scott-presents-money',
             'info_dict': {
                 'id': 'tom-scott-presents-money',
                 'title': 'Tom Scott Presents: Money',
@@ -287,13 +238,13 @@ class NebulaChannelIE(NebulaBaseIE):
             },
             'playlist_count': 5,
         }, {
-            'url': 'https://nebula.app/lindsayellis',
+            'url': 'https://nebula.tv/lindsayellis',
             'info_dict': {
                 'id': 'lindsayellis',
                 'title': 'Lindsay Ellis',
                 'description': 'Enjoy these hottest of takes on Disney, Transformers, and Musicals.',
             },
-            'playlist_mincount': 100,
+            'playlist_mincount': 2,
         },
     ]
 

From 9acca71237f42a4775008e51fe26e42f0a39c552 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Fri, 17 Feb 2023 18:05:46 +0900
Subject: [PATCH 082/871] [extractor/boxcast] Add extractor (#5983)

Authored by: HobbyistDev
Closes #5769
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/boxcast.py     | 102 ++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 yt_dlp/extractor/boxcast.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 70cb82277..797e5668a 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -245,6 +245,7 @@
 from .bongacams import BongaCamsIE
 from .bostonglobe import BostonGlobeIE
 from .box import BoxIE
+from .boxcast import BoxCastVideoIE
 from .booyah import BooyahClipsIE
 from .bpb import BpbIE
 from .br import (
diff --git a/yt_dlp/extractor/boxcast.py b/yt_dlp/extractor/boxcast.py
new file mode 100644
index 000000000..51f9eb787
--- /dev/null
+++ b/yt_dlp/extractor/boxcast.py
@@ -0,0 +1,102 @@
+from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    traverse_obj,
+    unified_timestamp
+)
+
+
+class BoxCastVideoIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+        https?://boxcast\.tv/(?:
+            view-embed/|
+            channel/\w+\?(?:[^#]+&)?b=|
+            video-portal/(?:\w+/){2}
+        )(?P<id>[\w-]+)'''
+    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>https?://boxcast\.tv/view-embed/[\w-]+)']
+    _TESTS = [{
+        'url': 'https://boxcast.tv/view-embed/in-the-midst-of-darkness-light-prevails-an-interdisciplinary-symposium-ozmq5eclj50ujl4bmpwx',
+        'info_dict': {
+            'id': 'da1eqqgkacngd5djlqld',
+            'ext': 'mp4',
+            'thumbnail': r're:https?://uploads\.boxcast\.com/(?:[\w+-]+/){3}.+\.png$',
+            'title': 'In the Midst of Darkness Light Prevails: An Interdisciplinary Symposium',
+            'release_timestamp': 1670686812,
+            'release_date': '20221210',
+            'uploader_id': 're8w0v8hohhvpqtbskpe',
+            'uploader': 'Children\'s Health Defense',
+        }
+    }, {
+        'url': 'https://boxcast.tv/video-portal/vctwevwntun3o0ikq7af/rvyblnn0fxbfjx5nwxhl/otbpltj2kzkveo2qz3ad',
+        'info_dict': {
+            'id': 'otbpltj2kzkveo2qz3ad',
+            'ext': 'mp4',
+            'uploader_id': 'vctwevwntun3o0ikq7af',
+            'uploader': 'Legacy Christian Church',
+            'title': 'The Quest | 1: Beginner\'s Bay | Jamie Schools',
+            'thumbnail': r're:https?://uploads.boxcast.com/(?:[\w-]+/){3}.+\.jpg'
+        }
+    }, {
+        'url': 'https://boxcast.tv/channel/z03fqwaeaby5lnaawox2?b=ssihlw5gvfij2by8tkev',
+        'info_dict': {
+            'id': 'ssihlw5gvfij2by8tkev',
+            'ext': 'mp4',
+            'thumbnail': r're:https?://uploads.boxcast.com/(?:[\w-]+/){3}.+\.jpg$',
+            'release_date': '20230101',
+            'uploader_id': 'ds25vaazhlu4ygcvffid',
+            'release_timestamp': 1672543201,
+            'uploader': 'Lighthouse Ministries International  - Beltsville, Maryland',
+            'description': 'md5:ac23e3d01b0b0be592e8f7fe0ec3a340',
+            'title': 'New Year\'s Eve CROSSOVER Service at LHMI | December 31, 2022',
+        }
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://childrenshealthdefense.eu/live-stream/',
+        'info_dict': {
+            'id': 'da1eqqgkacngd5djlqld',
+            'ext': 'mp4',
+            'thumbnail': r're:https?://uploads\.boxcast\.com/(?:[\w+-]+/){3}.+\.png$',
+            'title': 'In the Midst of Darkness Light Prevails: An Interdisciplinary Symposium',
+            'release_timestamp': 1670686812,
+            'release_date': '20221210',
+            'uploader_id': 're8w0v8hohhvpqtbskpe',
+            'uploader': 'Children\'s Health Defense',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        webpage_json_data = self._search_json(
+            r'var\s*BOXCAST_PRELOAD\s*=', webpage, 'broadcast data', display_id,
+            transform_source=js_to_json, default={})
+
+        # Ref: https://support.boxcast.com/en/articles/4235158-build-a-custom-viewer-experience-with-boxcast-api
+        broadcast_json_data = (
+            traverse_obj(webpage_json_data, ('broadcast', 'data'))
+            or self._download_json(f'https://api.boxcast.com/broadcasts/{display_id}', display_id))
+        view_json_data = (
+            traverse_obj(webpage_json_data, ('view', 'data'))
+            or self._download_json(f'https://api.boxcast.com/broadcasts/{display_id}/view',
+                                   display_id, fatal=False) or {})
+
+        formats, subtitles = [], {}
+        if view_json_data.get('status') == 'recorded':
+            formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+                view_json_data['playlist'], display_id)
+
+        return {
+            'id': str(broadcast_json_data['id']),
+            'title': (broadcast_json_data.get('name')
+                      or self._html_search_meta(['og:title', 'twitter:title'], webpage)),
+            'description': (broadcast_json_data.get('description')
+                            or self._html_search_meta(['og:description', 'twitter:description'], webpage)
+                            or None),
+            'thumbnail': (broadcast_json_data.get('preview')
+                          or self._html_search_meta(['og:image', 'twitter:image'], webpage)),
+            'formats': formats,
+            'subtitles': subtitles,
+            'release_timestamp': unified_timestamp(broadcast_json_data.get('streamed_at')),
+            'uploader': broadcast_json_data.get('account_name'),
+            'uploader_id': broadcast_json_data.get('account_id'),
+        }

From 30031be974d210f451100339699ef03b0ddb5f10 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Fri, 17 Feb 2023 18:16:46 +0900
Subject: [PATCH 083/871] [extractor/tempo] Add IVXPlayer extractor (#5837)

Authored by: HobbyistDev
---
 yt_dlp/extractor/_extractors.py |   2 +-
 yt_dlp/extractor/tempo.py       | 119 ++++++++++++++++++++++++--------
 2 files changed, 91 insertions(+), 30 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 797e5668a..6dab2636b 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1858,7 +1858,7 @@
 )
 from .teletask import TeleTaskIE
 from .telewebion import TelewebionIE
-from .tempo import TempoIE
+from .tempo import TempoIE, IVXPlayerIE
 from .tencent import (
     IflixEpisodeIE,
     IflixSeriesIE,
diff --git a/yt_dlp/extractor/tempo.py b/yt_dlp/extractor/tempo.py
index 1cfb956e5..9318d6f9a 100644
--- a/yt_dlp/extractor/tempo.py
+++ b/yt_dlp/extractor/tempo.py
@@ -1,5 +1,81 @@
+import re
+
 from .common import InfoExtractor
-from ..utils import int_or_none, parse_iso8601, str_or_none, traverse_obj
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    traverse_obj,
+    try_call
+)
+
+
+class IVXPlayerIE(InfoExtractor):
+    _VALID_URL = r'ivxplayer:(?P<video_id>\d+):(?P<player_key>\w+)'
+    _TESTS = [{
+        'url': 'ivxplayer:2366065:4a89dfe6bc8f002596b1dfbd600730b1',
+        'info_dict': {
+            'id': '2366065',
+            'ext': 'mp4',
+            'duration': 112,
+            'upload_date': '20221204',
+            'title': 'Film Indonesia di Disney Content Showcase Asia Pacific 2022',
+            'timestamp': 1670151746,
+            'thumbnail': 'https://ivx-image.ivideosmart.com/serve/image/video/2366065?width=300'
+        }
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://www.cantika.com/video/31737/film-indonesia-di-disney-content-showcase-asia-pacific-2022',
+        'info_dict': {
+            'id': '2374200',
+            'ext': 'mp4',
+            'duration': 110,
+            'title': 'Serial Indonesia di Disney Content Showcase Asia Pacific 2022',
+            'timestamp': 1670639416,
+            'upload_date': '20221210',
+            'thumbnail': 'https://ivx-image.ivideosmart.com/serve/image/video/2374200?width=300'
+        }
+    }, {
+        'url': 'https://www.gooto.com/video/11437/wuling-suv-ramai-dikunjungi-di-giias-2018',
+        'info_dict': {
+            'id': '892109',
+            'ext': 'mp4',
+            'title': 'Wuling SUV Ramai Dikunjungi di GIIAS 2018',
+            'upload_date': '20180811',
+            'description': 'md5:6d901483d0aacc664aecb4489719aafa',
+            'duration': 75,
+            'timestamp': 1534011263,
+            'thumbnail': 'https://ivx-image.ivideosmart.com/serve/image/video/892109?width=300'
+        }
+    }]
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        # more info at https://player.ivideosmart.com/ivsplayer/v4/dist/js/loader.js
+        mobj = re.search(
+            r'<ivs-player\s*[^>]+data-ivs-key\s*=\s*"(?P<player_key>[\w]+)\s*[^>]+\bdata-ivs-vid="(?P<video_id>[\w-]+)',
+            webpage)
+        if mobj:
+            yield f'ivxplayer:{mobj.group("video_id")}:{mobj.group("player_key")}'
+            raise cls.StopExtraction()
+
+    def _real_extract(self, url):
+        video_id, player_key = self._match_valid_url(url).group('video_id', 'player_key')
+        json_data = self._download_json(
+            f'https://ivxplayer.ivideosmart.com/prod/video/{video_id}?key={player_key}', video_id)
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            json_data['player']['video_url'], video_id)
+
+        return {
+            'id': str(json_data['ivx']['id']),
+            'title': traverse_obj(json_data, ('ivx', 'name')),
+            'description': traverse_obj(json_data, ('ivx', 'description')),
+            'duration': int_or_none(traverse_obj(json_data, ('ivx', 'duration'))),
+            'timestamp': parse_iso8601(traverse_obj(json_data, ('ivx', 'published_at'))),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnail': traverse_obj(json_data, ('ivx', 'thumbnail_url'))
+        }
 
 
 class TempoIE(InfoExtractor):
@@ -7,14 +83,14 @@ class TempoIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://video.tempo.co/read/30058/anies-baswedan-ajukan-banding-putusan-ptun-batalkan-ump-dki',
         'info_dict': {
-            'id': '2144438',
+            'id': '2144275',
+            'display_id': 'anies-baswedan-ajukan-banding-putusan-ptun-batalkan-ump-dki',
             'ext': 'mp4',
             'title': 'Anies Baswedan Ajukan Banding Putusan PTUN Batalkan UMP DKI',
-            'display_id': 'anies-baswedan-ajukan-banding-putusan-ptun-batalkan-ump-dki',
-            'duration': 84,
+            'duration': 85,
             'description': 'md5:a6822b7c4c874fa7e5bd63e96a387b66',
             'thumbnail': 'https://statik.tempo.co/data/2022/07/27/id_1128287/1128287_720.jpg',
-            'timestamp': 1658911277,
+            'timestamp': 1658907970,
             'upload_date': '20220727',
             'tags': ['Anies Baswedan', ' PTUN', ' PTUN | Pengadilan Tata Usaha Negara', ' PTUN Batalkan UMP DKI', ' UMP DKI'],
         }
@@ -24,30 +100,15 @@ def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        player_key, widget_id = self._search_regex(
-            r'<ivs-player\s*[^>]+data-ivs-key\s*=\s*"(?P<player_key>[\w]+)[^>]+\bdata-ivs-wid="(?P<widget_id>[\w-]+)',
-            webpage, 'player_key, widget_id', group=('player_key', 'widget_id'))
+        _, video_id, player_key = next(IVXPlayerIE._extract_embed_urls(url, webpage)).split(':')
 
         json_ld_data = self._search_json_ld(webpage, display_id)
 
-        json_data = self._download_json(
-            f'https://ivxplayer.ivideosmart.com/prod/widget/{widget_id}',
-            display_id, query={'key': player_key})
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
-            json_data['player']['video_url'], display_id, ext='mp4')
-
-        return {
-            'id': str(json_data['ivx']['id']),
-            'display_id': display_id,
-            'formats': formats,
-            'subtitles': subtitles,
-            'title': (self._html_search_meta('twitter:title', webpage) or self._og_search_title(webpage)
-                      or traverse_obj(json_data, ('ivx', 'name'))),
-            'duration': int_or_none(traverse_obj(json_data, ('ivx', 'duration'))),
-            'thumbnail': (self._html_search_meta('twitter:image:src', webpage) or self._og_search_thumbnail(webpage)
-                          or traverse_obj(json_data, ('ivx', 'thumbnail_url'))),
-            'description': (json_ld_data.get('description') or self._html_search_meta(['description', 'twitter:description'], webpage)
-                            or self._og_search_description(webpage)),
-            'timestamp': parse_iso8601(traverse_obj(json_data, ('ivx', 'created_at'))),
-            'tags': str_or_none(self._html_search_meta('keywords', webpage), '').split(','),
-        }
+        return self.url_result(
+            f'ivxplayer:{video_id}:{player_key}', display_id=display_id,
+            thumbnail=self._html_search_meta('twitter:image:src', webpage) or self._og_search_thumbnail(webpage),
+            tags=try_call(lambda: self._html_search_meta('keywords', webpage).split(',')),
+            description=(json_ld_data.get('description')
+                         or self._html_search_meta(('description', 'twitter:description'), webpage)
+                         or self._og_search_description(webpage)),
+            url_transparent=True)

From a5387729696a5b33f53f60ef06f48e45663b12dd Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 17 Feb 2023 17:52:22 +0530
Subject: [PATCH 084/871] [cleanup] Misc

Closes #5897
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |  4 +--
 .../ISSUE_TEMPLATE/2_site_support_request.yml |  4 +--
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |  4 +--
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |  4 +--
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |  4 +--
 .github/ISSUE_TEMPLATE/6_question.yml         |  4 +--
 .github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml |  2 +-
 .../2_site_support_request.yml                |  2 +-
 .../3_site_feature_request.yml                |  2 +-
 .github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml  |  2 +-
 .../ISSUE_TEMPLATE_tmpl/5_feature_request.yml |  2 +-
 .github/ISSUE_TEMPLATE_tmpl/6_question.yml    |  2 +-
 .github/PULL_REQUEST_TEMPLATE.md              |  2 +-
 CONTRIBUTORS                                  |  2 +-
 Collaborators.md                              |  4 ++-
 README.md                                     |  8 +++--
 devscripts/make_issue_template.py             |  2 +-
 yt_dlp/YoutubeDL.py                           |  4 ++-
 yt_dlp/cache.py                               |  1 -
 yt_dlp/extractor/amazonminitv.py              |  3 +-
 yt_dlp/extractor/embedly.py                   | 29 +++++++++++++++++++
 yt_dlp/extractor/radiko.py                    |  4 +--
 yt_dlp/extractor/youtube.py                   | 13 ---------
 yt_dlp/options.py                             |  2 +-
 yt_dlp/postprocessor/metadataparser.py        |  4 ++-
 yt_dlp/utils.py                               | 17 +++++++++--
 26 files changed, 84 insertions(+), 47 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index d116cd7c6..2237665e3 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -7,7 +7,7 @@ body:
       label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
       description: Fill all fields even if you think it is irrelevant for the issue
       options:
-        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+        - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field
           required: true
   - type: checkboxes
     id: checklist
@@ -24,7 +24,7 @@ body:
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index 2bbf93a93..0e2940d86 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -7,7 +7,7 @@ body:
       label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
       description: Fill all fields even if you think it is irrelevant for the issue
       options:
-        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+        - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field
           required: true
   - type: checkboxes
     id: checklist
@@ -24,7 +24,7 @@ body:
           required: true
         - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index d1d3514f2..92501be2e 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -7,7 +7,7 @@ body:
       label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
       description: Fill all fields even if you think it is irrelevant for the issue
       options:
-        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+        - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field
           required: true
   - type: checkboxes
     id: checklist
@@ -22,7 +22,7 @@ body:
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index 8c851a945..bdfc0efb8 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -7,7 +7,7 @@ body:
       label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
       description: Fill all fields even if you think it is irrelevant for the issue
       options:
-        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+        - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field
           required: true
   - type: checkboxes
     id: checklist
@@ -24,7 +24,7 @@ body:
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index 444df3c32..c9e3aba38 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -7,7 +7,7 @@ body:
       label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
       description: Fill all fields even if you think it is irrelevant for the issue
       options:
-        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+        - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field
           required: true
   - type: checkboxes
     id: checklist
@@ -22,7 +22,7 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **2023.01.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index 997278f21..fe6a4ee3f 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -7,7 +7,7 @@ body:
       label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
       description: Fill all fields even if you think it is irrelevant for the issue
       options:
-        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+        - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field
           required: true
   - type: markdown
     attributes:
@@ -28,7 +28,7 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **2023.01.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
index e1b1e5138..85900e92e 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml
index 12a1c6598..75d62e7bb 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
           required: true
         - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml
index 2b46650f7..18b30f578 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml
@@ -16,7 +16,7 @@ body:
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml
index 377efbe33..90f59e70b 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml
index 8bbc5d733..ef3bb2269 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml
@@ -16,7 +16,7 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/6_question.yml b/.github/ISSUE_TEMPLATE_tmpl/6_question.yml
index ee09e82a3..4bef82d5a 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/6_question.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/6_question.yml
@@ -22,7 +22,7 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
+        - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 7c271565f..c4d3e812e 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -30,7 +30,7 @@ ### Before submitting a *pull request* make sure you have:
 - [ ] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
 - [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8) and [ran relevant tests](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions)
 
-### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
+### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check all of the following options that apply:
 - [ ] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
 - [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence)
 
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 18fd70e4d..e3b95e2f3 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -4,6 +4,7 @@ coletdjnz/colethedj (collaborator)
 Ashish0804 (collaborator)
 nao20010128nao/Lesmiscore (collaborator)
 bashonly (collaborator)
+Grub4K (collaborator)
 h-h-h-h
 pauldubois98
 nixxo
@@ -319,7 +320,6 @@ columndeeply
 DoubleCouponDay
 Fabi019
 GautamMKGarg
-Grub4K
 itachi-19
 jeroenj
 josanabr
diff --git a/Collaborators.md b/Collaborators.md
index 3bce437c9..fe2a7f4b4 100644
--- a/Collaborators.md
+++ b/Collaborators.md
@@ -8,6 +8,7 @@ # Collaborators
 ## [pukkandan](https://github.com/pukkandan)
 
 [![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/pukkandan)
+[![gh-sponsor](https://img.shields.io/badge/_-Github-red.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/pukkandan)
 
 * Owner of the fork
 
@@ -25,8 +26,9 @@ ## [shirt](https://github.com/shirt-dev)
 
 ## [coletdjnz](https://github.com/coletdjnz)
 
-[![gh-sponsor](https://img.shields.io/badge/_-Sponsor-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/coletdjnz)
+[![gh-sponsor](https://img.shields.io/badge/_-Github-red.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/coletdjnz)
 
+* Improved plugin architecture
 * YouTube improvements including: age-gate bypass, private playlists, multiple-clients (to avoid throttling) and a lot of under-the-hood improvements
 * Added support for new websites YoutubeWebArchive, MainStreaming, PRX, nzherald, Mediaklikk, StarTV etc
 * Improved/fixed support for Patreon, panopto, gfycat, itv, pbs, SouthParkDE etc
diff --git a/README.md b/README.md
index 07c74d6c3..29a6c06fd 100644
--- a/README.md
+++ b/README.md
@@ -788,7 +788,7 @@ ## Workarounds:
     --prefer-insecure               Use an unencrypted connection to retrieve
                                     information about the video (Currently
                                     supported only for YouTube)
-    --add-header FIELD:VALUE        Specify a custom HTTP header and its value,
+    --add-headers FIELD:VALUE       Specify a custom HTTP header and its value,
                                     separated by a colon ":". You can use this
                                     option multiple times
     --bidi-workaround               Work around terminals that lack
@@ -1511,7 +1511,7 @@ ## Sorting Formats
  - `source`: The preference of the source
  - `proto`: Protocol used for download (`https`/`ftps` > `http`/`ftp` > `m3u8_native`/`m3u8` > `http_dash_segments`> `websocket_frag` > `mms`/`rtsp` > `f4f`/`f4m`)
  - `vcodec`: Video Codec (`av01` > `vp9.2` > `vp9` > `h265` > `h264` > `vp8` > `h263` > `theora` > other)
- - `acodec`: Audio Codec (`flac`/`alac` > `wav`/`aiff` > `opus` > `vorbis` > `aac` > `mp4a` > `mp3` `ac4` > > `eac3` > `ac3` > `dts` > other)
+ - `acodec`: Audio Codec (`flac`/`alac` > `wav`/`aiff` > `opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `ac4` > `eac3` > `ac3` > `dts` > other)
  - `codec`: Equivalent to `vcodec,acodec`
  - `vext`: Video Extension (`mp4` > `mov` > `webm` > `flv` > other). If `--prefer-free-formats` is used, `webm` is preferred.
  - `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other). If `--prefer-free-formats` is used, the order changes to `ogg` > `opus` > `webm` > `mp3` > `m4a` > `aac`
@@ -1741,6 +1741,8 @@ # EXTRACTOR ARGUMENTS
 
 Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. E.g. `--extractor-args "youtube:player-client=android_embedded,web;include_live_dash" --extractor-args "funimation:version=uncut"`
 
+Note: In CLI, `ARG` can use `-` instead of `_`; e.g. `youtube:player-client"` becomes `youtube:player_client"`
+
 The following extractors use this feature:
 
 #### youtube
@@ -1887,7 +1889,7 @@ # EMBEDDING YT-DLP
     ydl.download(URLS)
 ```
 
-Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L180).
+Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L184).
 
 **Tip**: If you are porting your code from youtube-dl to yt-dlp, one important point to look out for is that we do not guarantee the return value of `YoutubeDL.extract_info` to be json serializable, or even be a dictionary. It will be dictionary-like, but if you want to ensure it is a serializable dictionary, pass it through `YoutubeDL.sanitize_info` as shown in the [example below](#extracting-information)
 
diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index fd964c6c6..1ee00f2b8 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -58,7 +58,7 @@
       label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
       description: Fill all fields even if you think it is irrelevant for the issue
       options:
-        - label: I understand that I will be **blocked** if I remove or skip any mandatory\\* field
+        - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\\* field
           required: true
 '''.strip()
 
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 4e5c40b58..d6c5ce769 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3665,6 +3665,7 @@ def simplified_codec(f, field):
                 format_field(f, 'asr', '\t%s', func=format_decimal_suffix),
                 join_nonempty(
                     self._format_out('UNSUPPORTED', 'light red') if f.get('ext') in ('f4f', 'f4m') else None,
+                    self._format_out('DRM', 'light red') if f.get('has_drm') else None,
                     format_field(f, 'language', '[%s]'),
                     join_nonempty(format_field(f, 'format_note'),
                                   format_field(f, 'container', ignore=(None, f.get('ext'))),
@@ -3764,12 +3765,13 @@ def get_encoding(stream):
         source = detect_variant()
         if VARIANT not in (None, 'pip'):
             source += '*'
+        klass = type(self)
         write_debug(join_nonempty(
             f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
             __version__,
             f'[{RELEASE_GIT_HEAD}]' if RELEASE_GIT_HEAD else '',
             '' if source == 'unknown' else f'({source})',
-            '' if _IN_CLI else 'API',
+            '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}',
             delim=' '))
 
         if not _IN_CLI:
diff --git a/yt_dlp/cache.py b/yt_dlp/cache.py
index f8344fe77..9dd4f2f25 100644
--- a/yt_dlp/cache.py
+++ b/yt_dlp/cache.py
@@ -1,5 +1,4 @@
 import contextlib
-import errno
 import json
 import os
 import re
diff --git a/yt_dlp/extractor/amazonminitv.py b/yt_dlp/extractor/amazonminitv.py
index 730996853..b57d985d1 100644
--- a/yt_dlp/extractor/amazonminitv.py
+++ b/yt_dlp/extractor/amazonminitv.py
@@ -191,7 +191,7 @@ def _real_extract(self, url):
 class AmazonMiniTVSeasonIE(AmazonMiniTVBaseIE):
     IE_NAME = 'amazonminitv:season'
     _VALID_URL = r'amazonminitv:season:(?:amzn1\.dv\.gti\.)?(?P<id>[a-f0-9-]+)'
-    IE_DESC = 'Amazon MiniTV Series, "minitv:season:" prefix'
+    IE_DESC = 'Amazon MiniTV Season, "minitv:season:" prefix'
     _TESTS = [{
         'url': 'amazonminitv:season:amzn1.dv.gti.0aa996eb-6a1b-4886-a342-387fbd2f1db0',
         'playlist_mincount': 6,
@@ -250,6 +250,7 @@ def _real_extract(self, url):
 class AmazonMiniTVSeriesIE(AmazonMiniTVBaseIE):
     IE_NAME = 'amazonminitv:series'
     _VALID_URL = r'amazonminitv:series:(?:amzn1\.dv\.gti\.)?(?P<id>[a-f0-9-]+)'
+    IE_DESC = 'Amazon MiniTV Series, "minitv:series:" prefix'
     _TESTS = [{
         'url': 'amazonminitv:series:amzn1.dv.gti.56521d46-b040-4fd5-872e-3e70476a04b0',
         'playlist_mincount': 3,
diff --git a/yt_dlp/extractor/embedly.py b/yt_dlp/extractor/embedly.py
index 1b58fca60..458aaa0a0 100644
--- a/yt_dlp/extractor/embedly.py
+++ b/yt_dlp/extractor/embedly.py
@@ -61,6 +61,35 @@ class EmbedlyIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    _WEBPAGE_TESTS = [{
+        'url': 'http://www.permacultureetc.com/2022/12/comment-greffer-facilement-les-arbres-fruitiers.html',
+        'info_dict': {
+            'id': 'pfUK_ADTvgY',
+            'ext': 'mp4',
+            'title': 'Comment greffer facilement les arbres fruitiers ? (mois par mois)',
+            'description': 'md5:d3a876995e522f138aabb48e040bfb4c',
+            'view_count': int,
+            'upload_date': '20221210',
+            'comment_count': int,
+            'live_status': 'not_live',
+            'channel_id': 'UCsM4_jihNFYe4CtSkXvDR-Q',
+            'channel_follower_count': int,
+            'tags': ['permaculture', 'jardinage', 'dekarz', 'autonomie', 'greffe', 'fruitiers', 'arbres', 'jardin forêt', 'forêt comestible', 'damien'],
+            'playable_in_embed': True,
+            'uploader': 'permaculture agroécologie etc...',
+            'channel': 'permaculture agroécologie etc...',
+            'thumbnail': 'https://i.ytimg.com/vi/pfUK_ADTvgY/sddefault.jpg',
+            'duration': 1526,
+            'channel_url': 'https://www.youtube.com/channel/UCsM4_jihNFYe4CtSkXvDR-Q',
+            'age_limit': 0,
+            'uploader_id': 'permacultureetc',
+            'like_count': int,
+            'uploader_url': 'http://www.youtube.com/user/permacultureetc',
+            'categories': ['Education'],
+            'availability': 'public',
+        },
+    }]
+
     @classmethod
     def _extract_from_webpage(cls, url, webpage):
         # Bypass "ie=cls" and suitable check
diff --git a/yt_dlp/extractor/radiko.py b/yt_dlp/extractor/radiko.py
index 43eecba5f..7fdf78283 100644
--- a/yt_dlp/extractor/radiko.py
+++ b/yt_dlp/extractor/radiko.py
@@ -133,9 +133,9 @@ def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token,
                     'X-Radiko-AreaId': area_id,
                     'X-Radiko-AuthToken': auth_token,
                 })
-            not_preferred = is_onair and not pcu.startswith(self._HOSTS_FOR_LIVE) or (not is_onair and (pcu.startswith(self._HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED) or pcu.startswith(self._HOSTS_FOR_LIVE)))
             for sf in subformats:
-                if not_preferred:
+                if (is_onair ^ pcu.startswith(self._HOSTS_FOR_LIVE)) or (
+                        not is_onair and pcu.startswith(self._HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED)):
                     sf['preference'] = -100
                     sf['format_note'] = 'not preferred'
                 if not is_onair and url_attrib['timefree'] == '1' and time_to_skip:
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index d891d92a3..be82bc689 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4459,19 +4459,6 @@ def wrapper(self, url):
             return info_dict
         return wrapper
 
-    def _extract_channel_id(self, webpage):
-        channel_id = self._html_search_meta(
-            'channelId', webpage, 'channel id', default=None)
-        if channel_id:
-            return channel_id
-        channel_url = self._html_search_meta(
-            ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
-             'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
-             'twitter:app:url:googleplay'), webpage, 'channel url')
-        return self._search_regex(
-            r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
-            channel_url, 'channel id')
-
     @staticmethod
     def _extract_basic_item_renderer(item):
         # Modified from _extract_grid_item_renderer
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 68a3aecc4..fd60ff55f 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -1031,7 +1031,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         metavar='URL', dest='referer', default=None,
         help=optparse.SUPPRESS_HELP)
     workarounds.add_option(
-        '--add-header',
+        '--add-headers',
         metavar='FIELD:VALUE', dest='headers', default={}, type='str',
         action='callback', callback=_dict_from_options_callback,
         callback_kwargs={'multiple_keys': False},
diff --git a/yt_dlp/postprocessor/metadataparser.py b/yt_dlp/postprocessor/metadataparser.py
index f574f2330..1d6054294 100644
--- a/yt_dlp/postprocessor/metadataparser.py
+++ b/yt_dlp/postprocessor/metadataparser.py
@@ -1,7 +1,7 @@
 import re
 
 from .common import PostProcessor
-from ..utils import Namespace, filter_dict
+from ..utils import Namespace, filter_dict, function_with_repr
 
 
 class MetadataParserPP(PostProcessor):
@@ -60,6 +60,7 @@ def run(self, info):
             f(info)
         return [], info
 
+    @function_with_repr
     def interpretter(self, inp, out):
         def f(info):
             data_to_parse = self._downloader.evaluate_outtmpl(template, info)
@@ -76,6 +77,7 @@ def f(info):
         out_re = re.compile(self.format_to_regex(out))
         return f
 
+    @function_with_repr
     def replacer(self, field, search, replace):
         def f(info):
             val = info.get(field)
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 736468aef..9eb9495a0 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3650,7 +3650,8 @@ def get_compatible_ext(*, vcodecs, acodecs, vexts, aexts, preferences=None):
         },
     }
 
-    sanitize_codec = functools.partial(try_get, getter=lambda x: x[0].split('.')[0].replace('0', ''))
+    sanitize_codec = functools.partial(
+        try_get, getter=lambda x: x[0].split('.')[0].replace('0', '').lower())
     vcodec, acodec = sanitize_codec(vcodecs), sanitize_codec(acodecs)
 
     for ext in preferences or COMPATIBLE_CODECS.keys():
@@ -3915,7 +3916,7 @@ def __eq__(self, other):
                 and self.chapters == other.chapters and self.ranges == other.ranges)
 
     def __repr__(self):
-        return f'{type(self).__name__}({self.chapters}, {self.ranges})'
+        return f'{__name__}.{type(self).__name__}({self.chapters}, {self.ranges})'
 
 
 def parse_dfxp_time_expr(time_expr):
@@ -6018,6 +6019,18 @@ def __get__(self, _, cls):
         return self._cache[cls]
 
 
+class function_with_repr:
+    def __init__(self, func):
+        functools.update_wrapper(self, func)
+        self.func = func
+
+    def __call__(self, *args, **kwargs):
+        return self.func(*args, **kwargs)
+
+    def __repr__(self):
+        return f'{self.func.__module__}.{self.func.__qualname__}'
+
+
 class Namespace(types.SimpleNamespace):
     """Immutable namespace"""
 

From 45b2ee6f4fae139892a1a4335c269dcbb6671497 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 17 Feb 2023 16:51:34 +0530
Subject: [PATCH 085/871] Update to ytdl-commit-2dd6c6e

[YouTube] Avoid crash if uploader_id extraction fails
https://github.com/ytdl-org/youtube-dl/commit/2dd6c6edd8e0fc5e45865b8e6d865e35147de772

Except:
    * 295736c9cba714fb5de7d1c3dd31d86e50091cf8 [jsinterp] Improve parsing
    * 384f632e8a9b61e864a26678d85b2b39933b9bae [ITV] Overhaul ITV extractor
    * 33db85c571304bbd6863e3407ad8d08764c9e53b [feat]: Add support to external downloader aria2p
---
 README.md                               |   2 +-
 test/test_InfoExtractor.py              |   2 +
 test/test_age_restriction.py            |  19 +-
 yt_dlp/compat/_legacy.py                |  30 ++-
 yt_dlp/extractor/_extractors.py         |   7 +
 yt_dlp/extractor/americastestkitchen.py |  78 +++++-
 yt_dlp/extractor/blerp.py               | 167 ++++++++++++
 yt_dlp/extractor/callin.py              |  55 +++-
 yt_dlp/extractor/cammodels.py           |  39 +--
 yt_dlp/extractor/common.py              |   2 +-
 yt_dlp/extractor/ign.py                 | 337 +++++++++++++++++-------
 yt_dlp/extractor/kommunetv.py           |  31 +++
 yt_dlp/extractor/myvideoge.py           |  68 +++--
 yt_dlp/extractor/pr0gramm.py            |  97 +++++++
 yt_dlp/extractor/rbgtum.py              |  93 +++++++
 yt_dlp/extractor/unsupported.py         |   3 +
 yt_dlp/extractor/vimeo.py               |  55 ++--
 yt_dlp/extractor/xhamster.py            |   8 +-
 yt_dlp/utils.py                         |  28 +-
 19 files changed, 911 insertions(+), 210 deletions(-)
 create mode 100644 yt_dlp/extractor/blerp.py
 create mode 100644 yt_dlp/extractor/kommunetv.py
 create mode 100644 yt_dlp/extractor/pr0gramm.py
 create mode 100644 yt_dlp/extractor/rbgtum.py

diff --git a/README.md b/README.md
index 29a6c06fd..9b91775bc 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,7 @@
 
 # NEW FEATURES
 
-* Merged with **youtube-dl v2021.12.17+ [commit/195f22f](https://github.com/ytdl-org/youtube-dl/commit/195f22f)** <!--([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))--> and **youtube-dlc v2020.11.11-3+ [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
+* Merged with **youtube-dl v2021.12.17+ [commit/2dd6c6e](https://github.com/ytdl-org/youtube-dl/commit/2dd6c6e)** ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21)) and **youtube-dlc v2020.11.11-3+ [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 683ead315..e8d94a6ac 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -69,6 +69,7 @@ def test_opengraph(self):
             <meta name="og:test1" content='foo > < bar'/>
             <meta name="og:test2" content="foo >//< bar"/>
             <meta property=og-test3 content='Ill-formatted opengraph'/>
+            <meta property=og:test4 content=unquoted-value/>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
@@ -81,6 +82,7 @@ def test_opengraph(self):
         self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
         self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
         self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)
+        self.assertEqual(ie._og_search_property('test4', html), 'unquoted-value')
 
     def test_html_search_meta(self):
         ie = self.ie
diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py
index ff248432b..68107590e 100644
--- a/test/test_age_restriction.py
+++ b/test/test_age_restriction.py
@@ -10,6 +10,7 @@
 
 from test.helper import is_download_test, try_rm
 from yt_dlp import YoutubeDL
+from yt_dlp.utils import DownloadError
 
 
 def _download_restricted(url, filename, age):
@@ -25,10 +26,14 @@ def _download_restricted(url, filename, age):
     ydl.add_default_info_extractors()
     json_filename = os.path.splitext(filename)[0] + '.info.json'
     try_rm(json_filename)
-    ydl.download([url])
-    res = os.path.exists(json_filename)
-    try_rm(json_filename)
-    return res
+    try:
+        ydl.download([url])
+    except DownloadError:
+        pass
+    else:
+        return os.path.exists(json_filename)
+    finally:
+        try_rm(json_filename)
 
 
 @is_download_test
@@ -38,12 +43,12 @@ def _assert_restricted(self, url, filename, age, old_age=None):
         self.assertFalse(_download_restricted(url, filename, age))
 
     def test_youtube(self):
-        self._assert_restricted('07FYdnEawAQ', '07FYdnEawAQ.mp4', 10)
+        self._assert_restricted('HtVdAasjOgU', 'HtVdAasjOgU.mp4', 10)
 
     def test_youporn(self):
         self._assert_restricted(
-            'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
-            '505835.mp4', 2, old_age=25)
+            'https://www.youporn.com/watch/16715086/sex-ed-in-detention-18-asmr/',
+            '16715086.mp4', 2, old_age=25)
 
 
 if __name__ == '__main__':
diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py
index d19333d31..84d749209 100644
--- a/yt_dlp/compat/_legacy.py
+++ b/yt_dlp/compat/_legacy.py
@@ -1,5 +1,6 @@
 """ Do not use! """
 
+import base64
 import collections
 import ctypes
 import getpass
@@ -29,6 +30,7 @@
 from re import Pattern as compat_Pattern  # noqa: F401
 from re import match as compat_Match  # noqa: F401
 
+from . import compat_expanduser, compat_HTMLParseError, compat_realpath
 from .compat_utils import passthrough_module
 from ..dependencies import Cryptodome_AES as compat_pycrypto_AES  # noqa: F401
 from ..dependencies import brotli as compat_brotli  # noqa: F401
@@ -47,23 +49,25 @@ def compat_setenv(key, value, env=os.environ):
     env[key] = value
 
 
+compat_base64_b64decode = base64.b64decode
 compat_basestring = str
 compat_casefold = str.casefold
 compat_chr = chr
 compat_collections_abc = collections.abc
-compat_cookiejar = http.cookiejar
-compat_cookiejar_Cookie = http.cookiejar.Cookie
-compat_cookies = http.cookies
-compat_cookies_SimpleCookie = http.cookies.SimpleCookie
-compat_etree_Element = etree.Element
-compat_etree_register_namespace = etree.register_namespace
+compat_cookiejar = compat_http_cookiejar = http.cookiejar
+compat_cookiejar_Cookie = compat_http_cookiejar_Cookie = http.cookiejar.Cookie
+compat_cookies = compat_http_cookies = http.cookies
+compat_cookies_SimpleCookie = compat_http_cookies_SimpleCookie = http.cookies.SimpleCookie
+compat_etree_Element = compat_xml_etree_ElementTree_Element = etree.Element
+compat_etree_register_namespace = compat_xml_etree_register_namespace = etree.register_namespace
 compat_filter = filter
 compat_get_terminal_size = shutil.get_terminal_size
 compat_getenv = os.getenv
-compat_getpass = getpass.getpass
+compat_getpass = compat_getpass_getpass = getpass.getpass
 compat_html_entities = html.entities
 compat_html_entities_html5 = html.entities.html5
-compat_HTMLParser = html.parser.HTMLParser
+compat_html_parser_HTMLParseError = compat_HTMLParseError
+compat_HTMLParser = compat_html_parser_HTMLParser = html.parser.HTMLParser
 compat_http_client = http.client
 compat_http_server = http.server
 compat_input = input
@@ -72,6 +76,8 @@ def compat_setenv(key, value, env=os.environ):
 compat_kwargs = lambda kwargs: kwargs
 compat_map = map
 compat_numeric_types = (int, float, complex)
+compat_os_path_expanduser = compat_expanduser
+compat_os_path_realpath = compat_realpath
 compat_print = print
 compat_shlex_split = shlex.split
 compat_socket_create_connection = socket.create_connection
@@ -81,7 +87,9 @@ def compat_setenv(key, value, env=os.environ):
 compat_subprocess_get_DEVNULL = lambda: DEVNULL
 compat_tokenize_tokenize = tokenize.tokenize
 compat_urllib_error = urllib.error
+compat_urllib_HTTPError = urllib.error.HTTPError
 compat_urllib_parse = urllib.parse
+compat_urllib_parse_parse_qs = urllib.parse.parse_qs
 compat_urllib_parse_quote = urllib.parse.quote
 compat_urllib_parse_quote_plus = urllib.parse.quote_plus
 compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus
@@ -90,8 +98,10 @@ def compat_setenv(key, value, env=os.environ):
 compat_urllib_request = urllib.request
 compat_urllib_request_DataHandler = urllib.request.DataHandler
 compat_urllib_response = urllib.response
-compat_urlretrieve = urllib.request.urlretrieve
-compat_xml_parse_error = etree.ParseError
+compat_urlretrieve = compat_urllib_request_urlretrieve = urllib.request.urlretrieve
+compat_xml_parse_error = compat_xml_etree_ElementTree_ParseError = etree.ParseError
 compat_xpath = lambda xpath: xpath
 compat_zip = zip
 workaround_optparse_bug9161 = lambda: None
+
+legacy = []
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 6dab2636b..a7bcafb4c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -239,6 +239,7 @@
     BleacherReportIE,
     BleacherReportCMSIE,
 )
+from .blerp import BlerpIE
 from .blogger import BloggerIE
 from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
@@ -861,6 +862,7 @@
 from .kickstarter import KickStarterIE
 from .kinja import KinjaEmbedIE
 from .kinopoisk import KinoPoiskIE
+from .kommunetv import KommunetvIE
 from .kompas import KompasVideoIE
 from .konserthusetplay import KonserthusetPlayIE
 from .koo import KooIE
@@ -1460,6 +1462,7 @@
     PuhuTVIE,
     PuhuTVSerieIE,
 )
+from .pr0gramm import Pr0grammStaticIE, Pr0grammIE
 from .prankcast import PrankCastIE
 from .premiershiprugby import PremiershipRugbyIE
 from .presstv import PressTVIE
@@ -1521,6 +1524,10 @@
     RayWenderlichCourseIE,
 )
 from .rbmaradio import RBMARadioIE
+from .rbgtum import (
+    RbgTumIE,
+    RbgTumCourseIE,
+)
 from .rcs import (
     RCSIE,
     RCSEmbedsIE,
diff --git a/yt_dlp/extractor/americastestkitchen.py b/yt_dlp/extractor/americastestkitchen.py
index abda55dcf..e889458a2 100644
--- a/yt_dlp/extractor/americastestkitchen.py
+++ b/yt_dlp/extractor/americastestkitchen.py
@@ -11,7 +11,7 @@
 
 
 class AmericasTestKitchenIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com/(?:cooks(?:country|illustrated)/)?(?P<resource_type>episode|videos)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:americastestkitchen|cooks(?:country|illustrated))\.com/(?:cooks(?:country|illustrated)/)?(?P<resource_type>episode|videos)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.americastestkitchen.com/episode/582-weeknight-japanese-suppers',
         'md5': 'b861c3e365ac38ad319cfd509c30577f',
@@ -72,6 +72,12 @@ class AmericasTestKitchenIE(InfoExtractor):
     }, {
         'url': 'https://www.americastestkitchen.com/cooksillustrated/videos/4478-beef-wellington',
         'only_matching': True,
+    }, {
+        'url': 'https://www.cookscountry.com/episode/564-when-only-chocolate-will-do',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cooksillustrated.com/videos/4478-beef-wellington',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -100,7 +106,7 @@ def _real_extract(self, url):
 
 
 class AmericasTestKitchenSeasonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com(?P<show>/cookscountry)?/episodes/browse/season_(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<show>americastestkitchen|(?P<cooks>cooks(?:country|illustrated)))\.com(?:(?:/(?P<show2>cooks(?:country|illustrated)))?(?:/?$|(?<!ated)(?<!ated\.com)/episodes/browse/season_(?P<season>\d+)))'
     _TESTS = [{
         # ATK Season
         'url': 'https://www.americastestkitchen.com/episodes/browse/season_1',
@@ -117,29 +123,73 @@ class AmericasTestKitchenSeasonIE(InfoExtractor):
             'title': 'Season 12',
         },
         'playlist_count': 13,
+    }, {
+        # America's Test Kitchen Series
+        'url': 'https://www.americastestkitchen.com/',
+        'info_dict': {
+            'id': 'americastestkitchen',
+            'title': 'America\'s Test Kitchen',
+        },
+        'playlist_count': 558,
+    }, {
+        # Cooks Country Series
+        'url': 'https://www.americastestkitchen.com/cookscountry',
+        'info_dict': {
+            'id': 'cookscountry',
+            'title': 'Cook\'s Country',
+        },
+        'playlist_count': 199,
+    }, {
+        'url': 'https://www.americastestkitchen.com/cookscountry/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cookscountry.com/episodes/browse/season_12',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cookscountry.com',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.americastestkitchen.com/cooksillustrated/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cooksillustrated.com',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        show_path, season_number = self._match_valid_url(url).group('show', 'id')
-        season_number = int(season_number)
+        season_number, show1, show = self._match_valid_url(url).group('season', 'show', 'show2')
+        show_path = ('/' + show) if show else ''
+        show = show or show1
+        season_number = int_or_none(season_number)
 
-        slug = 'cco' if show_path == '/cookscountry' else 'atk'
+        slug, title = {
+            'americastestkitchen': ('atk', 'America\'s Test Kitchen'),
+            'cookscountry': ('cco', 'Cook\'s Country'),
+            'cooksillustrated': ('cio', 'Cook\'s Illustrated'),
+        }[show]
 
-        season = 'Season %d' % season_number
+        facet_filters = [
+            'search_document_klass:episode',
+            'search_show_slug:' + slug,
+        ]
+
+        if season_number:
+            playlist_id = 'season_%d' % season_number
+            playlist_title = 'Season %d' % season_number
+            facet_filters.append('search_season_list:' + playlist_title)
+        else:
+            playlist_id = show
+            playlist_title = title
 
         season_search = self._download_json(
             'https://y1fnzxui30-dsn.algolia.net/1/indexes/everest_search_%s_season_desc_production' % slug,
-            season, headers={
+            playlist_id, headers={
                 'Origin': 'https://www.americastestkitchen.com',
                 'X-Algolia-API-Key': '8d504d0099ed27c1b73708d22871d805',
                 'X-Algolia-Application-Id': 'Y1FNZXUI30',
             }, query={
-                'facetFilters': json.dumps([
-                    'search_season_list:' + season,
-                    'search_document_klass:episode',
-                    'search_show_slug:' + slug,
-                ]),
-                'attributesToRetrieve': 'description,search_%s_episode_number,search_document_date,search_url,title' % slug,
+                'facetFilters': json.dumps(facet_filters),
+                'attributesToRetrieve': 'description,search_%s_episode_number,search_document_date,search_url,title,search_atk_episode_season' % slug,
                 'attributesToHighlight': '',
                 'hitsPerPage': 1000,
             })
@@ -162,4 +212,4 @@ def entries():
                 }
 
         return self.playlist_result(
-            entries(), 'season_%d' % season_number, season)
+            entries(), playlist_id, playlist_title)
diff --git a/yt_dlp/extractor/blerp.py b/yt_dlp/extractor/blerp.py
new file mode 100644
index 000000000..4631ad2e9
--- /dev/null
+++ b/yt_dlp/extractor/blerp.py
@@ -0,0 +1,167 @@
+import json
+
+from .common import InfoExtractor
+from ..utils import strip_or_none, traverse_obj
+
+
+class BlerpIE(InfoExtractor):
+    IE_NAME = 'blerp'
+    _VALID_URL = r'https?://(?:www\.)?blerp\.com/soundbites/(?P<id>[0-9a-zA-Z]+)'
+    _TESTS = [{
+        'url': 'https://blerp.com/soundbites/6320fe8745636cb4dd677a5a',
+        'info_dict': {
+            'id': '6320fe8745636cb4dd677a5a',
+            'title': 'Samsung Galaxy S8 Over the Horizon Ringtone 2016',
+            'uploader': 'luminousaj',
+            'uploader_id': '5fb81e51aa66ae000c395478',
+            'ext': 'mp3',
+            'tags': ['samsung', 'galaxy', 's8', 'over the horizon', '2016', 'ringtone'],
+        }
+    }, {
+        'url': 'https://blerp.com/soundbites/5bc94ef4796001000498429f',
+        'info_dict': {
+            'id': '5bc94ef4796001000498429f',
+            'title': 'Yee',
+            'uploader': '179617322678353920',
+            'uploader_id': '5ba99cf71386730004552c42',
+            'ext': 'mp3',
+            'tags': ['YEE', 'YEET', 'wo ha haah catchy tune yee', 'yee']
+        }
+    }]
+
+    _GRAPHQL_OPERATIONNAME = "webBitePageGetBite"
+    _GRAPHQL_QUERY = (
+        '''query webBitePageGetBite($_id: MongoID!) {
+            web {
+                biteById(_id: $_id) {
+                    ...bitePageFrag
+                    __typename
+                }
+                __typename
+            }
+        }
+
+        fragment bitePageFrag on Bite {
+            _id
+            title
+            userKeywords
+            keywords
+            color
+            visibility
+            isPremium
+            owned
+            price
+            extraReview
+            isAudioExists
+            image {
+                filename
+                original {
+                    url
+                    __typename
+                }
+                __typename
+            }
+            userReactions {
+                _id
+                reactions
+                createdAt
+                __typename
+            }
+            topReactions
+            totalSaveCount
+            saved
+            blerpLibraryType
+            license
+            licenseMetaData
+            playCount
+            totalShareCount
+            totalFavoriteCount
+            totalAddedToBoardCount
+            userCategory
+            userAudioQuality
+            audioCreationState
+            transcription
+            userTranscription
+            description
+            createdAt
+            updatedAt
+            author
+            listingType
+            ownerObject {
+                _id
+                username
+                profileImage {
+                    filename
+                    original {
+                        url
+                        __typename
+                    }
+                    __typename
+                }
+                __typename
+            }
+            transcription
+            favorited
+            visibility
+            isCurated
+            sourceUrl
+            audienceRating
+            strictAudienceRating
+            ownerId
+            reportObject {
+                reportedContentStatus
+                __typename
+            }
+            giphy {
+                mp4
+                gif
+                __typename
+            }
+            audio {
+                filename
+                original {
+                    url
+                    __typename
+                }
+                mp3 {
+                    url
+                    __typename
+                }
+                __typename
+            }
+            __typename
+        }
+
+        ''')
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+
+        data = {
+            'operationName': self._GRAPHQL_OPERATIONNAME,
+            'query': self._GRAPHQL_QUERY,
+            'variables': {
+                '_id': audio_id
+            }
+        }
+
+        headers = {
+            'Content-Type': 'application/json'
+        }
+
+        json_result = self._download_json('https://api.blerp.com/graphql',
+                                          audio_id, data=json.dumps(data).encode('utf-8'), headers=headers)
+
+        bite_json = json_result['data']['web']['biteById']
+
+        info_dict = {
+            'id': bite_json['_id'],
+            'url': bite_json['audio']['mp3']['url'],
+            'title': bite_json['title'],
+            'uploader': traverse_obj(bite_json, ('ownerObject', 'username'), expected_type=strip_or_none),
+            'uploader_id': traverse_obj(bite_json, ('ownerObject', '_id'), expected_type=strip_or_none),
+            'ext': 'mp3',
+            'tags': list(filter(None, map(strip_or_none, (traverse_obj(bite_json, 'userKeywords', expected_type=list) or []))) or None)
+        }
+
+        return info_dict
diff --git a/yt_dlp/extractor/callin.py b/yt_dlp/extractor/callin.py
index e9668763e..c77179c7b 100644
--- a/yt_dlp/extractor/callin.py
+++ b/yt_dlp/extractor/callin.py
@@ -1,9 +1,5 @@
 from .common import InfoExtractor
-from ..utils import (
-    traverse_obj,
-    float_or_none,
-    int_or_none
-)
+from ..utils import float_or_none, int_or_none, make_archive_id, traverse_obj
 
 
 class CallinIE(InfoExtractor):
@@ -35,6 +31,54 @@ class CallinIE(InfoExtractor):
             'episode_number': 1,
             'episode_id': '218b979630a35ead12c6fd096f2996c56c37e4d0dc1f6dc0feada32dcf7b31cd'
         }
+    }, {
+        'url': 'https://www.callin.com/episode/fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
+        'md5': '14ede27ee2c957b7e4db93140fc0745c',
+        'info_dict': {
+            'id': 'c3dab47f237bf953d180d3f243477a84302798be0e0b29bc9ade6d60a69f04f5',
+            'ext': 'ts',
+            'title': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
+            'description': 'Or, why the government doesn’t like SpaceX',
+            'channel': 'The Pull Request',
+            'channel_url': 'https://callin.com/show/the-pull-request-ucnDJmEKAa',
+            'duration': 3182.472,
+            'series_id': '7e9c23156e4aecfdcaef46bfb2ed7ca268509622ec006c0f0f25d90e34496638',
+            'uploader_url': 'http://thepullrequest.com',
+            'upload_date': '20220902',
+            'episode': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
+            'display_id': 'fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
+            'series': 'The Pull Request',
+            'channel_id': '7e9c23156e4aecfdcaef46bfb2ed7ca268509622ec006c0f0f25d90e34496638',
+            'view_count': int,
+            'uploader': 'Antonio García Martínez',
+            'thumbnail': 'https://d1z76fhpoqkd01.cloudfront.net/shows/legacy/1ade9142625344045dc17cf523469ced1d93610762f4c886d06aa190a2f979e8.png',
+            'episode_id': 'c3dab47f237bf953d180d3f243477a84302798be0e0b29bc9ade6d60a69f04f5',
+            'timestamp': 1662100688.005,
+        }
+    }, {
+        'url': 'https://www.callin.com/episode/episode-81-elites-melt-down-over-student-debt-lzxMidUnjA',
+        'md5': '16f704ddbf82a27e3930533b12062f07',
+        'info_dict': {
+            'id': '8d06f869798f93a7814e380bceabea72d501417e620180416ff6bd510596e83c',
+            'ext': 'ts',
+            'title': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
+            'description': 'Let’s talk todays episode about the primary election shake up in NYC and the elites melting down over student debt cancelation.',
+            'channel': 'The DEBRIEF With Briahna Joy Gray',
+            'channel_url': 'https://callin.com/show/the-debrief-with-briahna-joy-gray-siiFDzGegm',
+            'duration': 10043.16,
+            'series_id': '61cea58444465fd26674069703bd8322993bc9e5b4f1a6d0872690554a046ff7',
+            'uploader_url': 'http://patreon.com/badfaithpodcast',
+            'upload_date': '20220826',
+            'episode': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
+            'display_id': 'episode-',
+            'series': 'The DEBRIEF With Briahna Joy Gray',
+            'channel_id': '61cea58444465fd26674069703bd8322993bc9e5b4f1a6d0872690554a046ff7',
+            'view_count': int,
+            'uploader': 'Briahna Gray',
+            'thumbnail': 'https://d1z76fhpoqkd01.cloudfront.net/shows/legacy/461ea0d86172cb6aff7d6c80fd49259cf5e64bdf737a4650f8bc24cf392ca218.png',
+            'episode_id': '8d06f869798f93a7814e380bceabea72d501417e620180416ff6bd510596e83c',
+            'timestamp': 1661476708.282,
+        }
     }]
 
     def try_get_user_name(self, d):
@@ -86,6 +130,7 @@ def _real_extract(self, url):
 
         return {
             'id': id,
+            '_old_archive_ids': [make_archive_id(self, display_id.rsplit('-', 1)[-1])],
             'display_id': display_id,
             'title': title,
             'formats': formats,
diff --git a/yt_dlp/extractor/cammodels.py b/yt_dlp/extractor/cammodels.py
index 0509057fc..135b31529 100644
--- a/yt_dlp/extractor/cammodels.py
+++ b/yt_dlp/extractor/cammodels.py
@@ -1,9 +1,5 @@
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    url_or_none,
-)
+from ..utils import int_or_none, url_or_none
 
 
 class CamModelsIE(InfoExtractor):
@@ -17,32 +13,11 @@ class CamModelsIE(InfoExtractor):
     def _real_extract(self, url):
         user_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            url, user_id, headers=self.geo_verification_headers())
-
-        manifest_root = self._html_search_regex(
-            r'manifestUrlRoot=([^&\']+)', webpage, 'manifest', default=None)
-
-        if not manifest_root:
-            ERRORS = (
-                ("I'm offline, but let's stay connected", 'This user is currently offline'),
-                ('in a private show', 'This user is in a private show'),
-                ('is currently performing LIVE', 'This model is currently performing live'),
-            )
-            for pattern, message in ERRORS:
-                if pattern in webpage:
-                    error = message
-                    expected = True
-                    break
-            else:
-                error = 'Unable to find manifest URL root'
-                expected = False
-            raise ExtractorError(error, expected=expected)
-
         manifest = self._download_json(
-            '%s%s.json' % (manifest_root, user_id), user_id)
+            'https://manifest-server.naiadsystems.com/live/s:%s.json' % user_id, user_id)
 
         formats = []
+        thumbnails = []
         for format_id, format_dict in manifest['formats'].items():
             if not isinstance(format_dict, dict):
                 continue
@@ -82,12 +57,20 @@ def _real_extract(self, url):
                         'quality': -10,
                     })
                 else:
+                    if format_id == 'jpeg':
+                        thumbnails.append({
+                            'url': f['url'],
+                            'width': f['width'],
+                            'height': f['height'],
+                            'format_id': f['format_id'],
+                        })
                     continue
                 formats.append(f)
 
         return {
             'id': user_id,
             'title': user_id,
+            'thumbnails': thumbnails,
             'is_live': True,
             'formats': formats,
             'age_limit': 18
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index b7c687bc3..ebacc87bc 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -1338,7 +1338,7 @@ def _get_tfa_info(self, note='two-factor verification code'):
     # Helper functions for extracting OpenGraph info
     @staticmethod
     def _og_regexes(prop):
-        content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
+        content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?)(?=\s|/?>))'
         property_re = (r'(?:name|property)=(?:\'og%(sep)s%(prop)s\'|"og%(sep)s%(prop)s"|\s*og%(sep)s%(prop)s\b)'
                        % {'prop': re.escape(prop), 'sep': '(?:&#x3A;|[:-])'})
         template = r'<meta[^>]+?%s[^>]+?%s'
diff --git a/yt_dlp/extractor/ign.py b/yt_dlp/extractor/ign.py
index d4797d35e..e4db7f9fa 100644
--- a/yt_dlp/extractor/ign.py
+++ b/yt_dlp/extractor/ign.py
@@ -1,17 +1,20 @@
 import re
+import urllib.error
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_urllib_parse_urlparse,
-)
+from ..compat import compat_parse_qs
 from ..utils import (
-    HEADRequest,
+    ExtractorError,
     determine_ext,
+    error_to_compat_str,
+    extract_attributes,
     int_or_none,
+    merge_dicts,
     parse_iso8601,
     strip_or_none,
-    try_get,
+    traverse_obj,
+    url_or_none,
+    urljoin,
 )
 
 
@@ -20,14 +23,90 @@ def _call_api(self, slug):
         return self._download_json(
             'http://apis.ign.com/{0}/v3/{0}s/slug/{1}'.format(self._PAGE_TYPE, slug), slug)
 
+    def _checked_call_api(self, slug):
+        try:
+            return self._call_api(slug)
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+                e.cause.args = e.cause.args or [
+                    e.cause.geturl(), e.cause.getcode(), e.cause.reason]
+                raise ExtractorError(
+                    'Content not found: expired?', cause=e.cause,
+                    expected=True)
+            raise
+
+    def _extract_video_info(self, video, fatal=True):
+        video_id = video['videoId']
+
+        formats = []
+        refs = traverse_obj(video, 'refs', expected_type=dict) or {}
+
+        m3u8_url = url_or_none(refs.get('m3uUrl'))
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
+
+        f4m_url = url_or_none(refs.get('f4mUrl'))
+        if f4m_url:
+            formats.extend(self._extract_f4m_formats(
+                f4m_url, video_id, f4m_id='hds', fatal=False))
+
+        for asset in (video.get('assets') or []):
+            asset_url = url_or_none(asset.get('url'))
+            if not asset_url:
+                continue
+            formats.append({
+                'url': asset_url,
+                'tbr': int_or_none(asset.get('bitrate'), 1000),
+                'fps': int_or_none(asset.get('frame_rate')),
+                'height': int_or_none(asset.get('height')),
+                'width': int_or_none(asset.get('width')),
+            })
+
+        mezzanine_url = traverse_obj(
+            video, ('system', 'mezzanineUrl'), expected_type=url_or_none)
+        if mezzanine_url:
+            formats.append({
+                'ext': determine_ext(mezzanine_url, 'mp4'),
+                'format_id': 'mezzanine',
+                'quality': 1,
+                'url': mezzanine_url,
+            })
+
+        thumbnails = traverse_obj(
+            video, ('thumbnails', ..., {'url': 'url'}), expected_type=url_or_none)
+        tags = traverse_obj(
+            video, ('tags', ..., 'displayName'),
+            expected_type=lambda x: x.strip() or None)
+
+        metadata = traverse_obj(video, 'metadata', expected_type=dict) or {}
+        title = traverse_obj(
+            metadata, 'longTitle', 'title', 'name',
+            expected_type=lambda x: x.strip() or None)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': strip_or_none(metadata.get('description')),
+            'timestamp': parse_iso8601(metadata.get('publishDate')),
+            'duration': int_or_none(metadata.get('duration')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'tags': tags,
+        }
+
 
 class IGNIE(IGNBaseIE):
     """
     Extractor for some of the IGN sites, like www.ign.com, es.ign.com de.ign.com.
     Some videos of it.ign.com are also supported
     """
-
-    _VALID_URL = r'https?://(?:.+?\.ign|www\.pcmag)\.com/videos/(?:\d{4}/\d{2}/\d{2}/)?(?P<id>[^/?&#]+)'
+    _VIDEO_PATH_RE = r'/(?:\d{4}/\d{2}/\d{2}/)?(?P<id>.+?)'
+    _PLAYLIST_PATH_RE = r'(?:/?\?(?P<filt>[^&#]+))?'
+    _VALID_URL = (
+        r'https?://(?:.+?\.ign|www\.pcmag)\.com/videos(?:%s)'
+        % '|'.join((_VIDEO_PATH_RE + r'(?:[/?&#]|$)', _PLAYLIST_PATH_RE)))
     IE_NAME = 'ign.com'
     _PAGE_TYPE = 'video'
 
@@ -42,7 +121,13 @@ class IGNIE(IGNBaseIE):
             'timestamp': 1370440800,
             'upload_date': '20130605',
             'tags': 'count:9',
-        }
+            'display_id': 'the-last-of-us-review',
+            'thumbnail': 'https://assets1.ignimgs.com/vid/thumbnails/user/2014/03/26/lastofusreviewmimig2.jpg',
+            'duration': 440,
+        },
+        'params': {
+            'nocheckcertificate': True,
+        },
     }, {
         'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data',
         'md5': 'f1581a6fe8c5121be5b807684aeac3f6',
@@ -54,84 +139,48 @@ class IGNIE(IGNBaseIE):
             'timestamp': 1420571160,
             'upload_date': '20150106',
             'tags': 'count:4',
-        }
+        },
+        'skip': '404 Not Found',
     }, {
         'url': 'https://www.ign.com/videos/is-a-resident-evil-4-remake-on-the-way-ign-daily-fix',
         'only_matching': True,
     }]
 
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        grids = re.findall(
+            r'''(?s)<section\b[^>]+\bclass\s*=\s*['"](?:[\w-]+\s+)*?content-feed-grid(?!\B|-)[^>]+>(.+?)</section[^>]*>''',
+            webpage)
+        return filter(None,
+                      (urljoin(url, m.group('path')) for m in re.finditer(
+                          r'''<a\b[^>]+\bhref\s*=\s*('|")(?P<path>/videos%s)\1'''
+                          % cls._VIDEO_PATH_RE, grids[0] if grids else '')))
+
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        video = self._call_api(display_id)
-        video_id = video['videoId']
-        metadata = video['metadata']
-        title = metadata.get('longTitle') or metadata.get('title') or metadata['name']
+        display_id, filt = self._match_valid_url(url).group('id', 'filt')
+        if display_id:
+            return self._extract_video(url, display_id)
+        return self._extract_playlist(url, filt or 'all')
 
-        formats = []
-        refs = video.get('refs') or {}
+    def _extract_playlist(self, url, display_id):
+        webpage = self._download_webpage(url, display_id)
 
-        m3u8_url = refs.get('m3uUrl')
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native',
-                m3u8_id='hls', fatal=False))
+        return self.playlist_result(
+            (self.url_result(u, self.ie_key())
+             for u in self._extract_embed_urls(url, webpage)),
+            playlist_id=display_id)
 
-        f4m_url = refs.get('f4mUrl')
-        if f4m_url:
-            formats.extend(self._extract_f4m_formats(
-                f4m_url, video_id, f4m_id='hds', fatal=False))
+    def _extract_video(self, url, display_id):
+        video = self._checked_call_api(display_id)
 
-        for asset in (video.get('assets') or []):
-            asset_url = asset.get('url')
-            if not asset_url:
-                continue
-            formats.append({
-                'url': asset_url,
-                'tbr': int_or_none(asset.get('bitrate'), 1000),
-                'fps': int_or_none(asset.get('frame_rate')),
-                'height': int_or_none(asset.get('height')),
-                'width': int_or_none(asset.get('width')),
-            })
+        info = self._extract_video_info(video)
 
-        mezzanine_url = try_get(video, lambda x: x['system']['mezzanineUrl'])
-        if mezzanine_url:
-            formats.append({
-                'ext': determine_ext(mezzanine_url, 'mp4'),
-                'format_id': 'mezzanine',
-                'quality': 1,
-                'url': mezzanine_url,
-            })
-
-        thumbnails = []
-        for thumbnail in (video.get('thumbnails') or []):
-            thumbnail_url = thumbnail.get('url')
-            if not thumbnail_url:
-                continue
-            thumbnails.append({
-                'url': thumbnail_url,
-            })
-
-        tags = []
-        for tag in (video.get('tags') or []):
-            display_name = tag.get('displayName')
-            if not display_name:
-                continue
-            tags.append(display_name)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': strip_or_none(metadata.get('description')),
-            'timestamp': parse_iso8601(metadata.get('publishDate')),
-            'duration': int_or_none(metadata.get('duration')),
+        return merge_dicts({
             'display_id': display_id,
-            'thumbnails': thumbnails,
-            'formats': formats,
-            'tags': tags,
-        }
+        }, info)
 
 
-class IGNVideoIE(InfoExtractor):
+class IGNVideoIE(IGNBaseIE):
     _VALID_URL = r'https?://.+?\.ign\.com/(?:[a-z]{2}/)?[^/]+/(?P<id>\d+)/(?:video|trailer)/'
     _TESTS = [{
         'url': 'http://me.ign.com/en/videos/112203/video/how-hitman-aims-to-be-different-than-every-other-s',
@@ -143,7 +192,16 @@ class IGNVideoIE(InfoExtractor):
             'description': 'Taking out assassination targets in Hitman has never been more stylish.',
             'timestamp': 1444665600,
             'upload_date': '20151012',
-        }
+            'display_id': '112203',
+            'thumbnail': 'https://sm.ign.com/ign_me/video/h/how-hitman/how-hitman-aims-to-be-different-than-every-other-s_8z14.jpg',
+            'duration': 298,
+            'tags': 'count:13',
+            'display_id': '112203',
+            'thumbnail': 'https://sm.ign.com/ign_me/video/h/how-hitman/how-hitman-aims-to-be-different-than-every-other-s_8z14.jpg',
+            'duration': 298,
+            'tags': 'count:13',
+        },
+        'expected_warnings': ['HTTP Error 400: Bad Request'],
     }, {
         'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds',
         'only_matching': True,
@@ -163,22 +221,38 @@ class IGNVideoIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        req = HEADRequest(url.rsplit('/', 1)[0] + '/embed')
-        url = self._request_webpage(req, video_id).geturl()
+        parsed_url = urllib.parse.urlparse(url)
+        embed_url = urllib.parse.urlunparse(
+            parsed_url._replace(path=parsed_url.path.rsplit('/', 1)[0] + '/embed'))
+
+        webpage, urlh = self._download_webpage_handle(embed_url, video_id)
+        new_url = urlh.geturl()
         ign_url = compat_parse_qs(
-            compat_urllib_parse_urlparse(url).query).get('url', [None])[0]
+            urllib.parse.urlparse(new_url).query).get('url', [None])[-1]
         if ign_url:
             return self.url_result(ign_url, IGNIE.ie_key())
-        return self.url_result(url)
+        video = self._search_regex(r'(<div\b[^>]+\bdata-video-id\s*=\s*[^>]+>)', webpage, 'video element', fatal=False)
+        if not video:
+            if new_url == url:
+                raise ExtractorError('Redirect loop: ' + url)
+            return self.url_result(new_url)
+        video = extract_attributes(video)
+        video_data = video.get('data-settings') or '{}'
+        video_data = self._parse_json(video_data, video_id)['video']
+        info = self._extract_video_info(video_data)
+
+        return merge_dicts({
+            'display_id': video_id,
+        }, info)
 
 
 class IGNArticleIE(IGNBaseIE):
-    _VALID_URL = r'https?://.+?\.ign\.com/(?:articles(?:/\d{4}/\d{2}/\d{2})?|(?:[a-z]{2}/)?feature/\d+)/(?P<id>[^/?&#]+)'
+    _VALID_URL = r'https?://.+?\.ign\.com/(?:articles(?:/\d{4}/\d{2}/\d{2})?|(?:[a-z]{2}/)?(?:[\w-]+/)*?feature/\d+)/(?P<id>[^/?&#]+)'
     _PAGE_TYPE = 'article'
     _TESTS = [{
         'url': 'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
         'info_dict': {
-            'id': '524497489e4e8ff5848ece34',
+            'id': '72113',
             'title': '100 Little Things in GTA 5 That Will Blow Your Mind',
         },
         'playlist': [
@@ -186,34 +260,43 @@ class IGNArticleIE(IGNBaseIE):
                 'info_dict': {
                     'id': '5ebbd138523268b93c9141af17bec937',
                     'ext': 'mp4',
-                    'title': 'GTA 5 Video Review',
+                    'title': 'Grand Theft Auto V Video Review',
                     'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
                     'timestamp': 1379339880,
                     'upload_date': '20130916',
+                    'tags': 'count:12',
+                    'thumbnail': 'https://assets1.ignimgs.com/thumbs/userUploaded/2021/8/16/gta-v-heistsjpg-e94705-1629138553533.jpeg',
+                    'display_id': 'grand-theft-auto-v-video-review',
+                    'duration': 501,
                 },
             },
             {
                 'info_dict': {
                     'id': '638672ee848ae4ff108df2a296418ee2',
                     'ext': 'mp4',
-                    'title': '26 Twisted Moments from GTA 5 in Slow Motion',
+                    'title': 'GTA 5 In Slow Motion',
                     'description': 'The twisted beauty of GTA 5 in stunning slow motion.',
                     'timestamp': 1386878820,
                     'upload_date': '20131212',
+                    'duration': 202,
+                    'tags': 'count:25',
+                    'display_id': 'gta-5-in-slow-motion',
+                    'thumbnail': 'https://assets1.ignimgs.com/vid/thumbnails/user/2013/11/03/GTA-SLO-MO-1.jpg',
                 },
             },
         ],
         'params': {
-            'playlist_items': '2-3',
             'skip_download': True,
         },
+        'expected_warnings': ['Backend fetch failed'],
     }, {
         'url': 'http://www.ign.com/articles/2014/08/15/rewind-theater-wild-trailer-gamescom-2014?watch',
         'info_dict': {
             'id': '53ee806780a81ec46e0790f8',
             'title': 'Rewind Theater - Wild Trailer Gamescom 2014',
         },
-        'playlist_count': 2,
+        'playlist_count': 1,
+        'expected_warnings': ['Backend fetch failed'],
     }, {
         # videoId pattern
         'url': 'http://www.ign.com/articles/2017/06/08/new-ducktales-short-donalds-birthday-doesnt-go-as-planned',
@@ -236,18 +319,84 @@ class IGNArticleIE(IGNBaseIE):
         'only_matching': True,
     }]
 
+    def _checked_call_api(self, slug):
+        try:
+            return self._call_api(slug)
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError):
+                e.cause.args = e.cause.args or [
+                    e.cause.geturl(), e.cause.getcode(), e.cause.reason]
+                if e.cause.code == 404:
+                    raise ExtractorError(
+                        'Content not found: expired?', cause=e.cause,
+                        expected=True)
+                elif e.cause.code == 503:
+                    self.report_warning(error_to_compat_str(e.cause))
+                    return
+            raise
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        article = self._call_api(display_id)
+        article = self._checked_call_api(display_id)
 
-        def entries():
-            media_url = try_get(article, lambda x: x['mediaRelations'][0]['media']['metadata']['url'])
-            if media_url:
-                yield self.url_result(media_url, IGNIE.ie_key())
-            for content in (article.get('content') or []):
-                for video_url in re.findall(r'(?:\[(?:ignvideo\s+url|youtube\s+clip_id)|<iframe[^>]+src)="([^"]+)"', content):
-                    yield self.url_result(video_url)
+        if article:
+            # obsolete ?
+            def entries():
+                media_url = traverse_obj(
+                    article, ('mediaRelations', 0, 'media', 'metadata', 'url'),
+                    expected_type=url_or_none)
+                if media_url:
+                    yield self.url_result(media_url, IGNIE.ie_key())
+                for content in (article.get('content') or []):
+                    for video_url in re.findall(r'(?:\[(?:ignvideo\s+url|youtube\s+clip_id)|<iframe[^>]+src)="([^"]+)"', content):
+                        if url_or_none(video_url):
+                            yield self.url_result(video_url)
+
+            return self.playlist_result(
+                entries(), article.get('articleId'),
+                traverse_obj(
+                    article, ('metadata', 'headline'),
+                    expected_type=lambda x: x.strip() or None))
+
+        webpage = self._download_webpage(url, display_id)
+
+        playlist_id = self._html_search_meta('dable:item_id', webpage, default=None)
+        if playlist_id:
+
+            def entries():
+                for m in re.finditer(
+                        r'''(?s)<object\b[^>]+\bclass\s*=\s*("|')ign-videoplayer\1[^>]*>(?P<params>.+?)</object''',
+                        webpage):
+                    flashvars = self._search_regex(
+                        r'''(<param\b[^>]+\bname\s*=\s*("|')flashvars\2[^>]*>)''',
+                        m.group('params'), 'flashvars', default='')
+                    flashvars = compat_parse_qs(extract_attributes(flashvars).get('value') or '')
+                    v_url = url_or_none((flashvars.get('url') or [None])[-1])
+                    if v_url:
+                        yield self.url_result(v_url)
+        else:
+            playlist_id = self._search_regex(
+                r'''\bdata-post-id\s*=\s*("|')(?P<id>[\da-f]+)\1''',
+                webpage, 'id', group='id', default=None)
+
+            nextjs_data = self._search_nextjs_data(webpage, display_id)
+
+            def entries():
+                for player in traverse_obj(
+                        nextjs_data,
+                        ('props', 'apolloState', 'ROOT_QUERY', lambda k, _: k.startswith('videoPlayerProps('), '__ref')):
+                    # skip promo links (which may not always be served, eg GH CI servers)
+                    if traverse_obj(nextjs_data,
+                                    ('props', 'apolloState', player.replace('PlayerProps', 'ModernContent')),
+                                    expected_type=dict):
+                        continue
+                    video = traverse_obj(nextjs_data, ('props', 'apolloState', player), expected_type=dict) or {}
+                    info = self._extract_video_info(video, fatal=False)
+                    if info:
+                        yield merge_dicts({
+                            'display_id': display_id,
+                        }, info)
 
         return self.playlist_result(
-            entries(), article.get('articleId'),
-            strip_or_none(try_get(article, lambda x: x['metadata']['headline'])))
+            entries(), playlist_id or display_id,
+            re.sub(r'\s+-\s+IGN\s*$', '', self._og_search_title(webpage, default='')) or None)
diff --git a/yt_dlp/extractor/kommunetv.py b/yt_dlp/extractor/kommunetv.py
new file mode 100644
index 000000000..e21e556be
--- /dev/null
+++ b/yt_dlp/extractor/kommunetv.py
@@ -0,0 +1,31 @@
+from .common import InfoExtractor
+from ..utils import update_url
+
+
+class KommunetvIE(InfoExtractor):
+    _VALID_URL = r'https://(\w+).kommunetv.no/archive/(?P<id>\w+)'
+    _TEST = {
+        'url': 'https://oslo.kommunetv.no/archive/921',
+        'md5': '5f102be308ee759be1e12b63d5da4bbc',
+        'info_dict': {
+            'id': '921',
+            'title': 'Bystyremøte',
+            'ext': 'mp4'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        headers = {
+            'Accept': 'application/json'
+        }
+        data = self._download_json('https://oslo.kommunetv.no/api/streams?streamType=1&id=%s' % video_id, video_id, headers=headers)
+        title = data['stream']['title']
+        file = data['playlist'][0]['playlist'][0]['file']
+        url = update_url(file, query=None, fragment=None)
+        formats = self._extract_m3u8_formats(url, video_id, ext='mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title
+        }
diff --git a/yt_dlp/extractor/myvideoge.py b/yt_dlp/extractor/myvideoge.py
index 513d4cb77..64cee48e7 100644
--- a/yt_dlp/extractor/myvideoge.py
+++ b/yt_dlp/extractor/myvideoge.py
@@ -1,5 +1,16 @@
+import re
+
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    MONTH_NAMES,
+    clean_html,
+    get_element_by_class,
+    get_element_by_id,
+    int_or_none,
+    js_to_json,
+    qualities,
+    unified_strdate,
+)
 
 
 class MyVideoGeIE(InfoExtractor):
@@ -11,37 +22,50 @@ class MyVideoGeIE(InfoExtractor):
             'id': '3941048',
             'ext': 'mp4',
             'title': 'The best prikol',
+            'upload_date': '20200611',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'md5:d72addd357b0dd914e704781f7f777d8',
-            'description': 'md5:5c0371f540f5888d603ebfedd46b6df3'
-        }
+            'uploader': 'chixa33',
+            'description': 'md5:5b067801318e33c2e6eea4ab90b1fdd3',
+        },
     }
+    _MONTH_NAMES_KA = ['იანვარი', 'თებერვალი', 'მარტი', 'აპრილი', 'მაისი', 'ივნისი', 'ივლისი', 'აგვისტო', 'სექტემბერი', 'ოქტომბერი', 'ნოემბერი', 'დეკემბერი']
+
+    _quality = staticmethod(qualities(('SD', 'HD')))
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h1[^>]*>([^<]+)</h1>', webpage, 'title')
-        description = self._og_search_description(webpage)
-        thumbnail = self._html_search_meta(['og:image'], webpage)
-        uploader = self._search_regex(r'<a[^>]+class="mv_user_name"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False)
+        title = (
+            self._og_search_title(webpage, default=None)
+            or clean_html(get_element_by_class('my_video_title', webpage))
+            or self._html_search_regex(r'<title\b[^>]*>([^<]+)</title\b', webpage, 'title'))
 
         jwplayer_sources = self._parse_json(
             self._search_regex(
-                r"(?s)jwplayer\(\"mvplayer\"\).setup\(.*?sources: (.*?])", webpage, 'jwplayer sources'),
-            video_id, transform_source=js_to_json)
+                r'''(?s)jwplayer\s*\(\s*['"]mvplayer['"]\s*\)\s*\.\s*setup\s*\(.*?\bsources\s*:\s*(\[.*?])\s*[,});]''', webpage, 'jwplayer sources', fatal=False)
+            or '',
+            video_id, transform_source=js_to_json, fatal=False)
 
-        def _formats_key(f):
-            if f['label'] == 'SD':
-                return -1
-            elif f['label'] == 'HD':
-                return 1
-            else:
-                return 0
+        formats = self._parse_jwplayer_formats(jwplayer_sources or [], video_id)
+        for f in formats or []:
+            f['quality'] = self._quality(f['format_id'])
 
-        jwplayer_sources = sorted(jwplayer_sources, key=_formats_key)
+        description = (
+            self._og_search_description(webpage)
+            or get_element_by_id('long_desc_holder', webpage)
+            or self._html_search_meta('description', webpage))
 
-        formats = self._parse_jwplayer_formats(jwplayer_sources, video_id)
+        uploader = self._search_regex(r'<a[^>]+class="mv_user_name"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False)
+
+        upload_date = get_element_by_class('mv_vid_upl_date', webpage)
+        # as ka locale may not be present roll a local date conversion
+        upload_date = (unified_strdate(
+            # translate any ka month to an en one
+            re.sub('|'.join(self._MONTH_NAMES_KA),
+                   lambda m: MONTH_NAMES['en'][self._MONTH_NAMES_KA.index(m.group(0))],
+                   upload_date, re.I))
+            if upload_date else None)
 
         return {
             'id': video_id,
@@ -49,5 +73,9 @@ def _formats_key(f):
             'description': description,
             'uploader': uploader,
             'formats': formats,
-            'thumbnail': thumbnail
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'upload_date': upload_date,
+            'view_count': int_or_none(get_element_by_class('mv_vid_views', webpage)),
+            'like_count': int_or_none(get_element_by_id('likes_count', webpage)),
+            'dislike_count': int_or_none(get_element_by_id('dislikes_count', webpage)),
         }
diff --git a/yt_dlp/extractor/pr0gramm.py b/yt_dlp/extractor/pr0gramm.py
new file mode 100644
index 000000000..2eb327fba
--- /dev/null
+++ b/yt_dlp/extractor/pr0gramm.py
@@ -0,0 +1,97 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import merge_dicts
+
+
+class Pr0grammStaticIE(InfoExtractor):
+    # Possible urls:
+    # https://pr0gramm.com/static/5466437
+    _VALID_URL = r'https?://pr0gramm\.com/static/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://pr0gramm.com/static/5466437',
+        'md5': '52fa540d70d3edc286846f8ca85938aa',
+        'info_dict': {
+            'id': '5466437',
+            'ext': 'mp4',
+            'title': 'pr0gramm-5466437 by g11st',
+            'uploader': 'g11st',
+            'upload_date': '20221221',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        # Fetch media sources
+        entries = self._parse_html5_media_entries(url, webpage, video_id)
+        media_info = entries[0]
+
+        # Fetch author
+        uploader = self._html_search_regex(r'by\W+([\w-]+)\W+', webpage, 'uploader')
+
+        # Fetch approx upload timestamp from filename
+        # Have None-defaults in case the extraction fails
+        uploadDay = None
+        uploadMon = None
+        uploadYear = None
+        uploadTimestr = None
+        # (//img.pr0gramm.com/2022/12/21/62ae8aa5e2da0ebf.mp4)
+        m = re.search(r'//img\.pr0gramm\.com/(?P<year>[\d]+)/(?P<mon>[\d]+)/(?P<day>[\d]+)/\w+\.\w{,4}', webpage)
+
+        if (m):
+            # Up to a day of accuracy should suffice...
+            uploadDay = m.groupdict().get('day')
+            uploadMon = m.groupdict().get('mon')
+            uploadYear = m.groupdict().get('year')
+            uploadTimestr = uploadYear + uploadMon + uploadDay
+
+        return merge_dicts({
+            'id': video_id,
+            'title': 'pr0gramm-%s%s' % (video_id, (' by ' + uploader) if uploader else ''),
+            'uploader': uploader,
+            'upload_date': uploadTimestr
+        }, media_info)
+
+
+# This extractor is for the primary url (used for sharing, and appears in the
+# location bar) Since this page loads the DOM via JS, yt-dl can't find any
+# video information here. So let's redirect to a compatibility version of
+# the site, which does contain the <video>-element  by itself,  without requiring
+# js to be ran.
+class Pr0grammIE(InfoExtractor):
+    # Possible urls:
+    # https://pr0gramm.com/new/546637
+    # https://pr0gramm.com/new/video/546637
+    # https://pr0gramm.com/top/546637
+    # https://pr0gramm.com/top/video/546637
+    # https://pr0gramm.com/user/g11st/uploads/5466437
+    # https://pr0gramm.com/user/froschler/dafur-ist-man-hier/5091290
+    # https://pr0gramm.com/user/froschler/reinziehen-1elf/5232030
+    # https://pr0gramm.com/user/froschler/1elf/5232030
+    # https://pr0gramm.com/new/5495710:comment62621020 <- this is not the id!
+    # https://pr0gramm.com/top/fruher war alles damals/5498175
+
+    _VALID_URL = r'https?:\/\/pr0gramm\.com\/(?!static/\d+).+?\/(?P<id>[\d]+)(:|$)'
+    _TEST = {
+        'url': 'https://pr0gramm.com/new/video/5466437',
+        'info_dict': {
+            'id': '5466437',
+            'ext': 'mp4',
+            'title': 'pr0gramm-5466437 by g11st',
+            'uploader': 'g11st',
+            'upload_date': '20221221',
+        }
+    }
+
+    def _generic_title():
+        return "oof"
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        return self.url_result(
+            'https://pr0gramm.com/static/' + video_id,
+            video_id=video_id,
+            ie=Pr0grammStaticIE.ie_key())
diff --git a/yt_dlp/extractor/rbgtum.py b/yt_dlp/extractor/rbgtum.py
new file mode 100644
index 000000000..47649cfc5
--- /dev/null
+++ b/yt_dlp/extractor/rbgtum.py
@@ -0,0 +1,93 @@
+import re
+
+from .common import InfoExtractor
+
+
+class RbgTumIE(InfoExtractor):
+    _VALID_URL = r'https://live\.rbg\.tum\.de/w/(?P<id>.+)'
+    _TESTS = [{
+        # Combined view
+        'url': 'https://live.rbg.tum.de/w/cpp/22128',
+        'md5': '53a5e7b3e07128e33bbf36687fe1c08f',
+        'info_dict': {
+            'id': 'cpp/22128',
+            'ext': 'mp4',
+            'title': 'Lecture: October 18. 2022',
+            'series': 'Concepts of C++ programming (IN2377)',
+        }
+    }, {
+        # Presentation only
+        'url': 'https://live.rbg.tum.de/w/I2DL/12349/PRES',
+        'md5': '36c584272179f3e56b0db5d880639cba',
+        'info_dict': {
+            'id': 'I2DL/12349/PRES',
+            'ext': 'mp4',
+            'title': 'Lecture 3: Introduction to Neural Networks',
+            'series': 'Introduction to Deep Learning (IN2346)',
+        }
+    }, {
+        # Camera only
+        'url': 'https://live.rbg.tum.de/w/fvv-info/16130/CAM',
+        'md5': 'e04189d92ff2f56aedf5cede65d37aad',
+        'info_dict': {
+            'id': 'fvv-info/16130/CAM',
+            'ext': 'mp4',
+            'title': 'Fachschaftsvollversammlung',
+            'series': 'Fachschaftsvollversammlung Informatik',
+        }
+    }, ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        m3u8 = self._html_search_regex(r'(https://.+?\.m3u8)', webpage, 'm3u8')
+        lecture_title = self._html_search_regex(r'(?si)<h1.*?>(.*)</h1>', webpage, 'title')
+        lecture_series_title = self._html_search_regex(
+            r'(?s)<title\b[^>]*>\s*(?:TUM-Live\s\|\s?)?([^:]+):?.*?</title>', webpage, 'series')
+
+        formats = self._extract_m3u8_formats(m3u8, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'title': lecture_title,
+            'series': lecture_series_title,
+            'formats': formats,
+        }
+
+
+class RbgTumCourseIE(InfoExtractor):
+    _VALID_URL = r'https://live\.rbg\.tum\.de/course/(?P<id>.+)'
+    _TESTS = [{
+        'url': 'https://live.rbg.tum.de/course/2022/S/fpv',
+        'info_dict': {
+            'title': 'Funktionale Programmierung und Verifikation (IN0003)',
+            'id': '2022/S/fpv',
+        },
+        'params': {
+            'noplaylist': False,
+        },
+        'playlist_count': 13,
+    }, {
+        'url': 'https://live.rbg.tum.de/course/2022/W/set',
+        'info_dict': {
+            'title': 'SET FSMPIC',
+            'id': '2022/W/set',
+        },
+        'params': {
+            'noplaylist': False,
+        },
+        'playlist_count': 6,
+    }, ]
+
+    def _real_extract(self, url):
+        course_id = self._match_id(url)
+        webpage = self._download_webpage(url, course_id)
+
+        lecture_series_title = self._html_search_regex(r'(?si)<h1.*?>(.*)</h1>', webpage, 'title')
+
+        lecture_urls = []
+        for lecture_url in re.findall(r'(?i)href="/w/(.+)(?<!/cam)(?<!/pres)(?<!/chat)"', webpage):
+            lecture_urls.append(self.url_result('https://live.rbg.tum.de/w/' + lecture_url, ie=RbgTumIE.ie_key()))
+
+        return self.playlist_result(lecture_urls, course_id, lecture_series_title)
diff --git a/yt_dlp/extractor/unsupported.py b/yt_dlp/extractor/unsupported.py
index b9cb31beb..a56bd284f 100644
--- a/yt_dlp/extractor/unsupported.py
+++ b/yt_dlp/extractor/unsupported.py
@@ -130,6 +130,9 @@ class KnownPiracyIE(UnsupportedInfoExtractor):
 
     URLS = (
         r'dood\.(?:to|watch|so|pm|wf|re)',
+        # Sites youtube-dl supports, but we won't
+        r'https://viewsb\.com',
+        r'https://filemoon\.sx',
     )
 
     _TESTS = [{
diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py
index 88d143e71..d81d9c551 100644
--- a/yt_dlp/extractor/vimeo.py
+++ b/yt_dlp/extractor/vimeo.py
@@ -304,27 +304,33 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
     # _VALID_URL matches Vimeo URLs
     _VALID_URL = r'''(?x)
-                    https?://
-                        (?:
-                            (?:
-                                www|
-                                player
-                            )
-                            \.
-                        )?
-                        vimeo\.com/
-                        (?!(?:channels|album|showcase)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
-                        (?:[^/]+/)*?
-                        (?:
-                            (?:
-                                play_redirect_hls|
-                                moogaloop\.swf)\?clip_id=
-                            )?
-                        (?:videos?/)?
-                        (?P<id>[0-9]+)
-                        (?:/(?P<unlisted_hash>[\da-f]{10}))?
-                        /?(?:[?&].*)?(?:[#].*)?$
-                    '''
+                     https?://
+                         (?:
+                             (?:
+                                 www|
+                                 player
+                             )
+                             \.
+                         )?
+                         vimeo\.com/
+                         (?:
+                             (?P<u>user)|
+                             (?!(?:channels|album|showcase)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
+                             (?:.*?/)??
+                             (?P<q>
+                                 (?:
+                                     play_redirect_hls|
+                                     moogaloop\.swf)\?clip_id=
+                             )?
+                             (?:videos?/)?
+                         )
+                         (?P<id>[0-9]+)
+                         (?(u)
+                             /(?!videos|likes)[^/?#]+/?|
+                             (?(q)|/(?P<unlisted_hash>[\da-f]{10}))?
+                         )
+                         (?:(?(q)[&]|(?(u)|/?)[?]).*?)?(?:[#].*)?$
+                 '''
     IE_NAME = 'vimeo'
     _EMBED_REGEX = [
         # iframe
@@ -705,7 +711,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'params': {
                 'skip_download': True,
             },
-        }
+        },
+        {
+            # user playlist alias -> https://vimeo.com/258705797
+            'url': 'https://vimeo.com/user26785108/newspiritualguide',
+            'only_matching': True,
+        },
         # https://gettingthingsdone.com/workflowmap/
         # vimeo embed with check-password page protected by Referer header
     ]
diff --git a/yt_dlp/extractor/xhamster.py b/yt_dlp/extractor/xhamster.py
index 59eececb6..7af6c8f03 100644
--- a/yt_dlp/extractor/xhamster.py
+++ b/yt_dlp/extractor/xhamster.py
@@ -21,7 +21,7 @@
 
 
 class XHamsterIE(InfoExtractor):
-    _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster\d+\.com|xhday\.com)'
+    _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster\d+\.com|xhday\.com|xhvid\.com)'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:.+?\.)?%s/
@@ -120,6 +120,9 @@ class XHamsterIE(InfoExtractor):
     }, {
         'url': 'https://xhday.com/videos/strapless-threesome-xhh7yVf',
         'only_matching': True,
+    }, {
+        'url': 'https://xhvid.com/videos/lk-mm-xhc6wn6',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -422,6 +425,9 @@ class XHamsterUserIE(InfoExtractor):
     }, {
         'url': 'https://xhday.com/users/mobhunter',
         'only_matching': True,
+    }, {
+        'url': 'https://xhvid.com/users/pelushe21',
+        'only_matching': True,
     }]
 
     def _entries(self, user_id):
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 9eb9495a0..994239897 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3149,14 +3149,28 @@ def urlencode_postdata(*args, **kargs):
     return urllib.parse.urlencode(*args, **kargs).encode('ascii')
 
 
+def update_url(url, *, query_update=None, **kwargs):
+    """Replace URL components specified by kwargs
+       @param url           str or parse url tuple
+       @param query_update  update query
+       @returns             str
+    """
+    if isinstance(url, str):
+        if not kwargs and not query_update:
+            return url
+        else:
+            url = urllib.parse.urlparse(url)
+    if query_update:
+        assert 'query' not in kwargs, 'query_update and query cannot be specified at the same time'
+        kwargs['query'] = urllib.parse.urlencode({
+            **urllib.parse.parse_qs(url.query),
+            **query_update
+        }, True)
+    return urllib.parse.urlunparse(url._replace(**kwargs))
+
+
 def update_url_query(url, query):
-    if not query:
-        return url
-    parsed_url = urllib.parse.urlparse(url)
-    qs = urllib.parse.parse_qs(parsed_url.query)
-    qs.update(query)
-    return urllib.parse.urlunparse(parsed_url._replace(
-        query=urllib.parse.urlencode(qs, True)))
+    return update_url(url, query_update=query)
 
 
 def update_Request(req, url=None, data=None, headers=None, query=None):

From a0a7c0154252900b7b154898744b698624d92b2a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 17 Feb 2023 17:22:03 +0530
Subject: [PATCH 086/871] Release 2023.02.17

---
 CONTRIBUTORS      |  24 +++++
 Changelog.md      | 253 ++++++++++++++++++++++++++++++++++++++++++++++
 Collaborators.md  |   8 ++
 supportedsites.md |  33 ++++--
 4 files changed, 312 insertions(+), 6 deletions(-)

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index e3b95e2f3..10fb5775b 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -381,3 +381,27 @@ gschizas
 JC-Chung
 mzhou
 OndrejBakan
+ab4cbef
+aionescu
+amra
+ByteDream
+carusocr
+chexxor
+felixonmars
+FrankZ85
+FriedrichRehren
+gregsadetsky
+LeoniePhiline
+LowSuggestion912
+Matumo
+OIRNOIR
+OMEGARAZER
+oxamun
+pmitchell86
+qbnu
+qulaz
+rebane2001
+road-master
+rohieb
+sdht0
+seproDev
diff --git a/Changelog.md b/Changelog.md
index e4cc7fd30..36856e016 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -10,6 +10,259 @@ # Instuctions for creating release
 * Dispatch the workflow https://github.com/yt-dlp/yt-dlp/actions/workflows/build.yml on master
 -->
 
+# 2023.02.17
+
+* Merge youtube-dl: Upto [commit/2dd6c6e](https://github.com/ytdl-org/youtube-dl/commit/2dd6c6e)
+* Fix `--concat-playlist`
+* Imply `--no-progress` when `--print`
+* Improve default subtitle language selection by [sdht0](https://github.com/sdht0)
+* Make `title` completely non-fatal
+* Sanitize formats before sorting by [pukkandan](https://github.com/pukkandan)
+* Support module level `__bool__` and `property`
+* [dependencies] Standardize `Cryptodome` imports
+* [hls] Allow extractors to provide AES key by [Grub4K](https://github.com/Grub4K), [bashonly](https://github.com/bashonly)
+* [ExtractAudio] Handle outtmpl without ext by [carusocr](https://github.com/carusocr)
+* [extractor/common] Fix `_search_nuxt_data` by [LowSuggestion912](https://github.com/LowSuggestion912)
+* [extractor/generic] Avoid catastrophic backtracking in KVS regex by [bashonly](https://github.com/bashonly)
+* [jsinterp] Support `if` statements
+* [plugins] Fix zip search paths
+* [utils] `traverse_obj`:  Various improvements by [Grub4K](https://github.com/Grub4K)
+* [utils] `traverse_obj`: Fix more bugs
+* [utils] `traverse_obj`: Fix several behavioral problems by [Grub4K](https://github.com/Grub4K)
+* [utils] Don't use Content-length with encoding by [felixonmars](https://github.com/felixonmars)
+* [utils] Fix `time_seconds` to use the provided TZ by [Grub4K](https://github.com/Grub4K), [Lesmiscore](https://github.com/Lesmiscore)
+* [utils] Fix race condition in `make_dir` by [aionescu](https://github.com/aionescu)
+* [utils] Use local kernel32 for file locking on Windows by [Grub4K](https://github.com/Grub4K)
+* [compat_utils] Improve `passthrough_module`
+* [compat_utils] Simplify `EnhancedModule`
+* [build] Update pyinstaller
+* [pyinst] Fix for pyinstaller 5.8
+* [devscripts] Provide `pyinstaller` hooks
+* [devscripts/pyinstaller] Analyze sub-modules of `Cryptodome`
+* [cleanup] Misc fixes and cleanup
+* [extractor/anchorfm] Add episode extractor by [HobbyistDev](https://github.com/HobbyistDev), [bashonly](https://github.com/bashonly)
+* [extractor/boxcast] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/ebay] Add extractor by [JChris246](https://github.com/JChris246)
+* [extractor/hypergryph] Add extractor by [HobbyistDev](https://github.com/HobbyistDev), [bashonly](https://github.com/bashonly)
+* [extractor/NZOnScreen] Add extractor by [gregsadetsky](https://github.com/gregsadetsky), [pukkandan](https://github.com/pukkandan)
+* [extractor/rozhlas] Add extractor RozhlasVltavaIE by [amra](https://github.com/amra)
+* [extractor/tempo] Add IVXPlayer extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/txxx] Add extractors by [chio0hai](https://github.com/chio0hai)
+* [extractor/vocaroo] Add extractor by [SuperSonicHub1](https://github.com/SuperSonicHub1), [qbnu](https://github.com/qbnu)
+* [extractor/wrestleuniverse] Add extractors by [Grub4K](https://github.com/Grub4K), [bashonly](https://github.com/bashonly)
+* [extractor/yappy] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* **[extractor/youtube] Fix `uploader_id` extraction** by [bashonly](https://github.com/bashonly)
+* [extractor/youtube] Add hyperpipe instances by [Generator](https://github.com/Generator)
+* [extractor/youtube] Handle `consent.youtube`
+* [extractor/youtube] Support `/live/` URL
+* [extractor/youtube] Update invidious and piped instances by [rohieb](https://github.com/rohieb)
+* [extractor/91porn] Fix title and comment extraction by [pmitchell86](https://github.com/pmitchell86)
+* [extractor/AbemaTV] Cache user token whenever appropriate by [Lesmiscore](https://github.com/Lesmiscore)
+* [extractor/bfmtv] Support `rmc` prefix by [carusocr](https://github.com/carusocr)
+* [extractor/biliintl] Add intro and ending chapters by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/clyp] Support `wav` by [qulaz](https://github.com/qulaz)
+* [extractor/crunchyroll] Add intro chapter by [ByteDream](https://github.com/ByteDream)
+* [extractor/crunchyroll] Better message for premium videos
+* [extractor/crunchyroll] Fix incorrect premium-only error by [Grub4K](https://github.com/Grub4K)
+* [extractor/DouyuTV] Use new API by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [extractor/embedly] Embedded links may be for other extractors
+* [extractor/freesound] Workaround invalid URL in webpage by [rebane2001](https://github.com/rebane2001)
+* [extractor/GoPlay] Use new API by [jeroenj](https://github.com/jeroenj)
+* [extractor/Hidive] Fix subtitles and age-restriction by [chexxor](https://github.com/chexxor)
+* [extractor/huya] Support HD streams by [felixonmars](https://github.com/felixonmars)
+* [extractor/moviepilot] Fix extractor by [panatexxa](https://github.com/panatexxa)
+* [extractor/nbc] Fix `NBC` and `NBCStations` extractors by [bashonly](https://github.com/bashonly)
+* [extractor/nbc] Fix XML parsing by [bashonly](https://github.com/bashonly)
+* [extractor/nebula] Remove broken cookie support by [hheimbuerger](https://github.com/hheimbuerger)
+* [extractor/nfl] Add `NFLPlus` extractors by [bashonly](https://github.com/bashonly)
+* [extractor/niconico] Add support for like history by [Matumo](https://github.com/Matumo), [pukkandan](https://github.com/pukkandan)
+* [extractor/nitter] Update instance list by [OIRNOIR](https://github.com/OIRNOIR)
+* [extractor/npo] Fix extractor and add HD support by [seproDev](https://github.com/seproDev)
+* [extractor/odkmedia] Add `OnDemandChinaEpisodeIE` by [HobbyistDev](https://github.com/HobbyistDev), [pukkandan](https://github.com/pukkandan)
+* [extractor/pornez] Handle relative URLs in iframe by [JChris246](https://github.com/JChris246)
+* [extractor/radiko] Fix format sorting for Time Free by [road-master](https://github.com/road-master)
+* [extractor/rcs] Fix extractors by [nixxo](https://github.com/nixxo), [pukkandan](https://github.com/pukkandan)
+* [extractor/reddit] Support user posts by [OMEGARAZER](https://github.com/OMEGARAZER)
+* [extractor/rumble] Fix format sorting by [pukkandan](https://github.com/pukkandan)
+* [extractor/servus] Rewrite extractor by [Ashish0804](https://github.com/Ashish0804), [FrankZ85](https://github.com/FrankZ85), [StefanLobbenmeier](https://github.com/StefanLobbenmeier)
+* [extractor/slideslive] Fix slides and chapters/duration by [bashonly](https://github.com/bashonly)
+* [extractor/SportDeutschland] Fix extractor by [FriedrichRehren](https://github.com/FriedrichRehren)
+* [extractor/Stripchat] Fix extractor by [JChris246](https://github.com/JChris246), [bashonly](https://github.com/bashonly)
+* [extractor/tnaflix] Fix extractor by [bashonly](https://github.com/bashonly), [oxamun](https://github.com/oxamun)
+* [extractor/tvp] Support `stream.tvp.pl` by [selfisekai](https://github.com/selfisekai)
+* [extractor/twitter] Fix `--no-playlist` and add media `view_count` when using GraphQL by [Grub4K](https://github.com/Grub4K)
+* [extractor/twitter] Fix graphql extraction on some tweets by [selfisekai](https://github.com/selfisekai)
+* [extractor/vimeo] Fix `playerConfig` extraction by [LeoniePhiline](https://github.com/LeoniePhiline), [bashonly](https://github.com/bashonly)
+* [extractor/viu] Add `ViuOTTIndonesiaIE` extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/vk] Fix playlists for new API by [the-marenga](https://github.com/the-marenga)
+* [extractor/vlive] Replace with `VLiveWebArchiveIE` by [seproDev](https://github.com/seproDev)
+* [extractor/ximalaya] Update album `_VALID_URL` by [carusocr](https://github.com/carusocr)
+* [extractor/zdf] Use android API endpoint for UHD downloads by [seproDev](https://github.com/seproDev)
+* [extractor/drtv] Fix bug in [ab4cbef](https://github.com/yt-dlp/yt-dlp/commit/ab4cbef) by [bashonly](https://github.com/bashonly)
+
+
+### 2023.02.17
+
+#### Core changes
+### Core changes
+- [Bugfix for 39f32f1715c0dffb7626dda7307db6388bb7abaa](https://github.com/yt-dlp/yt-dlp/commit/9ebac35577e61c3d25fafc959655fa3ab04ca7ef) by [pukkandan](https://github.com/pukkandan)
+- [Bugfix for 39f32f1715c0dffb7626dda7307db6388bb7abaa](https://github.com/yt-dlp/yt-dlp/commit/c154302c588c3d4362cec4fc5545e7e5d2bcf7a3) by [pukkandan](https://github.com/pukkandan)
+- [Fix `--concat-playlist`](https://github.com/yt-dlp/yt-dlp/commit/59d7de0da545944c48a82fc2937b996d7cd8cc9c) by [pukkandan](https://github.com/pukkandan)
+- [Imply `--no-progress` when `--print`](https://github.com/yt-dlp/yt-dlp/commit/5712943b764ba819ef479524c32700228603817a) by [pukkandan](https://github.com/pukkandan)
+- [Improve default subtitle language selection](https://github.com/yt-dlp/yt-dlp/commit/376aa24b1541e2bfb23337c0ae9bafa5bb3787f1) ([#6240](https://github.com/yt-dlp/yt-dlp/issues/6240)) by [sdht0](https://github.com/sdht0)
+- [Make `title` completely non-fatal](https://github.com/yt-dlp/yt-dlp/commit/7aefd19afed357c80743405ec2ace2148cba42e3) by [pukkandan](https://github.com/pukkandan)
+- [Sanitize formats before sorting](https://github.com/yt-dlp/yt-dlp/commit/39f32f1715c0dffb7626dda7307db6388bb7abaa) by [pukkandan](https://github.com/pukkandan)
+- [Support module level `__bool__` and `property`](https://github.com/yt-dlp/yt-dlp/commit/754c84e2e416cf6609dd0e4632b4985a08d34043) by [pukkandan](https://github.com/pukkandan)
+- [Update to ytdl-commit-2dd6c6e](https://github.com/yt-dlp/yt-dlp/commit/48fde8ac4ccbaaea868f6378814dde395f649fbf) by [pukkandan](https://github.com/pukkandan)
+- [extractor/douyutv]: [Use new API](https://github.com/yt-dlp/yt-dlp/commit/f14c2333481c63c24017a41ded7d8f36726504b7) ([#6074](https://github.com/yt-dlp/yt-dlp/issues/6074)) by [hatienl0i261299](https://github.com/hatienl0i261299)
+- compat_utils
+    - [Improve `passthrough_module`](https://github.com/yt-dlp/yt-dlp/commit/88426d9446758c707fb511408f2d6f56de952db4) by [pukkandan](https://github.com/pukkandan)
+    - [Simplify `EnhancedModule`](https://github.com/yt-dlp/yt-dlp/commit/768a00178109508893488e53a0e720b117fbccf6) by [pukkandan](https://github.com/pukkandan)
+- dependencies
+    - [Standardize `Cryptodome` imports](https://github.com/yt-dlp/yt-dlp/commit/f6a765ceb59c55aea06921880c1c87d1ff36e5de) by [pukkandan](https://github.com/pukkandan)
+- jsinterp
+    - [Support `if` statements](https://github.com/yt-dlp/yt-dlp/commit/8b008d62544b82e24a0ba36c30e8e51855d93419) by [pukkandan](https://github.com/pukkandan)
+- plugins
+    - [Fix zip search paths](https://github.com/yt-dlp/yt-dlp/commit/88d8928bf7630801865cf8728ae5c77234324b7b) by [pukkandan](https://github.com/pukkandan)
+- utils
+    - [Don't use Content-length with encoding](https://github.com/yt-dlp/yt-dlp/commit/65e5c021e7c5f23ecbc6a982b72a02ac6cd6900d) ([#6176](https://github.com/yt-dlp/yt-dlp/issues/6176)) by [felixonmars](https://github.com/felixonmars)
+    - [Fix `time_seconds` to use the provided TZ](https://github.com/yt-dlp/yt-dlp/commit/83c4970e52839ce8761ec61bd19d549aed7d7920) ([#6118](https://github.com/yt-dlp/yt-dlp/issues/6118)) by [Grub4K](https://github.com/Grub4K), [Lesmiscore](https://github.com/Lesmiscore)
+    - [Fix race condition in `make_dir`](https://github.com/yt-dlp/yt-dlp/commit/b25d6cb96337d479bdcb41768356da414c3aa835) ([#6089](https://github.com/yt-dlp/yt-dlp/issues/6089)) by [aionescu](https://github.com/aionescu)
+    - [Use local kernel32 for file locking on Windows](https://github.com/yt-dlp/yt-dlp/commit/37e325b92ff9d784715ac0e5d1f7d96bf5f45ad9) by [Grub4K](https://github.com/Grub4K)
+    - traverse_obj
+        - [Fix more bugs](https://github.com/yt-dlp/yt-dlp/commit/6839ae1f6dde4c0442619e351b3f0442312ab4f9) by [pukkandan](https://github.com/pukkandan)
+        - [Fix several behavioral problems](https://github.com/yt-dlp/yt-dlp/commit/b1bde57bef878478e3503ab07190fd207914ade9) by [Grub4K](https://github.com/Grub4K)
+        - [Various improvements](https://github.com/yt-dlp/yt-dlp/commit/776995bc109c5cd1aa56b684fada2ce718a386ec) by [Grub4K](https://github.com/Grub4K)
+### Extractor changes
+- [Fix `_search_nuxt_data`](https://github.com/yt-dlp/yt-dlp/commit/b23167e7542c177f32b22b29857b637dc4aede69) ([#6062](https://github.com/yt-dlp/yt-dlp/issues/6062)) by [LowSuggestion912](https://github.com/LowSuggestion912)
+- 91porn
+    - [Fix title and comment extraction](https://github.com/yt-dlp/yt-dlp/commit/c085cc2def9862ac8a7619ce8ea5dcc177325719) ([#5932](https://github.com/yt-dlp/yt-dlp/issues/5932)) by [pmitchell86](https://github.com/pmitchell86)
+- abematv
+    - [Cache user token whenever appropriate](https://github.com/yt-dlp/yt-dlp/commit/a4f16832213d9e29beecf685d6cd09a2f0b48c87) ([#6216](https://github.com/yt-dlp/yt-dlp/issues/6216)) by [Lesmiscore](https://github.com/Lesmiscore)
+- anchorfm
+    - [Add episode extractor](https://github.com/yt-dlp/yt-dlp/commit/a4ad59ff2ded208bf33f6fe07299a3449eadccdc) ([#6092](https://github.com/yt-dlp/yt-dlp/issues/6092)) by [bashonly](https://github.com/bashonly), [HobbyistDev](https://github.com/HobbyistDev)
+- bfmtv
+    - [Support `rmc` prefix](https://github.com/yt-dlp/yt-dlp/commit/20266508dd6247dd3cf0e97b9b9f14c3afc046db) ([#6025](https://github.com/yt-dlp/yt-dlp/issues/6025)) by [carusocr](https://github.com/carusocr)
+- biliintl
+    - [Add intro and ending chapters](https://github.com/yt-dlp/yt-dlp/commit/0ba87dd279d3565ed93c559cf7880ad61eb83af8) ([#6018](https://github.com/yt-dlp/yt-dlp/issues/6018)) by [HobbyistDev](https://github.com/HobbyistDev)
+- boxcast
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/9acca71237f42a4775008e51fe26e42f0a39c552) ([#5983](https://github.com/yt-dlp/yt-dlp/issues/5983)) by [HobbyistDev](https://github.com/HobbyistDev)
+- clyp
+    - [Support `wav`](https://github.com/yt-dlp/yt-dlp/commit/cc13293c2819b5461be211a9729fd02bb1e2f476) ([#6102](https://github.com/yt-dlp/yt-dlp/issues/6102)) by [qulaz](https://github.com/qulaz)
+- crunchyroll
+    - [Add intro chapter](https://github.com/yt-dlp/yt-dlp/commit/93abb7406b95793f6872d12979b91d5f336b4f43) ([#6023](https://github.com/yt-dlp/yt-dlp/issues/6023)) by [ByteDream](https://github.com/ByteDream)
+    - [Better message for premium videos](https://github.com/yt-dlp/yt-dlp/commit/44699d10dc8de9c6a338f4a8e5c63506ec4d2118) by [pukkandan](https://github.com/pukkandan)
+    - [Fix incorrect premium-only error](https://github.com/yt-dlp/yt-dlp/commit/c9d14bd22ab31e2a41f9f8061843668a06db583b) by [Grub4K](https://github.com/Grub4K)
+- drtv
+    - [Fix bug in ab4cbef](https://github.com/yt-dlp/yt-dlp/commit/7481998b169b2a52049fc33bff82034d6563ead4) ([#6034](https://github.com/yt-dlp/yt-dlp/issues/6034)) by [bashonly](https://github.com/bashonly)
+- ebay
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/da880559a6ecbbf374cc9f3378e696b55b9599af) ([#6170](https://github.com/yt-dlp/yt-dlp/issues/6170)) by [JChris246](https://github.com/JChris246)
+- embedly
+    - [Embedded links may be for other extractors](https://github.com/yt-dlp/yt-dlp/commit/87ebab0615b1bf9b14b478b055e7059d630b4833) by [pukkandan](https://github.com/pukkandan)
+- freesound
+    - [Workaround invalid URL in webpage](https://github.com/yt-dlp/yt-dlp/commit/9cfdbcbf3f17be51f5b6bb9bb6d880b2f3d67362) ([#6147](https://github.com/yt-dlp/yt-dlp/issues/6147)) by [rebane2001](https://github.com/rebane2001)
+- generic
+    - [Avoid catastrophic backtracking in KVS regex](https://github.com/yt-dlp/yt-dlp/commit/8aa0bd5d10627ece3c1815c01d02fb8bf22847a7) by [bashonly](https://github.com/bashonly)
+- goplay
+    - [Use new API](https://github.com/yt-dlp/yt-dlp/commit/d27bde98832e3b7ffb39f3cf6346011b97bb3bc3) ([#6151](https://github.com/yt-dlp/yt-dlp/issues/6151)) by [jeroenj](https://github.com/jeroenj)
+- hidive
+    - [Fix subtitles and age-restriction](https://github.com/yt-dlp/yt-dlp/commit/7708df8da05c94270b43e0630e4e20f6d2d62c55) ([#5828](https://github.com/yt-dlp/yt-dlp/issues/5828)) by [chexxor](https://github.com/chexxor)
+- huya
+    - [Support HD streams](https://github.com/yt-dlp/yt-dlp/commit/fbbb5508ea98ed8709847f5ecced7d70ff05e0ee) ([#6172](https://github.com/yt-dlp/yt-dlp/issues/6172)) by [felixonmars](https://github.com/felixonmars)
+- hypergryph
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/31c279a2a2c2ef402a9e6dad9992b310d16439a6) ([#6094](https://github.com/yt-dlp/yt-dlp/issues/6094)) by [bashonly](https://github.com/bashonly), [HobbyistDev](https://github.com/HobbyistDev)
+- moviepilot
+    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/c62e64cf0122e52fa2175dd1b004ca6b8e1d82af) ([#5954](https://github.com/yt-dlp/yt-dlp/issues/5954)) by [panatexxa](https://github.com/panatexxa)
+- nbc
+    - [Fix XML parsing](https://github.com/yt-dlp/yt-dlp/commit/176a068cde4f2d9dfa0336168caead0b1edcb8ac) by [bashonly](https://github.com/bashonly)
+    - [Fix `NBC` and `NBCStations` extractors](https://github.com/yt-dlp/yt-dlp/commit/cb73b8460c3ce6d37ab651a4e44bb23b10056154) ([#6033](https://github.com/yt-dlp/yt-dlp/issues/6033)) by [bashonly](https://github.com/bashonly)
+- nebula
+    - [Remove broken cookie support](https://github.com/yt-dlp/yt-dlp/commit/d50ea3ce5abc3b0defc0e5d1e22b22ce9b01b07b) ([#5979](https://github.com/yt-dlp/yt-dlp/issues/5979)) by [hheimbuerger](https://github.com/hheimbuerger)
+- nfl
+    - [Add `NFLPlus` extractors](https://github.com/yt-dlp/yt-dlp/commit/8b37c58f8b5494504acdb5ebe3f8bbd26230f725) ([#6222](https://github.com/yt-dlp/yt-dlp/issues/6222)) by [bashonly](https://github.com/bashonly)
+- niconico
+    - [Add support for like history](https://github.com/yt-dlp/yt-dlp/commit/3b161265add30613bde2e46fca214fe94d09e651) ([#5705](https://github.com/yt-dlp/yt-dlp/issues/5705)) by [Matumo](https://github.com/Matumo), [pukkandan](https://github.com/pukkandan)
+- nitter
+    - [Update instance list](https://github.com/yt-dlp/yt-dlp/commit/a9189510baadf0dccd2d4d363bc6f3a441128bb0) ([#6236](https://github.com/yt-dlp/yt-dlp/issues/6236)) by [OIRNOIR](https://github.com/OIRNOIR)
+- npo
+    - [Fix extractor and add HD support](https://github.com/yt-dlp/yt-dlp/commit/cc2389c8ac72a514d4e002a0f6ca5a7d65c7eff0) ([#6155](https://github.com/yt-dlp/yt-dlp/issues/6155)) by [seproDev](https://github.com/seproDev)
+- nzonscreen
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/d3bb187f01e1e30db05e639fc23a2e1935d777fe) ([#6208](https://github.com/yt-dlp/yt-dlp/issues/6208)) by [gregsadetsky](https://github.com/gregsadetsky), [pukkandan](https://github.com/pukkandan)
+- odkmedia
+    - [Add `OnDemandChinaEpisodeIE`](https://github.com/yt-dlp/yt-dlp/commit/10fd9e6ee833c88edf6c633f864f42843a708d32) ([#6116](https://github.com/yt-dlp/yt-dlp/issues/6116)) by [HobbyistDev](https://github.com/HobbyistDev), [pukkandan](https://github.com/pukkandan)
+- pornez
+    - [Handle relative URLs in iframe](https://github.com/yt-dlp/yt-dlp/commit/f7efe6dc958eb0689cb9534ff0b4e592040be8df) ([#6171](https://github.com/yt-dlp/yt-dlp/issues/6171)) by [JChris246](https://github.com/JChris246)
+- radiko
+    - [Fix format sorting for Time Free](https://github.com/yt-dlp/yt-dlp/commit/203a06f8554df6db07d8f20f465ecbfe8a14e591) ([#6159](https://github.com/yt-dlp/yt-dlp/issues/6159)) by [road-master](https://github.com/road-master)
+- rcs
+    - [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/c6b657867ad68af6b930ed0aa11ec5d93ee187b7) ([#5700](https://github.com/yt-dlp/yt-dlp/issues/5700)) by [nixxo](https://github.com/nixxo), [pukkandan](https://github.com/pukkandan)
+- reddit
+    - [Support user posts](https://github.com/yt-dlp/yt-dlp/commit/c77df98b1a477a020a57141464d10c0f4d0fdbc9) ([#6173](https://github.com/yt-dlp/yt-dlp/issues/6173)) by [OMEGARAZER](https://github.com/OMEGARAZER)
+- rozhlas
+    - [Add extractor RozhlasVltavaIE](https://github.com/yt-dlp/yt-dlp/commit/355d781bed497cbcb254bf2a2737b83fa51c84ea) ([#5951](https://github.com/yt-dlp/yt-dlp/issues/5951)) by [amra](https://github.com/amra)
+- rumble
+    - [Fix format sorting](https://github.com/yt-dlp/yt-dlp/commit/acacb57c7e173b93c6e0f0c43e61b9b2912719d8) by [pukkandan](https://github.com/pukkandan)
+- servus
+    - [Rewrite extractor](https://github.com/yt-dlp/yt-dlp/commit/f40e32fb1ac67be5bdbc8e32a3c235abfc4be260) ([#6036](https://github.com/yt-dlp/yt-dlp/issues/6036)) by [Ashish0804](https://github.com/Ashish0804), [FrankZ85](https://github.com/FrankZ85), [StefanLobbenmeier](https://github.com/StefanLobbenmeier)
+- slideslive
+    - [Fix slides and chapters/duration](https://github.com/yt-dlp/yt-dlp/commit/5ab3534d44231f7711398bc3cfc520e2efd09f50) ([#6024](https://github.com/yt-dlp/yt-dlp/issues/6024)) by [bashonly](https://github.com/bashonly)
+- sportdeutschland
+    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/5e1a54f63e393c218a40949012ff0de0ce63cb15) ([#6041](https://github.com/yt-dlp/yt-dlp/issues/6041)) by [FriedrichRehren](https://github.com/FriedrichRehren)
+- stripchat
+    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/7d5f919bad07017f4b39b55725491b1e9717d47a) ([#5985](https://github.com/yt-dlp/yt-dlp/issues/5985)) by [bashonly](https://github.com/bashonly), [JChris246](https://github.com/JChris246)
+- tempo
+    - [Add IVXPlayer extractor](https://github.com/yt-dlp/yt-dlp/commit/30031be974d210f451100339699ef03b0ddb5f10) ([#5837](https://github.com/yt-dlp/yt-dlp/issues/5837)) by [HobbyistDev](https://github.com/HobbyistDev)
+- tnaflix
+    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/989f47b6315541989bb507f26b431d9586430995) ([#6086](https://github.com/yt-dlp/yt-dlp/issues/6086)) by [bashonly](https://github.com/bashonly), [oxamun](https://github.com/oxamun)
+- tvp
+    - [Support `stream.tvp.pl`](https://github.com/yt-dlp/yt-dlp/commit/a31d0fa6c315b1145d682361149003d98f1e3782) ([#6139](https://github.com/yt-dlp/yt-dlp/issues/6139)) by [selfisekai](https://github.com/selfisekai)
+- twitter
+    - [Fix `--no-playlist` and add media `view_count` when using GraphQL](https://github.com/yt-dlp/yt-dlp/commit/b6795fd310f1dd61dddc9fd08e52fe485bdc8a3e) ([#6211](https://github.com/yt-dlp/yt-dlp/issues/6211)) by [Grub4K](https://github.com/Grub4K)
+    - [Fix graphql extraction on some tweets](https://github.com/yt-dlp/yt-dlp/commit/7543c9c99bcb116b085fdb1f41b84a0ead04c05d) ([#6075](https://github.com/yt-dlp/yt-dlp/issues/6075)) by [selfisekai](https://github.com/selfisekai)
+- txxx
+    - [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/389896df85ed14eaf74f72531da6c4491d6b73b0) ([#5240](https://github.com/yt-dlp/yt-dlp/issues/5240)) by [chio0hai](https://github.com/chio0hai)
+- vimeo
+    - [Fix `playerConfig` extraction](https://github.com/yt-dlp/yt-dlp/commit/c0cd13fb1c71b842c3d272d0273c03542b467766) ([#6203](https://github.com/yt-dlp/yt-dlp/issues/6203)) by [bashonly](https://github.com/bashonly), [LeoniePhiline](https://github.com/LeoniePhiline)
+- viu
+    - [Add `ViuOTTIndonesiaIE` extractor](https://github.com/yt-dlp/yt-dlp/commit/72671a212d7c939329cb5d34335fa089dd3acbd3) ([#6099](https://github.com/yt-dlp/yt-dlp/issues/6099)) by [HobbyistDev](https://github.com/HobbyistDev)
+- vk
+    - [Fix playlists for new API](https://github.com/yt-dlp/yt-dlp/commit/a9c685453f7019bee94170f936619c6db76c964e) ([#6122](https://github.com/yt-dlp/yt-dlp/issues/6122)) by [the-marenga](https://github.com/the-marenga)
+- vlive
+    - [Replace with `VLiveWebArchiveIE`](https://github.com/yt-dlp/yt-dlp/commit/b3eaab7ca2e118d4db73dcb44afd9c8717db8b67) ([#6196](https://github.com/yt-dlp/yt-dlp/issues/6196)) by [seproDev](https://github.com/seproDev)
+- vocaroo
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/e4a8b1769e19755acba6d8f212208359905a3159) ([#6117](https://github.com/yt-dlp/yt-dlp/issues/6117)) by [qbnu](https://github.com/qbnu), [SuperSonicHub1](https://github.com/SuperSonicHub1)
+- wrestleuniverse
+    - [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/e61acb40b2cb6ef45508d72235026d458c9d5dff) ([#6158](https://github.com/yt-dlp/yt-dlp/issues/6158)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+- ximalaya
+    - [Update album `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/417cdaae08fc447c9d15c53a88e2e9a027cdbf0a) ([#6110](https://github.com/yt-dlp/yt-dlp/issues/6110)) by [carusocr](https://github.com/carusocr)
+- yappy
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/361630015535026712bdb67f804a15b65ff9ee7e) ([#6111](https://github.com/yt-dlp/yt-dlp/issues/6111)) by [HobbyistDev](https://github.com/HobbyistDev)
+- youtube
+    - [Add hyperpipe instances](https://github.com/yt-dlp/yt-dlp/commit/78a78fa74dbc888d20f1b65e1382bf99131597d5) ([#6020](https://github.com/yt-dlp/yt-dlp/issues/6020)) by [Generator](https://github.com/Generator)
+    - [Fix `uploader_id` extraction](https://github.com/yt-dlp/yt-dlp/commit/149eb0bbf34fa8fdf8d1e2aa28e17479d099e26b) by [bashonly](https://github.com/bashonly)
+    - [Handle `consent.youtube`](https://github.com/yt-dlp/yt-dlp/commit/b032ff0f032512bd6fc70c9c1994d906eacc06cb) by [pukkandan](https://github.com/pukkandan)
+    - [Support `/live/` URL](https://github.com/yt-dlp/yt-dlp/commit/dad2210c0cb9cf03702a9511817ee5ec646d7bc8) by [pukkandan](https://github.com/pukkandan)
+    - [Update invidious and piped instances](https://github.com/yt-dlp/yt-dlp/commit/05799a48c7dec12b34c8bf951c8d2eceedda59f8) ([#6030](https://github.com/yt-dlp/yt-dlp/issues/6030)) by [rohieb](https://github.com/rohieb)
+    - [`uploader_id` includes `@` with handle](https://github.com/yt-dlp/yt-dlp/commit/c61cf091a54d3aa3c611722035ccde5ecfe981bb) by [bashonly](https://github.com/bashonly)
+- zdf
+    - [Use android API endpoint for UHD downloads](https://github.com/yt-dlp/yt-dlp/commit/0fe87a8730638490415d630f48e61d264d89c358) ([#6150](https://github.com/yt-dlp/yt-dlp/issues/6150)) by [seproDev](https://github.com/seproDev)
+### Downloader changes
+- hls
+    - [Allow extractors to provide AES key](https://github.com/yt-dlp/yt-dlp/commit/7e68567e508168b345266c0c19812ad50a829eaa) ([#6158](https://github.com/yt-dlp/yt-dlp/issues/6158)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+### Postprocessor changes
+- extractaudio
+    - [Handle outtmpl without ext](https://github.com/yt-dlp/yt-dlp/commit/f737fb16d8234408c85bc189ccc926fea000515b) ([#6005](https://github.com/yt-dlp/yt-dlp/issues/6005)) by [carusocr](https://github.com/carusocr)
+- pyinst
+    - [Fix for pyinstaller 5.8](https://github.com/yt-dlp/yt-dlp/commit/2e269bd998c61efaf7500907d114a56e5e83e65e) by [pukkandan](https://github.com/pukkandan)
+### Misc. changes
+- build
+    - [Update pyinstaller](https://github.com/yt-dlp/yt-dlp/commit/365b9006051ac7d735c20bb63c4907b758233048) by [pukkandan](https://github.com/pukkandan)
+- cleanup
+    - Miscellaneous: [76c9c52](https://github.com/yt-dlp/yt-dlp/commit/76c9c523071150053df7b56956646b680b6a6e05) by [pukkandan](https://github.com/pukkandan)
+- devscripts
+    - [Provide pyinstaller hooks](https://github.com/yt-dlp/yt-dlp/commit/acb1042a9ffa8769fe691beac1011d6da1fcf321) by [pukkandan](https://github.com/pukkandan)
+- pyinstaller
+    - [Analyze sub-modules of `Cryptodome`](https://github.com/yt-dlp/yt-dlp/commit/b85faf6ffb700058e774e99c04304a7a9257cdd0) by [pukkandan](https://github.com/pukkandan)
 
 ### 2023.01.06
 
diff --git a/Collaborators.md b/Collaborators.md
index fe2a7f4b4..83dfbe389 100644
--- a/Collaborators.md
+++ b/Collaborators.md
@@ -59,3 +59,11 @@ ## [bashonly](https://github.com/bashonly)
 * `--cookies-from-browser` support for Firefox containers
 * Added support for new websites Genius, Kick, NBCStations, Triller, VideoKen etc
 * Improved/fixed support for Anvato, Brightcove, Instagram, ParamountPlus, Reddit, SlidesLive, TikTok, Twitter, Vimeo etc
+
+
+## [Grub4K](https://github.com/Grub4K)
+
+[![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/Grub4K) [![gh-sponsor](https://img.shields.io/badge/_-Github-red.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/Grub4K)
+
+* Rework internals like `traverse_obj`, various core refactors and bugs fixes
+* Helped fix crunchyroll, Twitter, wrestleuniverse, wistia, slideslive etc
diff --git a/supportedsites.md b/supportedsites.md
index 5cef7ac90..b545ec540 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -63,14 +63,15 @@ # Supported sites
  - **AluraCourse**: [<abbr title="netrc machine"><em>aluracourse</em></abbr>]
  - **Amara**
  - **AmazonMiniTV**
- - **amazonminitv:season**: Amazon MiniTV Series, "minitv:season:" prefix
- - **amazonminitv:series**
+ - **amazonminitv:season**: Amazon MiniTV Season, "minitv:season:" prefix
+ - **amazonminitv:series**: Amazon MiniTV Series, "minitv:series:" prefix
  - **AmazonReviews**
  - **AmazonStore**
  - **AMCNetworks**
  - **AmericasTestKitchen**
  - **AmericasTestKitchenSeason**
  - **AmHistoryChannel**
+ - **AnchorFMEpisode**
  - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Angel**
  - **AnimalPlanet**
@@ -177,6 +178,7 @@ # Supported sites
  - **BlackboardCollaborate**
  - **BleacherReport**
  - **BleacherReportCMS**
+ - **blerp**
  - **blogger.com**
  - **Bloomberg**
  - **BokeCC**
@@ -184,6 +186,7 @@ # Supported sites
  - **BooyahClips**
  - **BostonGlobe**
  - **Box**
+ - **BoxCastVideo**
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk
  - **BravoTV**
@@ -364,6 +367,7 @@ # Supported sites
  - **dw:article**
  - **EaglePlatform**
  - **EbaumsWorld**
+ - **Ebay**
  - **EchoMsk**
  - **egghead:course**: egghead.io course
  - **egghead:lesson**: egghead.io lesson
@@ -595,6 +599,7 @@ # Supported sites
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **ivideon**: Ivideon TV
+ - **IVXPlayer**
  - **Iwara**
  - **iwara:playlist**
  - **iwara:user**
@@ -626,6 +631,7 @@ # Supported sites
  - **KickVOD**
  - **KinjaEmbed**
  - **KinoPoisk**
+ - **Kommunetv**
  - **KompasVideo**
  - **KonserthusetPlay**
  - **Koo**
@@ -773,6 +779,7 @@ # Supported sites
  - **Mofosex**
  - **MofosexEmbed**
  - **Mojvideo**
+ - **MonsterSirenHypergryphMusic**
  - **Morningstar**: morningstar.com
  - **Motherless**
  - **MotherlessGroup**
@@ -878,6 +885,8 @@ # Supported sites
  - **NFHSNetwork**
  - **nfl.com**
  - **nfl.com:article**
+ - **nfl.com:​plus:episode**
+ - **nfl.com:​plus:replay**
  - **NhkForSchoolBangumi**
  - **NhkForSchoolProgramList**
  - **NhkForSchoolSubject**: Portal page for each school subjects, like Japanese (kokugo, 国語) or math (sansuu/suugaku or 算数・数学)
@@ -890,7 +899,7 @@ # Supported sites
  - **nickelodeonru**
  - **nicknight**
  - **niconico**: [<abbr title="netrc machine"><em>niconico</em></abbr>] ニコニコ動画
- - **niconico:history**: NicoNico user history. Requires cookies.
+ - **niconico:history**: NicoNico user history or likes. Requires cookies.
  - **niconico:playlist**
  - **niconico:series**
  - **niconico:tag**: NicoNico video tag URLs
@@ -940,6 +949,7 @@ # Supported sites
  - **NYTimesArticle**
  - **NYTimesCooking**
  - **nzherald**
+ - **NZOnScreen**
  - **NZZ**
  - **ocw.mit.edu**
  - **OdaTV**
@@ -949,6 +959,7 @@ # Supported sites
  - **OktoberfestTV**
  - **OlympicsReplay**
  - **on24**: ON24
+ - **OnDemandChinaEpisode**
  - **OnDemandKorea**
  - **OneFootball**
  - **OnePlacePodcast**
@@ -1063,7 +1074,10 @@ # Supported sites
  - **Pornotube**
  - **PornoVoisines**
  - **PornoXO**
+ - **PornTop**
  - **PornTube**
+ - **Pr0gramm**
+ - **Pr0grammStatic**
  - **PrankCast**
  - **PremiershipRugby**
  - **PressTV**
@@ -1115,6 +1129,8 @@ # Supported sites
  - **RaiSudtirol**
  - **RayWenderlich**
  - **RayWenderlichCourse**
+ - **RbgTum**
+ - **RbgTumCourse**
  - **RBMARadio**
  - **RCS**
  - **RCSEmbeds**
@@ -1149,6 +1165,7 @@ # Supported sites
  - **RoosterTeethSeries**: [<abbr title="netrc machine"><em>roosterteeth</em></abbr>]
  - **RottenTomatoes**
  - **Rozhlas**
+ - **RozhlasVltava**
  - **RTBF**: [<abbr title="netrc machine"><em>rtbf</em></abbr>]
  - **RTDocumentry**
  - **RTDocumentryPlaylist**
@@ -1485,6 +1502,7 @@ # Supported sites
  - **twitter:card**
  - **twitter:shortener**
  - **twitter:spaces**
+ - **Txxx**
  - **udemy**: [<abbr title="netrc machine"><em>udemy</em></abbr>]
  - **udemy:course**: [<abbr title="netrc machine"><em>udemy</em></abbr>]
  - **UDNEmbed**: 聯合影音
@@ -1572,14 +1590,13 @@ # Supported sites
  - **Viu**
  - **viu:ott**: [<abbr title="netrc machine"><em>viu</em></abbr>]
  - **viu:playlist**
+ - **ViuOTTIndonesia**
  - **Vivo**: vivo.sx
  - **vk**: [<abbr title="netrc machine"><em>vk</em></abbr>] VK
  - **vk:uservideos**: [<abbr title="netrc machine"><em>vk</em></abbr>] VK - User's Videos
  - **vk:wallpost**: [<abbr title="netrc machine"><em>vk</em></abbr>]
- - **vlive**: [<abbr title="netrc machine"><em>vlive</em></abbr>]
- - **vlive:channel**: [<abbr title="netrc machine"><em>vlive</em></abbr>]
- - **vlive:post**: [<abbr title="netrc machine"><em>vlive</em></abbr>]
  - **vm.tiktok**
+ - **Vocaroo**
  - **Vodlocker**
  - **VODPl**
  - **VODPlatform**
@@ -1628,6 +1645,7 @@ # Supported sites
  - **wdr:mobile**: (**Currently broken**)
  - **WDRElefant**
  - **WDRPage**
+ - **web.archive:vlive**: web.archive.org saved vlive videos
  - **web.archive:youtube**: web.archive.org saved youtube videos, "ytarchive:" prefix
  - **Webcamerapl**
  - **Webcaster**
@@ -1653,6 +1671,8 @@ # Supported sites
  - **WorldStarHipHop**
  - **wppilot**
  - **wppilot:channels**
+ - **WrestleUniversePPV**
+ - **WrestleUniverseVOD**
  - **WSJ**: Wall Street Journal
  - **WSJArticle**
  - **WWE**
@@ -1689,6 +1709,7 @@ # Supported sites
  - **YandexVideo**
  - **YandexVideoPreview**
  - **YapFiles**
+ - **Yappy**
  - **YesJapan**
  - **yinyuetai:video**: 音悦Tai
  - **YleAreena**

From 41bd0dc4d71919dceeb84a3aab9c9934d46eee9f Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@example.com>
Date: Fri, 17 Feb 2023 12:31:30 +0000
Subject: [PATCH 087/871] [version] update

Created by: pukkandan

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml          | 8 ++++----
 .github/ISSUE_TEMPLATE/2_site_support_request.yml | 8 ++++----
 .github/ISSUE_TEMPLATE/3_site_feature_request.yml | 8 ++++----
 .github/ISSUE_TEMPLATE/4_bug_report.yml           | 8 ++++----
 .github/ISSUE_TEMPLATE/5_feature_request.yml      | 8 ++++----
 .github/ISSUE_TEMPLATE/6_question.yml             | 8 ++++----
 yt_dlp/version.py                                 | 4 ++--
 7 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index 2237665e3..e1103fb84 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a broken site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.01.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -62,7 +62,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.01.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -70,8 +70,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.01.06, Current version: 2023.01.06
-        yt-dlp is up to date (2023.01.06)
+        Latest version: 2023.02.17, Current version: 2023.02.17
+        yt-dlp is up to date (2023.02.17)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index 0e2940d86..90d7294ac 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.01.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -74,7 +74,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.01.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -82,8 +82,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.01.06, Current version: 2023.01.06
-        yt-dlp is up to date (2023.01.06)
+        Latest version: 2023.02.17, Current version: 2023.02.17
+        yt-dlp is up to date (2023.02.17)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index 92501be2e..5b59852c7 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.01.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -70,7 +70,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.01.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -78,8 +78,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.01.06, Current version: 2023.01.06
-        yt-dlp is up to date (2023.01.06)
+        Latest version: 2023.02.17, Current version: 2023.02.17
+        yt-dlp is up to date (2023.02.17)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index bdfc0efb8..bd4695f87 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.01.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -55,7 +55,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.01.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -63,8 +63,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.01.06, Current version: 2023.01.06
-        yt-dlp is up to date (2023.01.06)
+        Latest version: 2023.02.17, Current version: 2023.02.17
+        yt-dlp is up to date (2023.02.17)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index c9e3aba38..8c7f315e9 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.01.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -51,7 +51,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.01.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -59,7 +59,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.01.06, Current version: 2023.01.06
-        yt-dlp is up to date (2023.01.06)
+        Latest version: 2023.02.17, Current version: 2023.02.17
+        yt-dlp is up to date (2023.02.17)
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index fe6a4ee3f..4a1344628 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.01.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -57,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.01.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -65,7 +65,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.01.06, Current version: 2023.01.06
-        yt-dlp is up to date (2023.01.06)
+        Latest version: 2023.02.17, Current version: 2023.02.17
+        yt-dlp is up to date (2023.02.17)
         <more lines>
       render: shell
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index f722ec665..3c92a85e1 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.01.06'
+__version__ = '2023.02.17'
 
-RELEASE_GIT_HEAD = '6becd2508'
+RELEASE_GIT_HEAD = 'a0a7c0154'
 
 VARIANT = None
 

From 17ca19ab60a6a13eb8a629c51442b5248b0d8394 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 17 Feb 2023 18:38:05 +0530
Subject: [PATCH 088/871] [cleanup] Fix `Changelog`

---
 Changelog.md | 165 +--------------------------------------------------
 1 file changed, 1 insertion(+), 164 deletions(-)

diff --git a/Changelog.md b/Changelog.md
index 36856e016..8d3ac089c 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -10,7 +10,7 @@ # Instuctions for creating release
 * Dispatch the workflow https://github.com/yt-dlp/yt-dlp/actions/workflows/build.yml on master
 -->
 
-# 2023.02.17
+### 2023.02.17
 
 * Merge youtube-dl: Upto [commit/2dd6c6e](https://github.com/ytdl-org/youtube-dl/commit/2dd6c6e)
 * Fix `--concat-playlist`
@@ -101,169 +101,6 @@ # 2023.02.17
 * [extractor/drtv] Fix bug in [ab4cbef](https://github.com/yt-dlp/yt-dlp/commit/ab4cbef) by [bashonly](https://github.com/bashonly)
 
 
-### 2023.02.17
-
-#### Core changes
-### Core changes
-- [Bugfix for 39f32f1715c0dffb7626dda7307db6388bb7abaa](https://github.com/yt-dlp/yt-dlp/commit/9ebac35577e61c3d25fafc959655fa3ab04ca7ef) by [pukkandan](https://github.com/pukkandan)
-- [Bugfix for 39f32f1715c0dffb7626dda7307db6388bb7abaa](https://github.com/yt-dlp/yt-dlp/commit/c154302c588c3d4362cec4fc5545e7e5d2bcf7a3) by [pukkandan](https://github.com/pukkandan)
-- [Fix `--concat-playlist`](https://github.com/yt-dlp/yt-dlp/commit/59d7de0da545944c48a82fc2937b996d7cd8cc9c) by [pukkandan](https://github.com/pukkandan)
-- [Imply `--no-progress` when `--print`](https://github.com/yt-dlp/yt-dlp/commit/5712943b764ba819ef479524c32700228603817a) by [pukkandan](https://github.com/pukkandan)
-- [Improve default subtitle language selection](https://github.com/yt-dlp/yt-dlp/commit/376aa24b1541e2bfb23337c0ae9bafa5bb3787f1) ([#6240](https://github.com/yt-dlp/yt-dlp/issues/6240)) by [sdht0](https://github.com/sdht0)
-- [Make `title` completely non-fatal](https://github.com/yt-dlp/yt-dlp/commit/7aefd19afed357c80743405ec2ace2148cba42e3) by [pukkandan](https://github.com/pukkandan)
-- [Sanitize formats before sorting](https://github.com/yt-dlp/yt-dlp/commit/39f32f1715c0dffb7626dda7307db6388bb7abaa) by [pukkandan](https://github.com/pukkandan)
-- [Support module level `__bool__` and `property`](https://github.com/yt-dlp/yt-dlp/commit/754c84e2e416cf6609dd0e4632b4985a08d34043) by [pukkandan](https://github.com/pukkandan)
-- [Update to ytdl-commit-2dd6c6e](https://github.com/yt-dlp/yt-dlp/commit/48fde8ac4ccbaaea868f6378814dde395f649fbf) by [pukkandan](https://github.com/pukkandan)
-- [extractor/douyutv]: [Use new API](https://github.com/yt-dlp/yt-dlp/commit/f14c2333481c63c24017a41ded7d8f36726504b7) ([#6074](https://github.com/yt-dlp/yt-dlp/issues/6074)) by [hatienl0i261299](https://github.com/hatienl0i261299)
-- compat_utils
-    - [Improve `passthrough_module`](https://github.com/yt-dlp/yt-dlp/commit/88426d9446758c707fb511408f2d6f56de952db4) by [pukkandan](https://github.com/pukkandan)
-    - [Simplify `EnhancedModule`](https://github.com/yt-dlp/yt-dlp/commit/768a00178109508893488e53a0e720b117fbccf6) by [pukkandan](https://github.com/pukkandan)
-- dependencies
-    - [Standardize `Cryptodome` imports](https://github.com/yt-dlp/yt-dlp/commit/f6a765ceb59c55aea06921880c1c87d1ff36e5de) by [pukkandan](https://github.com/pukkandan)
-- jsinterp
-    - [Support `if` statements](https://github.com/yt-dlp/yt-dlp/commit/8b008d62544b82e24a0ba36c30e8e51855d93419) by [pukkandan](https://github.com/pukkandan)
-- plugins
-    - [Fix zip search paths](https://github.com/yt-dlp/yt-dlp/commit/88d8928bf7630801865cf8728ae5c77234324b7b) by [pukkandan](https://github.com/pukkandan)
-- utils
-    - [Don't use Content-length with encoding](https://github.com/yt-dlp/yt-dlp/commit/65e5c021e7c5f23ecbc6a982b72a02ac6cd6900d) ([#6176](https://github.com/yt-dlp/yt-dlp/issues/6176)) by [felixonmars](https://github.com/felixonmars)
-    - [Fix `time_seconds` to use the provided TZ](https://github.com/yt-dlp/yt-dlp/commit/83c4970e52839ce8761ec61bd19d549aed7d7920) ([#6118](https://github.com/yt-dlp/yt-dlp/issues/6118)) by [Grub4K](https://github.com/Grub4K), [Lesmiscore](https://github.com/Lesmiscore)
-    - [Fix race condition in `make_dir`](https://github.com/yt-dlp/yt-dlp/commit/b25d6cb96337d479bdcb41768356da414c3aa835) ([#6089](https://github.com/yt-dlp/yt-dlp/issues/6089)) by [aionescu](https://github.com/aionescu)
-    - [Use local kernel32 for file locking on Windows](https://github.com/yt-dlp/yt-dlp/commit/37e325b92ff9d784715ac0e5d1f7d96bf5f45ad9) by [Grub4K](https://github.com/Grub4K)
-    - traverse_obj
-        - [Fix more bugs](https://github.com/yt-dlp/yt-dlp/commit/6839ae1f6dde4c0442619e351b3f0442312ab4f9) by [pukkandan](https://github.com/pukkandan)
-        - [Fix several behavioral problems](https://github.com/yt-dlp/yt-dlp/commit/b1bde57bef878478e3503ab07190fd207914ade9) by [Grub4K](https://github.com/Grub4K)
-        - [Various improvements](https://github.com/yt-dlp/yt-dlp/commit/776995bc109c5cd1aa56b684fada2ce718a386ec) by [Grub4K](https://github.com/Grub4K)
-### Extractor changes
-- [Fix `_search_nuxt_data`](https://github.com/yt-dlp/yt-dlp/commit/b23167e7542c177f32b22b29857b637dc4aede69) ([#6062](https://github.com/yt-dlp/yt-dlp/issues/6062)) by [LowSuggestion912](https://github.com/LowSuggestion912)
-- 91porn
-    - [Fix title and comment extraction](https://github.com/yt-dlp/yt-dlp/commit/c085cc2def9862ac8a7619ce8ea5dcc177325719) ([#5932](https://github.com/yt-dlp/yt-dlp/issues/5932)) by [pmitchell86](https://github.com/pmitchell86)
-- abematv
-    - [Cache user token whenever appropriate](https://github.com/yt-dlp/yt-dlp/commit/a4f16832213d9e29beecf685d6cd09a2f0b48c87) ([#6216](https://github.com/yt-dlp/yt-dlp/issues/6216)) by [Lesmiscore](https://github.com/Lesmiscore)
-- anchorfm
-    - [Add episode extractor](https://github.com/yt-dlp/yt-dlp/commit/a4ad59ff2ded208bf33f6fe07299a3449eadccdc) ([#6092](https://github.com/yt-dlp/yt-dlp/issues/6092)) by [bashonly](https://github.com/bashonly), [HobbyistDev](https://github.com/HobbyistDev)
-- bfmtv
-    - [Support `rmc` prefix](https://github.com/yt-dlp/yt-dlp/commit/20266508dd6247dd3cf0e97b9b9f14c3afc046db) ([#6025](https://github.com/yt-dlp/yt-dlp/issues/6025)) by [carusocr](https://github.com/carusocr)
-- biliintl
-    - [Add intro and ending chapters](https://github.com/yt-dlp/yt-dlp/commit/0ba87dd279d3565ed93c559cf7880ad61eb83af8) ([#6018](https://github.com/yt-dlp/yt-dlp/issues/6018)) by [HobbyistDev](https://github.com/HobbyistDev)
-- boxcast
-    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/9acca71237f42a4775008e51fe26e42f0a39c552) ([#5983](https://github.com/yt-dlp/yt-dlp/issues/5983)) by [HobbyistDev](https://github.com/HobbyistDev)
-- clyp
-    - [Support `wav`](https://github.com/yt-dlp/yt-dlp/commit/cc13293c2819b5461be211a9729fd02bb1e2f476) ([#6102](https://github.com/yt-dlp/yt-dlp/issues/6102)) by [qulaz](https://github.com/qulaz)
-- crunchyroll
-    - [Add intro chapter](https://github.com/yt-dlp/yt-dlp/commit/93abb7406b95793f6872d12979b91d5f336b4f43) ([#6023](https://github.com/yt-dlp/yt-dlp/issues/6023)) by [ByteDream](https://github.com/ByteDream)
-    - [Better message for premium videos](https://github.com/yt-dlp/yt-dlp/commit/44699d10dc8de9c6a338f4a8e5c63506ec4d2118) by [pukkandan](https://github.com/pukkandan)
-    - [Fix incorrect premium-only error](https://github.com/yt-dlp/yt-dlp/commit/c9d14bd22ab31e2a41f9f8061843668a06db583b) by [Grub4K](https://github.com/Grub4K)
-- drtv
-    - [Fix bug in ab4cbef](https://github.com/yt-dlp/yt-dlp/commit/7481998b169b2a52049fc33bff82034d6563ead4) ([#6034](https://github.com/yt-dlp/yt-dlp/issues/6034)) by [bashonly](https://github.com/bashonly)
-- ebay
-    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/da880559a6ecbbf374cc9f3378e696b55b9599af) ([#6170](https://github.com/yt-dlp/yt-dlp/issues/6170)) by [JChris246](https://github.com/JChris246)
-- embedly
-    - [Embedded links may be for other extractors](https://github.com/yt-dlp/yt-dlp/commit/87ebab0615b1bf9b14b478b055e7059d630b4833) by [pukkandan](https://github.com/pukkandan)
-- freesound
-    - [Workaround invalid URL in webpage](https://github.com/yt-dlp/yt-dlp/commit/9cfdbcbf3f17be51f5b6bb9bb6d880b2f3d67362) ([#6147](https://github.com/yt-dlp/yt-dlp/issues/6147)) by [rebane2001](https://github.com/rebane2001)
-- generic
-    - [Avoid catastrophic backtracking in KVS regex](https://github.com/yt-dlp/yt-dlp/commit/8aa0bd5d10627ece3c1815c01d02fb8bf22847a7) by [bashonly](https://github.com/bashonly)
-- goplay
-    - [Use new API](https://github.com/yt-dlp/yt-dlp/commit/d27bde98832e3b7ffb39f3cf6346011b97bb3bc3) ([#6151](https://github.com/yt-dlp/yt-dlp/issues/6151)) by [jeroenj](https://github.com/jeroenj)
-- hidive
-    - [Fix subtitles and age-restriction](https://github.com/yt-dlp/yt-dlp/commit/7708df8da05c94270b43e0630e4e20f6d2d62c55) ([#5828](https://github.com/yt-dlp/yt-dlp/issues/5828)) by [chexxor](https://github.com/chexxor)
-- huya
-    - [Support HD streams](https://github.com/yt-dlp/yt-dlp/commit/fbbb5508ea98ed8709847f5ecced7d70ff05e0ee) ([#6172](https://github.com/yt-dlp/yt-dlp/issues/6172)) by [felixonmars](https://github.com/felixonmars)
-- hypergryph
-    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/31c279a2a2c2ef402a9e6dad9992b310d16439a6) ([#6094](https://github.com/yt-dlp/yt-dlp/issues/6094)) by [bashonly](https://github.com/bashonly), [HobbyistDev](https://github.com/HobbyistDev)
-- moviepilot
-    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/c62e64cf0122e52fa2175dd1b004ca6b8e1d82af) ([#5954](https://github.com/yt-dlp/yt-dlp/issues/5954)) by [panatexxa](https://github.com/panatexxa)
-- nbc
-    - [Fix XML parsing](https://github.com/yt-dlp/yt-dlp/commit/176a068cde4f2d9dfa0336168caead0b1edcb8ac) by [bashonly](https://github.com/bashonly)
-    - [Fix `NBC` and `NBCStations` extractors](https://github.com/yt-dlp/yt-dlp/commit/cb73b8460c3ce6d37ab651a4e44bb23b10056154) ([#6033](https://github.com/yt-dlp/yt-dlp/issues/6033)) by [bashonly](https://github.com/bashonly)
-- nebula
-    - [Remove broken cookie support](https://github.com/yt-dlp/yt-dlp/commit/d50ea3ce5abc3b0defc0e5d1e22b22ce9b01b07b) ([#5979](https://github.com/yt-dlp/yt-dlp/issues/5979)) by [hheimbuerger](https://github.com/hheimbuerger)
-- nfl
-    - [Add `NFLPlus` extractors](https://github.com/yt-dlp/yt-dlp/commit/8b37c58f8b5494504acdb5ebe3f8bbd26230f725) ([#6222](https://github.com/yt-dlp/yt-dlp/issues/6222)) by [bashonly](https://github.com/bashonly)
-- niconico
-    - [Add support for like history](https://github.com/yt-dlp/yt-dlp/commit/3b161265add30613bde2e46fca214fe94d09e651) ([#5705](https://github.com/yt-dlp/yt-dlp/issues/5705)) by [Matumo](https://github.com/Matumo), [pukkandan](https://github.com/pukkandan)
-- nitter
-    - [Update instance list](https://github.com/yt-dlp/yt-dlp/commit/a9189510baadf0dccd2d4d363bc6f3a441128bb0) ([#6236](https://github.com/yt-dlp/yt-dlp/issues/6236)) by [OIRNOIR](https://github.com/OIRNOIR)
-- npo
-    - [Fix extractor and add HD support](https://github.com/yt-dlp/yt-dlp/commit/cc2389c8ac72a514d4e002a0f6ca5a7d65c7eff0) ([#6155](https://github.com/yt-dlp/yt-dlp/issues/6155)) by [seproDev](https://github.com/seproDev)
-- nzonscreen
-    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/d3bb187f01e1e30db05e639fc23a2e1935d777fe) ([#6208](https://github.com/yt-dlp/yt-dlp/issues/6208)) by [gregsadetsky](https://github.com/gregsadetsky), [pukkandan](https://github.com/pukkandan)
-- odkmedia
-    - [Add `OnDemandChinaEpisodeIE`](https://github.com/yt-dlp/yt-dlp/commit/10fd9e6ee833c88edf6c633f864f42843a708d32) ([#6116](https://github.com/yt-dlp/yt-dlp/issues/6116)) by [HobbyistDev](https://github.com/HobbyistDev), [pukkandan](https://github.com/pukkandan)
-- pornez
-    - [Handle relative URLs in iframe](https://github.com/yt-dlp/yt-dlp/commit/f7efe6dc958eb0689cb9534ff0b4e592040be8df) ([#6171](https://github.com/yt-dlp/yt-dlp/issues/6171)) by [JChris246](https://github.com/JChris246)
-- radiko
-    - [Fix format sorting for Time Free](https://github.com/yt-dlp/yt-dlp/commit/203a06f8554df6db07d8f20f465ecbfe8a14e591) ([#6159](https://github.com/yt-dlp/yt-dlp/issues/6159)) by [road-master](https://github.com/road-master)
-- rcs
-    - [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/c6b657867ad68af6b930ed0aa11ec5d93ee187b7) ([#5700](https://github.com/yt-dlp/yt-dlp/issues/5700)) by [nixxo](https://github.com/nixxo), [pukkandan](https://github.com/pukkandan)
-- reddit
-    - [Support user posts](https://github.com/yt-dlp/yt-dlp/commit/c77df98b1a477a020a57141464d10c0f4d0fdbc9) ([#6173](https://github.com/yt-dlp/yt-dlp/issues/6173)) by [OMEGARAZER](https://github.com/OMEGARAZER)
-- rozhlas
-    - [Add extractor RozhlasVltavaIE](https://github.com/yt-dlp/yt-dlp/commit/355d781bed497cbcb254bf2a2737b83fa51c84ea) ([#5951](https://github.com/yt-dlp/yt-dlp/issues/5951)) by [amra](https://github.com/amra)
-- rumble
-    - [Fix format sorting](https://github.com/yt-dlp/yt-dlp/commit/acacb57c7e173b93c6e0f0c43e61b9b2912719d8) by [pukkandan](https://github.com/pukkandan)
-- servus
-    - [Rewrite extractor](https://github.com/yt-dlp/yt-dlp/commit/f40e32fb1ac67be5bdbc8e32a3c235abfc4be260) ([#6036](https://github.com/yt-dlp/yt-dlp/issues/6036)) by [Ashish0804](https://github.com/Ashish0804), [FrankZ85](https://github.com/FrankZ85), [StefanLobbenmeier](https://github.com/StefanLobbenmeier)
-- slideslive
-    - [Fix slides and chapters/duration](https://github.com/yt-dlp/yt-dlp/commit/5ab3534d44231f7711398bc3cfc520e2efd09f50) ([#6024](https://github.com/yt-dlp/yt-dlp/issues/6024)) by [bashonly](https://github.com/bashonly)
-- sportdeutschland
-    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/5e1a54f63e393c218a40949012ff0de0ce63cb15) ([#6041](https://github.com/yt-dlp/yt-dlp/issues/6041)) by [FriedrichRehren](https://github.com/FriedrichRehren)
-- stripchat
-    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/7d5f919bad07017f4b39b55725491b1e9717d47a) ([#5985](https://github.com/yt-dlp/yt-dlp/issues/5985)) by [bashonly](https://github.com/bashonly), [JChris246](https://github.com/JChris246)
-- tempo
-    - [Add IVXPlayer extractor](https://github.com/yt-dlp/yt-dlp/commit/30031be974d210f451100339699ef03b0ddb5f10) ([#5837](https://github.com/yt-dlp/yt-dlp/issues/5837)) by [HobbyistDev](https://github.com/HobbyistDev)
-- tnaflix
-    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/989f47b6315541989bb507f26b431d9586430995) ([#6086](https://github.com/yt-dlp/yt-dlp/issues/6086)) by [bashonly](https://github.com/bashonly), [oxamun](https://github.com/oxamun)
-- tvp
-    - [Support `stream.tvp.pl`](https://github.com/yt-dlp/yt-dlp/commit/a31d0fa6c315b1145d682361149003d98f1e3782) ([#6139](https://github.com/yt-dlp/yt-dlp/issues/6139)) by [selfisekai](https://github.com/selfisekai)
-- twitter
-    - [Fix `--no-playlist` and add media `view_count` when using GraphQL](https://github.com/yt-dlp/yt-dlp/commit/b6795fd310f1dd61dddc9fd08e52fe485bdc8a3e) ([#6211](https://github.com/yt-dlp/yt-dlp/issues/6211)) by [Grub4K](https://github.com/Grub4K)
-    - [Fix graphql extraction on some tweets](https://github.com/yt-dlp/yt-dlp/commit/7543c9c99bcb116b085fdb1f41b84a0ead04c05d) ([#6075](https://github.com/yt-dlp/yt-dlp/issues/6075)) by [selfisekai](https://github.com/selfisekai)
-- txxx
-    - [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/389896df85ed14eaf74f72531da6c4491d6b73b0) ([#5240](https://github.com/yt-dlp/yt-dlp/issues/5240)) by [chio0hai](https://github.com/chio0hai)
-- vimeo
-    - [Fix `playerConfig` extraction](https://github.com/yt-dlp/yt-dlp/commit/c0cd13fb1c71b842c3d272d0273c03542b467766) ([#6203](https://github.com/yt-dlp/yt-dlp/issues/6203)) by [bashonly](https://github.com/bashonly), [LeoniePhiline](https://github.com/LeoniePhiline)
-- viu
-    - [Add `ViuOTTIndonesiaIE` extractor](https://github.com/yt-dlp/yt-dlp/commit/72671a212d7c939329cb5d34335fa089dd3acbd3) ([#6099](https://github.com/yt-dlp/yt-dlp/issues/6099)) by [HobbyistDev](https://github.com/HobbyistDev)
-- vk
-    - [Fix playlists for new API](https://github.com/yt-dlp/yt-dlp/commit/a9c685453f7019bee94170f936619c6db76c964e) ([#6122](https://github.com/yt-dlp/yt-dlp/issues/6122)) by [the-marenga](https://github.com/the-marenga)
-- vlive
-    - [Replace with `VLiveWebArchiveIE`](https://github.com/yt-dlp/yt-dlp/commit/b3eaab7ca2e118d4db73dcb44afd9c8717db8b67) ([#6196](https://github.com/yt-dlp/yt-dlp/issues/6196)) by [seproDev](https://github.com/seproDev)
-- vocaroo
-    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/e4a8b1769e19755acba6d8f212208359905a3159) ([#6117](https://github.com/yt-dlp/yt-dlp/issues/6117)) by [qbnu](https://github.com/qbnu), [SuperSonicHub1](https://github.com/SuperSonicHub1)
-- wrestleuniverse
-    - [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/e61acb40b2cb6ef45508d72235026d458c9d5dff) ([#6158](https://github.com/yt-dlp/yt-dlp/issues/6158)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
-- ximalaya
-    - [Update album `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/417cdaae08fc447c9d15c53a88e2e9a027cdbf0a) ([#6110](https://github.com/yt-dlp/yt-dlp/issues/6110)) by [carusocr](https://github.com/carusocr)
-- yappy
-    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/361630015535026712bdb67f804a15b65ff9ee7e) ([#6111](https://github.com/yt-dlp/yt-dlp/issues/6111)) by [HobbyistDev](https://github.com/HobbyistDev)
-- youtube
-    - [Add hyperpipe instances](https://github.com/yt-dlp/yt-dlp/commit/78a78fa74dbc888d20f1b65e1382bf99131597d5) ([#6020](https://github.com/yt-dlp/yt-dlp/issues/6020)) by [Generator](https://github.com/Generator)
-    - [Fix `uploader_id` extraction](https://github.com/yt-dlp/yt-dlp/commit/149eb0bbf34fa8fdf8d1e2aa28e17479d099e26b) by [bashonly](https://github.com/bashonly)
-    - [Handle `consent.youtube`](https://github.com/yt-dlp/yt-dlp/commit/b032ff0f032512bd6fc70c9c1994d906eacc06cb) by [pukkandan](https://github.com/pukkandan)
-    - [Support `/live/` URL](https://github.com/yt-dlp/yt-dlp/commit/dad2210c0cb9cf03702a9511817ee5ec646d7bc8) by [pukkandan](https://github.com/pukkandan)
-    - [Update invidious and piped instances](https://github.com/yt-dlp/yt-dlp/commit/05799a48c7dec12b34c8bf951c8d2eceedda59f8) ([#6030](https://github.com/yt-dlp/yt-dlp/issues/6030)) by [rohieb](https://github.com/rohieb)
-    - [`uploader_id` includes `@` with handle](https://github.com/yt-dlp/yt-dlp/commit/c61cf091a54d3aa3c611722035ccde5ecfe981bb) by [bashonly](https://github.com/bashonly)
-- zdf
-    - [Use android API endpoint for UHD downloads](https://github.com/yt-dlp/yt-dlp/commit/0fe87a8730638490415d630f48e61d264d89c358) ([#6150](https://github.com/yt-dlp/yt-dlp/issues/6150)) by [seproDev](https://github.com/seproDev)
-### Downloader changes
-- hls
-    - [Allow extractors to provide AES key](https://github.com/yt-dlp/yt-dlp/commit/7e68567e508168b345266c0c19812ad50a829eaa) ([#6158](https://github.com/yt-dlp/yt-dlp/issues/6158)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
-### Postprocessor changes
-- extractaudio
-    - [Handle outtmpl without ext](https://github.com/yt-dlp/yt-dlp/commit/f737fb16d8234408c85bc189ccc926fea000515b) ([#6005](https://github.com/yt-dlp/yt-dlp/issues/6005)) by [carusocr](https://github.com/carusocr)
-- pyinst
-    - [Fix for pyinstaller 5.8](https://github.com/yt-dlp/yt-dlp/commit/2e269bd998c61efaf7500907d114a56e5e83e65e) by [pukkandan](https://github.com/pukkandan)
-### Misc. changes
-- build
-    - [Update pyinstaller](https://github.com/yt-dlp/yt-dlp/commit/365b9006051ac7d735c20bb63c4907b758233048) by [pukkandan](https://github.com/pukkandan)
-- cleanup
-    - Miscellaneous: [76c9c52](https://github.com/yt-dlp/yt-dlp/commit/76c9c523071150053df7b56956646b680b6a6e05) by [pukkandan](https://github.com/pukkandan)
-- devscripts
-    - [Provide pyinstaller hooks](https://github.com/yt-dlp/yt-dlp/commit/acb1042a9ffa8769fe691beac1011d6da1fcf321) by [pukkandan](https://github.com/pukkandan)
-- pyinstaller
-    - [Analyze sub-modules of `Cryptodome`](https://github.com/yt-dlp/yt-dlp/commit/b85faf6ffb700058e774e99c04304a7a9257cdd0) by [pukkandan](https://github.com/pukkandan)
-
 ### 2023.01.06
 
 * Fix config locations by [Grub4k](https://github.com/Grub4k), [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)

From 18d295c9e0f95adc179eef345b7af64d6372db78 Mon Sep 17 00:00:00 2001
From: Zhong Lufan <lufanzhong@gmail.com>
Date: Fri, 17 Feb 2023 21:11:16 +0800
Subject: [PATCH 089/871] [extractor/tencent] Add more formats and info (#5950)

Authored by: Hill-98
---
 yt_dlp/extractor/tencent.py | 88 ++++++++++++++++++++++++++-----------
 1 file changed, 63 insertions(+), 25 deletions(-)

diff --git a/yt_dlp/extractor/tencent.py b/yt_dlp/extractor/tencent.py
index 44cae0472..42a2175b0 100644
--- a/yt_dlp/extractor/tencent.py
+++ b/yt_dlp/extractor/tencent.py
@@ -8,6 +8,7 @@
 from ..aes import aes_cbc_encrypt_bytes
 from ..utils import (
     ExtractorError,
+    float_or_none,
     determine_ext,
     int_or_none,
     js_to_json,
@@ -19,6 +20,16 @@
 class TencentBaseIE(InfoExtractor):
     """Subclasses must set _API_URL, _APP_VERSION, _PLATFORM, _HOST, _REFERER"""
 
+    def _check_api_response(self, api_response):
+        msg = api_response.get('msg')
+        if api_response.get('code') != '0.0' and msg is not None:
+            if msg in (
+                '您所在区域暂无此内容版权（如设置VPN请关闭后重试）',
+                'This content is not available in your area due to copyright restrictions. Please choose other videos.'
+            ):
+                self.raise_geo_restricted()
+            raise ExtractorError(f'Tencent said: {msg}')
+
     def _get_ckey(self, video_id, url, guid):
         ua = self.get_param('http_headers')['User-Agent']
 
@@ -47,6 +58,11 @@ def _get_video_api_response(self, video_url, video_id, series_id, subtitle_forma
             'sphttps': '1',  # Enable HTTPS
             'otype': 'json',
             'spwm': '1',
+            'hevclv': '28',  # Enable HEVC
+            'drm': '40',  # Enable DRM
+            # For HDR
+            'spvideo': '4',
+            'spsfrhdr': '100',
             # For SHD
             'host': self._HOST,
             'referer': self._REFERER,
@@ -63,7 +79,6 @@ def _get_video_api_response(self, video_url, video_id, series_id, subtitle_forma
 
     def _extract_video_formats_and_subtitles(self, api_response, video_id):
         video_response = api_response['vl']['vi'][0]
-        video_width, video_height = video_response.get('vw'), video_response.get('vh')
 
         formats, subtitles = [], {}
         for video_format in video_response['ul']['ui']:
@@ -71,47 +86,61 @@ def _extract_video_formats_and_subtitles(self, api_response, video_id):
                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
                     video_format['url'] + traverse_obj(video_format, ('hls', 'pt'), default=''),
                     video_id, 'mp4', fatal=False)
-                for f in fmts:
-                    f.update({'width': video_width, 'height': video_height})
 
                 formats.extend(fmts)
                 self._merge_subtitles(subs, target=subtitles)
             else:
                 formats.append({
                     'url': f'{video_format["url"]}{video_response["fn"]}?vkey={video_response["fvkey"]}',
-                    'width': video_width,
-                    'height': video_height,
                     'ext': 'mp4',
                 })
 
+        identifier = video_response.get('br')
+        format_response = traverse_obj(
+            api_response, ('fl', 'fi', lambda _, v: v['br'] == identifier),
+            expected_type=dict, get_all=False) or {}
+        common_info = {
+            'width': video_response.get('vw'),
+            'height': video_response.get('vh'),
+            'abr': float_or_none(format_response.get('audiobandwidth'), scale=1000),
+            'vbr': float_or_none(format_response.get('bandwidth'), scale=1000),
+            'fps': format_response.get('vfps'),
+            'format': format_response.get('sname'),
+            'format_id': format_response.get('name'),
+            'format_note': format_response.get('resolution'),
+            'dynamic_range': {'hdr10': 'hdr10'}.get(format_response.get('name'), 'sdr'),
+            'has_drm': format_response.get('drm', 0) != 0,
+        }
+        for f in formats:
+            f.update(common_info)
+
         return formats, subtitles
 
-    def _extract_video_native_subtitles(self, api_response, subtitles_format):
+    def _extract_video_native_subtitles(self, api_response):
         subtitles = {}
         for subtitle in traverse_obj(api_response, ('sfl', 'fi')) or ():
             subtitles.setdefault(subtitle['lang'].lower(), []).append({
                 'url': subtitle['url'],
-                'ext': subtitles_format,
+                'ext': 'srt' if subtitle.get('captionType') == 1 else 'vtt',
                 'protocol': 'm3u8_native' if determine_ext(subtitle['url']) == 'm3u8' else 'http',
             })
 
         return subtitles
 
     def _extract_all_video_formats_and_subtitles(self, url, video_id, series_id):
+        api_responses = [self._get_video_api_response(url, video_id, series_id, 'srt', 'hls', 'hd')]
+        self._check_api_response(api_responses[0])
+        qualities = traverse_obj(api_responses, (0, 'fl', 'fi', ..., 'name')) or ('shd', 'fhd')
+        for q in qualities:
+            if q not in ('ld', 'sd', 'hd'):
+                api_responses.append(self._get_video_api_response(
+                    url, video_id, series_id, 'vtt', 'hls', q))
+                self._check_api_response(api_responses[-1])
+
         formats, subtitles = [], {}
-        for video_format, subtitle_format, video_quality in (
-                # '': 480p, 'shd': 720p, 'fhd': 1080p
-                ('mp4', 'srt', ''), ('hls', 'vtt', 'shd'), ('hls', 'vtt', 'fhd')):
-            api_response = self._get_video_api_response(
-                url, video_id, series_id, subtitle_format, video_format, video_quality)
-
-            if api_response.get('em') != 0 and api_response.get('exem') != 0:
-                if '您所在区域暂无此内容版权' in api_response.get('msg'):
-                    self.raise_geo_restricted()
-                raise ExtractorError(f'Tencent said: {api_response.get("msg")}')
-
+        for api_response in api_responses:
             fmts, subs = self._extract_video_formats_and_subtitles(api_response, video_id)
-            native_subtitles = self._extract_video_native_subtitles(api_response, subtitle_format)
+            native_subtitles = self._extract_video_native_subtitles(api_response)
 
             formats.extend(fmts)
             self._merge_subtitles(subs, native_subtitles, target=subtitles)
@@ -120,7 +149,7 @@ def _extract_all_video_formats_and_subtitles(self, url, video_id, series_id):
 
     def _get_clean_title(self, title):
         return re.sub(
-            r'\s*[_\-]\s*(?:Watch online|腾讯视频|(?:高清)?1080P在线观看平台).*?$',
+            r'\s*[_\-]\s*(?:Watch online|Watch HD Video Online|WeTV|腾讯视频|(?:高清)?1080P在线观看平台).*?$',
             '', title or '').strip() or None
 
 
@@ -147,27 +176,29 @@ class VQQVideoIE(VQQBaseIE):
 
     _TESTS = [{
         'url': 'https://v.qq.com/x/page/q326831cny0.html',
-        'md5': '826ef93682df09e3deac4a6e6e8cdb6e',
+        'md5': '84568b3722e15e9cd023b5594558c4a7',
         'info_dict': {
             'id': 'q326831cny0',
             'ext': 'mp4',
             'title': '我是选手：雷霆裂阵，终极时刻',
             'description': 'md5:e7ed70be89244017dac2a835a10aeb1e',
             'thumbnail': r're:^https?://[^?#]+q326831cny0',
+            'format_id': r're:^shd',
         },
     }, {
         'url': 'https://v.qq.com/x/page/o3013za7cse.html',
-        'md5': 'b91cbbeada22ef8cc4b06df53e36fa21',
+        'md5': 'cc431c4f9114a55643893c2c8ebf5592',
         'info_dict': {
             'id': 'o3013za7cse',
             'ext': 'mp4',
             'title': '欧阳娜娜VLOG',
             'description': 'md5:29fe847497a98e04a8c3826e499edd2e',
             'thumbnail': r're:^https?://[^?#]+o3013za7cse',
+            'format_id': r're:^shd',
         },
     }, {
         'url': 'https://v.qq.com/x/cover/7ce5noezvafma27/a00269ix3l8.html',
-        'md5': '71459c5375c617c265a22f083facce67',
+        'md5': '87968df6238a65d2478f19c25adf850b',
         'info_dict': {
             'id': 'a00269ix3l8',
             'ext': 'mp4',
@@ -175,10 +206,11 @@ class VQQVideoIE(VQQBaseIE):
             'description': 'md5:8cae3534327315b3872fbef5e51b5c5b',
             'thumbnail': r're:^https?://[^?#]+7ce5noezvafma27',
             'series': '鸡毛飞上天',
+            'format_id': r're:^shd',
         },
     }, {
         'url': 'https://v.qq.com/x/cover/mzc00200p29k31e/s0043cwsgj0.html',
-        'md5': '96b9fd4a189fdd4078c111f21d7ac1bc',
+        'md5': 'fadd10bf88aec3420f06f19ee1d24c5b',
         'info_dict': {
             'id': 's0043cwsgj0',
             'ext': 'mp4',
@@ -186,6 +218,7 @@ class VQQVideoIE(VQQBaseIE):
             'description': 'md5:1d8c3a0b8729ae3827fa5b2d3ebd5213',
             'thumbnail': r're:^https?://[^?#]+s0043cwsgj0',
             'series': '青年理工工作者生活研究所',
+            'format_id': r're:^shd',
         },
     }, {
         # Geo-restricted to China
@@ -319,6 +352,7 @@ class WeTvEpisodeIE(WeTvBaseIE):
             'episode': 'Episode 1',
             'episode_number': 1,
             'duration': 2835,
+            'format_id': r're:^shd',
         },
     }, {
         'url': 'https://wetv.vip/en/play/u37kgfnfzs73kiu/p0039b9nvik',
@@ -333,6 +367,7 @@ class WeTvEpisodeIE(WeTvBaseIE):
             'episode': 'Episode 1',
             'episode_number': 1,
             'duration': 2454,
+            'format_id': r're:^shd',
         },
     }, {
         'url': 'https://wetv.vip/en/play/lcxgwod5hapghvw-WeTV-PICK-A-BOO/i0042y00lxp-Zhao-Lusi-Describes-The-First-Experiences-She-Had-In-Who-Rules-The-World-%7C-WeTV-PICK-A-BOO',
@@ -342,11 +377,12 @@ class WeTvEpisodeIE(WeTvBaseIE):
             'ext': 'mp4',
             'title': 'md5:f7a0857dbe5fbbe2e7ad630b92b54e6a',
             'description': 'md5:76260cb9cdc0ef76826d7ca9d92fadfa',
-            'thumbnail': r're:^https?://[^?#]+lcxgwod5hapghvw',
+            'thumbnail': r're:^https?://[^?#]+i0042y00lxp',
             'series': 'WeTV PICK-A-BOO',
             'episode': 'Episode 0',
             'episode_number': 0,
             'duration': 442,
+            'format_id': r're:^shd',
         },
     }]
 
@@ -406,6 +442,7 @@ class IflixEpisodeIE(IflixBaseIE):
             'episode': 'Episode 1',
             'episode_number': 1,
             'duration': 2639,
+            'format_id': r're:^shd',
         },
     }, {
         'url': 'https://www.iflix.com/en/play/fvvrcc3ra9lbtt1-Take-My-Brother-Away/i0029sd3gm1-EP1%EF%BC%9ATake-My-Brother-Away',
@@ -420,6 +457,7 @@ class IflixEpisodeIE(IflixBaseIE):
             'episode': 'Episode 1',
             'episode_number': 1,
             'duration': 228,
+            'format_id': r're:^shd',
         },
     }]
 

From da8e2912b165005f76779a115a071cd6132ceedf Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Thu, 23 Feb 2023 04:18:45 +0100
Subject: [PATCH 090/871] [utils] `Popen`: Shim undocumented `text_mode`
 property

Fixes #6317

Authored by: Grub4K
---
 yt_dlp/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 994239897..4fe718bf0 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -879,6 +879,7 @@ def __init__(self, *args, env=None, text=False, **kwargs):
             env = os.environ.copy()
         self._fix_pyinstaller_ld_path(env)
 
+        self.__text_mode = kwargs.get('encoding') or kwargs.get('errors') or text or kwargs.get('universal_newlines')
         if text is True:
             kwargs['universal_newlines'] = True  # For 3.6 compatibility
             kwargs.setdefault('encoding', 'utf-8')
@@ -900,7 +901,7 @@ def kill(self, *, timeout=0):
     @classmethod
     def run(cls, *args, timeout=None, **kwargs):
         with cls(*args, **kwargs) as proc:
-            default = '' if proc.text_mode else b''
+            default = '' if proc.__text_mode else b''
             stdout, stderr = proc.communicate_or_kill(timeout=timeout)
             return stdout or default, stderr or default, proc.returncode
 

From cc09083636ce21e58ff74f45eac2dbda507462b0 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 24 Feb 2023 10:39:43 +0530
Subject: [PATCH 091/871] [utils] `LenientJSONDecoder`: Parse unclosed objects

---
 yt_dlp/utils.py | 38 ++++++++++++++++++++++++++++++--------
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 4fe718bf0..9ff096433 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -593,21 +593,43 @@ def clean_html(html):
 
 
 class LenientJSONDecoder(json.JSONDecoder):
-    def __init__(self, *args, transform_source=None, ignore_extra=False, **kwargs):
+    # TODO: Write tests
+    def __init__(self, *args, transform_source=None, ignore_extra=False, close_objects=0, **kwargs):
         self.transform_source, self.ignore_extra = transform_source, ignore_extra
+        self._close_attempts = 2 * close_objects
         super().__init__(*args, **kwargs)
 
+    @staticmethod
+    def _close_object(err):
+        doc = err.doc[:err.pos]
+        # We need to add comma first to get the correct error message
+        if err.msg.startswith('Expecting \',\''):
+            return doc + ','
+        elif not doc.endswith(','):
+            return
+
+        if err.msg.startswith('Expecting property name'):
+            return doc[:-1] + '}'
+        elif err.msg.startswith('Expecting value'):
+            return doc[:-1] + ']'
+
     def decode(self, s):
         if self.transform_source:
             s = self.transform_source(s)
-        try:
-            if self.ignore_extra:
-                return self.raw_decode(s.lstrip())[0]
-            return super().decode(s)
-        except json.JSONDecodeError as e:
-            if e.pos is not None:
+        for attempt in range(self._close_attempts + 1):
+            try:
+                if self.ignore_extra:
+                    return self.raw_decode(s.lstrip())[0]
+                return super().decode(s)
+            except json.JSONDecodeError as e:
+                if e.pos is None:
+                    raise
+                elif attempt < self._close_attempts:
+                    s = self._close_object(e)
+                    if s is not None:
+                        continue
                 raise type(e)(f'{e.msg} in {s[e.pos-10:e.pos+10]!r}', s, e.pos)
-            raise
+        assert False, 'Too many attempts to decode JSON'
 
 
 def sanitize_open(filename, open_mode):

From 43a3eaf96393b712d60cbcf5c6cb1e90ed7f42f5 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 26 Feb 2023 10:16:30 +0530
Subject: [PATCH 092/871] [extractor] Fix DRM detection in m3u8

Fixes https://github.com/ytdl-org/youtube-dl/issues/31693#issuecomment-1445202857
---
 yt_dlp/extractor/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index ebacc87bc..86bef173f 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -2063,6 +2063,7 @@ def extract_media(x_media_line):
                     'protocol': entry_protocol,
                     'preference': preference,
                     'quality': quality,
+                    'has_drm': has_drm,
                     'vcodec': 'none' if media_type == 'AUDIO' else None,
                 } for idx in _extract_m3u8_playlist_indices(manifest_url))
 
@@ -2122,6 +2123,7 @@ def build_stream_name():
                         'protocol': entry_protocol,
                         'preference': preference,
                         'quality': quality,
+                        'has_drm': has_drm,
                     }
                     resolution = last_stream_inf.get('RESOLUTION')
                     if resolution:

From 8e9fe43cd393e69fa49b3d842aa3180c1d105b8f Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 26 Feb 2023 10:27:04 +0530
Subject: [PATCH 093/871] [extractor/generic] Handle basic-auth when checking
 redirects

Closes #6352
---
 yt_dlp/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 55e55d524..d76ef3e31 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -15,6 +15,7 @@
     UnsupportedError,
     determine_ext,
     dict_get,
+    extract_basic_auth,
     format_field,
     int_or_none,
     is_html,
@@ -2372,9 +2373,8 @@ def _real_extract(self, url):
             **smuggled_data.get('http_headers', {})
         })
         new_url = full_response.geturl()
-        if new_url == urllib.parse.urlparse(url)._replace(scheme='https').geturl():
-            url = new_url
-        elif url != new_url:
+        url = urllib.parse.urlparse(url)._replace(scheme=urllib.parse.urlparse(new_url).scheme).geturl()
+        if new_url != extract_basic_auth(url)[0]:
             self.report_following_redirect(new_url)
             if force_videoid:
                 new_url = smuggle_url(new_url, {'force_videoid': force_videoid})

From 4d248e29d20d983ededab0b03d4fe69dff9eb4ed Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 28 Feb 2023 23:09:20 +0530
Subject: [PATCH 094/871] [extractor/GoogleDrive] Fix some audio

Only those with source url, but no confirmation page
---
 yt_dlp/extractor/googledrive.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/googledrive.py b/yt_dlp/extractor/googledrive.py
index e027ea7c4..9e2ccde00 100644
--- a/yt_dlp/extractor/googledrive.py
+++ b/yt_dlp/extractor/googledrive.py
@@ -3,8 +3,8 @@
 from .common import InfoExtractor
 from ..compat import compat_parse_qs
 from ..utils import (
-    determine_ext,
     ExtractorError,
+    determine_ext,
     get_element_by_class,
     int_or_none,
     lowercase_escape,
@@ -163,15 +163,13 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
         video_info = compat_parse_qs(self._download_webpage(
             'https://drive.google.com/get_video_info',
-            video_id, query={'docid': video_id}))
+            video_id, 'Downloading video webpage', query={'docid': video_id}))
 
         def get_value(key):
             return try_get(video_info, lambda x: x[key][0])
 
         reason = get_value('reason')
         title = get_value('title')
-        if not title and reason:
-            raise ExtractorError(reason, expected=True)
 
         formats = []
         fmt_stream_map = (get_value('fmt_stream_map') or '').split(',')
@@ -216,6 +214,11 @@ def request_source_file(source_url, kind):
         urlh = request_source_file(source_url, 'source')
         if urlh:
             def add_source_format(urlh):
+                nonlocal title
+                if not title:
+                    title = self._search_regex(
+                        r'\bfilename="([^"]+)"', urlh.headers.get('Content-Disposition'),
+                        'title', default=None)
                 formats.append({
                     # Use redirect URLs as download URLs in order to calculate
                     # correct cookies in _calc_cookies.
@@ -251,7 +254,10 @@ def add_source_format(urlh):
                             or 'unable to extract confirmation code')
 
         if not formats and reason:
-            self.raise_no_formats(reason, expected=True)
+            if title:
+                self.raise_no_formats(reason, expected=True)
+            else:
+                raise ExtractorError(reason, expected=True)
 
         hl = get_value('hl')
         subtitles_id = None

From 5038f6d713303e0967d002216e7a88652401c22a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 28 Feb 2023 23:03:44 +0530
Subject: [PATCH 095/871] [extractor/youtube] Construct dash formats with
 `range` query

Closes #6369
---
 yt_dlp/extractor/youtube.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index be82bc689..0227a1f83 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3776,10 +3776,19 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             if no_video:
                 dct['abr'] = tbr
             if no_audio or no_video:
-                dct['downloader_options'] = {
-                    # Youtube throttles chunks >~10M
-                    'http_chunk_size': 10485760,
-                }
+                CHUNK_SIZE = 10 << 20
+                dct.update({
+                    'request_data': b'x',
+                    'protocol': 'http_dash_segments',
+                    'fragments': [{
+                        'url': update_url_query(dct['url'], {
+                            'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
+                        })
+                    } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
+                } if dct['filesize'] else {
+                    'downloader_options': {'http_chunk_size': CHUNK_SIZE}  # No longer useful?
+                })
+
                 if dct.get('ext'):
                     dct['container'] = dct['ext'] + '_dash'
 

From b059188383eee4fa336ef728dda3ff4bb7335625 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 28 Feb 2023 22:32:20 +0530
Subject: [PATCH 096/871] [plugins] Don't look in `.egg` directories

Closes #6306
---
 yt_dlp/plugins.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/plugins.py b/yt_dlp/plugins.py
index 6eecdb4d0..6422c7a51 100644
--- a/yt_dlp/plugins.py
+++ b/yt_dlp/plugins.py
@@ -88,7 +88,7 @@ def _get_package_paths(*root_paths, containing_folder='plugins'):
             candidate = path / parts
             if candidate.is_dir():
                 yield candidate
-            elif path.suffix in ('.zip', '.egg', '.whl'):
+            elif path.suffix in ('.zip', '.egg', '.whl') and path.is_file():
                 if parts in dirs_in_zip(path):
                     yield candidate
 

From 65f6e807804d2af5e00f2aecd72bfc43af19324a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 28 Feb 2023 23:10:54 +0530
Subject: [PATCH 097/871] [dependencies] Simplify `Cryptodome`

Closes #6292, closes #6272, closes #6338
---
 test/test_aes.py                    |  4 +--
 yt_dlp/__pyinstaller/hook-yt_dlp.py | 28 +---------------
 yt_dlp/aes.py                       |  6 ++--
 yt_dlp/compat/_legacy.py            |  2 +-
 yt_dlp/compat/compat_utils.py       |  2 +-
 yt_dlp/dependencies/Cryptodome.py   | 50 ++++++++++++++++++-----------
 yt_dlp/dependencies/__init__.py     |  2 +-
 yt_dlp/downloader/hls.py            |  2 +-
 yt_dlp/extractor/bilibili.py        |  6 ++--
 yt_dlp/extractor/ivi.py             |  8 ++---
 yt_dlp/extractor/wrestleuniverse.py |  6 ++--
 11 files changed, 52 insertions(+), 64 deletions(-)

diff --git a/test/test_aes.py b/test/test_aes.py
index 18f15fecb..a26abfd7d 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -48,7 +48,7 @@ def test_cbc_decrypt(self):
         data = b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\x27\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd'
         decrypted = intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(data), self.key, self.iv))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
-        if Cryptodome:
+        if Cryptodome.AES:
             decrypted = aes_cbc_decrypt_bytes(data, intlist_to_bytes(self.key), intlist_to_bytes(self.iv))
             self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
@@ -78,7 +78,7 @@ def test_gcm_decrypt(self):
         decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify(
             bytes_to_intlist(data), self.key, bytes_to_intlist(authentication_tag), self.iv[:12]))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
-        if Cryptodome:
+        if Cryptodome.AES:
             decrypted = aes_gcm_decrypt_and_verify_bytes(
                 data, intlist_to_bytes(self.key), authentication_tag, intlist_to_bytes(self.iv[:12]))
             self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
diff --git a/yt_dlp/__pyinstaller/hook-yt_dlp.py b/yt_dlp/__pyinstaller/hook-yt_dlp.py
index 057cfef2f..63dcdffe0 100644
--- a/yt_dlp/__pyinstaller/hook-yt_dlp.py
+++ b/yt_dlp/__pyinstaller/hook-yt_dlp.py
@@ -1,30 +1,8 @@
-import ast
-import os
 import sys
-from pathlib import Path
 
 from PyInstaller.utils.hooks import collect_submodules
 
 
-def find_attribute_accesses(node, name, path=()):
-    if isinstance(node, ast.Attribute):
-        path = [*path, node.attr]
-        if isinstance(node.value, ast.Name) and node.value.id == name:
-            yield path[::-1]
-    for child in ast.iter_child_nodes(node):
-        yield from find_attribute_accesses(child, name, path)
-
-
-def collect_used_submodules(name, level):
-    for dirpath, _, filenames in os.walk(Path(__file__).parent.parent):
-        for filename in filenames:
-            if not filename.endswith('.py'):
-                continue
-            with open(Path(dirpath) / filename, encoding='utf8') as f:
-                for submodule in find_attribute_accesses(ast.parse(f.read()), name):
-                    yield '.'.join(submodule[:level])
-
-
 def pycryptodome_module():
     try:
         import Cryptodome  # noqa: F401
@@ -41,12 +19,8 @@ def pycryptodome_module():
 
 def get_hidden_imports():
     yield 'yt_dlp.compat._legacy'
+    yield pycryptodome_module()
     yield from collect_submodules('websockets')
-
-    crypto = pycryptodome_module()
-    for sm in set(collect_used_submodules('Cryptodome', 2)):
-        yield f'{crypto}.{sm}'
-
     # These are auto-detected, but explicitly add them just in case
     yield from ('mutagen', 'brotli', 'certifi')
 
diff --git a/yt_dlp/aes.py b/yt_dlp/aes.py
index deff0a2b3..b3a383cd9 100644
--- a/yt_dlp/aes.py
+++ b/yt_dlp/aes.py
@@ -5,14 +5,14 @@
 from .dependencies import Cryptodome
 from .utils import bytes_to_intlist, intlist_to_bytes
 
-if Cryptodome:
+if Cryptodome.AES:
     def aes_cbc_decrypt_bytes(data, key, iv):
         """ Decrypt bytes with AES-CBC using pycryptodome """
-        return Cryptodome.Cipher.AES.new(key, Cryptodome.Cipher.AES.MODE_CBC, iv).decrypt(data)
+        return Cryptodome.AES.new(key, Cryptodome.AES.MODE_CBC, iv).decrypt(data)
 
     def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce):
         """ Decrypt bytes with AES-GCM using pycryptodome """
-        return Cryptodome.Cipher.AES.new(key, Cryptodome.Cipher.AES.MODE_GCM, nonce).decrypt_and_verify(data, tag)
+        return Cryptodome.AES.new(key, Cryptodome.AES.MODE_GCM, nonce).decrypt_and_verify(data, tag)
 
 else:
     def aes_cbc_decrypt_bytes(data, key, iv):
diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py
index 84d749209..83bf869a8 100644
--- a/yt_dlp/compat/_legacy.py
+++ b/yt_dlp/compat/_legacy.py
@@ -32,9 +32,9 @@
 
 from . import compat_expanduser, compat_HTMLParseError, compat_realpath
 from .compat_utils import passthrough_module
-from ..dependencies import Cryptodome_AES as compat_pycrypto_AES  # noqa: F401
 from ..dependencies import brotli as compat_brotli  # noqa: F401
 from ..dependencies import websockets as compat_websockets  # noqa: F401
+from ..dependencies.Cryptodome import AES as compat_pycrypto_AES  # noqa: F401
 
 passthrough_module(__name__, '...utils', ('WINDOWS_VT_MODE', 'windows_enable_vt_mode'))
 
diff --git a/yt_dlp/compat/compat_utils.py b/yt_dlp/compat/compat_utils.py
index 8956b3bf1..3ca46d270 100644
--- a/yt_dlp/compat/compat_utils.py
+++ b/yt_dlp/compat/compat_utils.py
@@ -48,7 +48,7 @@ def passthrough_module(parent, child, allowed_attributes=(..., ), *, callback=la
     """Passthrough parent module into a child module, creating the parent if necessary"""
     def __getattr__(attr):
         if _is_package(parent):
-            with contextlib.suppress(ImportError):
+            with contextlib.suppress(ModuleNotFoundError):
                 return importlib.import_module(f'.{attr}', parent.__name__)
 
         ret = from_child(attr)
diff --git a/yt_dlp/dependencies/Cryptodome.py b/yt_dlp/dependencies/Cryptodome.py
index 2adc51374..a50bce4d4 100644
--- a/yt_dlp/dependencies/Cryptodome.py
+++ b/yt_dlp/dependencies/Cryptodome.py
@@ -1,8 +1,5 @@
 import types
 
-from ..compat import functools
-from ..compat.compat_utils import passthrough_module
-
 try:
     import Cryptodome as _parent
 except ImportError:
@@ -12,19 +9,36 @@
         _parent = types.ModuleType('no_Cryptodome')
         __bool__ = lambda: False
 
-passthrough_module(__name__, _parent, (..., '__version__'))
-del passthrough_module
-
-
-@property
-@functools.cache
-def _yt_dlp__identifier():
-    if _parent.__name__ == 'Crypto':
+__version__ = ''
+AES = PKCS1_v1_5 = Blowfish = PKCS1_OAEP = SHA1 = CMAC = RSA = None
+try:
+    if _parent.__name__ == 'Cryptodome':
+        from Cryptodome import __version__
+        from Cryptodome.Cipher import AES
+        from Cryptodome.Cipher import PKCS1_v1_5
+        from Cryptodome.Cipher import Blowfish
+        from Cryptodome.Cipher import PKCS1_OAEP
+        from Cryptodome.Hash import SHA1
+        from Cryptodome.Hash import CMAC
+        from Cryptodome.PublicKey import RSA
+    elif _parent.__name__ == 'Crypto':
+        from Crypto import __version__
         from Crypto.Cipher import AES
-        try:
-            # In pycrypto, mode defaults to ECB. See:
-            # https://www.pycryptodome.org/en/latest/src/vs_pycrypto.html#:~:text=not%20have%20ECB%20as%20default%20mode
-            AES.new(b'abcdefghijklmnop')
-        except TypeError:
-            return 'pycrypto'
-    return _parent.__name__
+        from Crypto.Cipher import PKCS1_v1_5
+        from Crypto.Cipher import Blowfish
+        from Crypto.Cipher import PKCS1_OAEP
+        from Crypto.Hash import SHA1
+        from Crypto.Hash import CMAC
+        from Crypto.PublicKey import RSA
+except ImportError:
+    __version__ = f'broken {__version__}'.strip()
+
+
+_yt_dlp__identifier = _parent.__name__
+if AES and _yt_dlp__identifier == 'Crypto':
+    try:
+        # In pycrypto, mode defaults to ECB. See:
+        # https://www.pycryptodome.org/en/latest/src/vs_pycrypto.html#:~:text=not%20have%20ECB%20as%20default%20mode
+        AES.new(b'abcdefghijklmnop')
+    except TypeError:
+        _yt_dlp__identifier = 'pycrypto'
diff --git a/yt_dlp/dependencies/__init__.py b/yt_dlp/dependencies/__init__.py
index c2214e6db..6e7d29c5c 100644
--- a/yt_dlp/dependencies/__init__.py
+++ b/yt_dlp/dependencies/__init__.py
@@ -73,7 +73,7 @@
 
 
 # Deprecated
-Cryptodome_AES = Cryptodome.Cipher.AES if Cryptodome else None
+Cryptodome_AES = Cryptodome.AES
 
 
 __all__ = [
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index 29d6f6241..f2868dc52 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -70,7 +70,7 @@ def real_download(self, filename, info_dict):
         can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
         if can_download:
             has_ffmpeg = FFmpegFD.available()
-            no_crypto = not Cryptodome and '#EXT-X-KEY:METHOD=AES-128' in s
+            no_crypto = not Cryptodome.AES and '#EXT-X-KEY:METHOD=AES-128' in s
             if no_crypto and has_ffmpeg:
                 can_download, message = False, 'The stream has AES-128 encryption and pycryptodomex is not available'
             elif no_crypto:
diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index f4180633a..2252840b3 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -894,15 +894,15 @@ def _parse_video_metadata(self, video_data):
         }
 
     def _perform_login(self, username, password):
-        if not Cryptodome:
+        if not Cryptodome.RSA:
             raise ExtractorError('pycryptodomex not found. Please install', expected=True)
 
         key_data = self._download_json(
             'https://passport.bilibili.tv/x/intl/passport-login/web/key?lang=en-US', None,
             note='Downloading login key', errnote='Unable to download login key')['data']
 
-        public_key = Cryptodome.PublicKey.RSA.importKey(key_data['key'])
-        password_hash = Cryptodome.Cipher.PKCS1_v1_5.new(public_key).encrypt((key_data['hash'] + password).encode('utf-8'))
+        public_key = Cryptodome.RSA.importKey(key_data['key'])
+        password_hash = Cryptodome.PKCS1_v1_5.new(public_key).encrypt((key_data['hash'] + password).encode('utf-8'))
         login_post = self._download_json(
             'https://passport.bilibili.tv/x/intl/passport-login/web/login/password?lang=en-US', None, data=urlencode_postdata({
                 'username': username,
diff --git a/yt_dlp/extractor/ivi.py b/yt_dlp/extractor/ivi.py
index 96220bea9..fa5ceec95 100644
--- a/yt_dlp/extractor/ivi.py
+++ b/yt_dlp/extractor/ivi.py
@@ -91,7 +91,7 @@ def _real_extract(self, url):
         for site in (353, 183):
             content_data = (data % site).encode()
             if site == 353:
-                if not Cryptodome:
+                if not Cryptodome.CMAC:
                     continue
 
                 timestamp = (self._download_json(
@@ -105,8 +105,8 @@ def _real_extract(self, url):
 
                 query = {
                     'ts': timestamp,
-                    'sign': Cryptodome.Hash.CMAC.new(self._LIGHT_KEY, timestamp.encode() + content_data,
-                                                     Cryptodome.Cipher.Blowfish).hexdigest(),
+                    'sign': Cryptodome.CMAC.new(self._LIGHT_KEY, timestamp.encode() + content_data,
+                                                Cryptodome.Blowfish).hexdigest(),
                 }
             else:
                 query = {}
@@ -126,7 +126,7 @@ def _real_extract(self, url):
                     extractor_msg = 'Video %s does not exist'
                 elif site == 353:
                     continue
-                elif not Cryptodome:
+                elif not Cryptodome.CMAC:
                     raise ExtractorError('pycryptodomex not found. Please install', expected=True)
                 elif message:
                     extractor_msg += ': ' + message
diff --git a/yt_dlp/extractor/wrestleuniverse.py b/yt_dlp/extractor/wrestleuniverse.py
index 78e7c83ab..5c6dec2c4 100644
--- a/yt_dlp/extractor/wrestleuniverse.py
+++ b/yt_dlp/extractor/wrestleuniverse.py
@@ -50,10 +50,10 @@ def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={
             data=data, headers=headers, query=query, fatal=fatal)
 
     def _call_encrypted_api(self, video_id, param='', msg='API', data={}, query={}, fatal=True):
-        if not Cryptodome:
+        if not Cryptodome.RSA:
             raise ExtractorError('pycryptodomex not found. Please install', expected=True)
-        private_key = Cryptodome.PublicKey.RSA.generate(2048)
-        cipher = Cryptodome.Cipher.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.Hash.SHA1)
+        private_key = Cryptodome.RSA.generate(2048)
+        cipher = Cryptodome.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.SHA1)
 
         def decrypt(data):
             if not data:

From f34804b2f920f62a6e893a14a9e2a2144b14dd23 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 28 Feb 2023 23:34:43 +0530
Subject: [PATCH 098/871] [extractor/youtube] Fix
 5038f6d713303e0967d002216e7a88652401c22a

* [fragment] Fix `request_data`
* [youtube] Don't use POST for now. It may be easier to break in future

Authored by: bashonly, coletdjnz
---
 yt_dlp/downloader/fragment.py | 3 ++-
 yt_dlp/extractor/common.py    | 1 +
 yt_dlp/extractor/youtube.py   | 1 -
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 039cb1492..377f138b7 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -466,7 +466,8 @@ def error_callback(err, count, retries):
             for retry in RetryManager(self.params.get('fragment_retries'), error_callback):
                 try:
                     ctx['fragment_count'] = fragment.get('fragment_count')
-                    if not self._download_fragment(ctx, fragment['url'], info_dict, headers):
+                    if not self._download_fragment(
+                            ctx, fragment['url'], info_dict, headers, info_dict.get('request_data')):
                         return
                 except (urllib.error.HTTPError, http.client.IncompleteRead) as err:
                     retry.error = err
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 86bef173f..98efe0e9d 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -132,6 +132,7 @@ class InfoExtractor:
                                        is parsed from a string (in case of
                                        fragmented media)
                                    for MSS - URL of the ISM manifest.
+                    * request_data  Data to send in POST request to the URL
                     * manifest_url
                                  The URL of the manifest file in case of
                                  fragmented media:
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 0227a1f83..f5ffce775 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3778,7 +3778,6 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             if no_audio or no_video:
                 CHUNK_SIZE = 10 << 20
                 dct.update({
-                    'request_data': b'x',
                     'protocol': 'http_dash_segments',
                     'fragments': [{
                         'url': update_url_query(dct['url'], {

From 31e183557fcd1b937582f9429f29207c1261f501 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 28 Feb 2023 23:50:34 +0530
Subject: [PATCH 099/871] [extractor/youtube] Extract channel `view_count` when
 `/about` tab is passed

---
 yt_dlp/extractor/youtube.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index f5ffce775..d1696349a 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4905,6 +4905,10 @@ def _get_uncropped(url):
         info['view_count'] = self._get_count(playlist_stats, 1)
         if info['view_count'] is None:  # 0 is allowed
             info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText')
+        if info['view_count'] is None:
+            info['view_count'] = self._get_count(data, (
+                'contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., 'tabRenderer', 'content', 'sectionListRenderer',
+                'contents', ..., 'itemSectionRenderer', 'contents', ..., 'channelAboutFullMetadataRenderer', 'viewCountText'))
 
         info['playlist_count'] = self._get_count(playlist_stats, 0)
         if info['playlist_count'] is None:  # 0 is allowed
@@ -6124,6 +6128,23 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             }
         }],
         'params': {'extract_flat': True},
+    }, {
+        'url': 'https://www.youtube.com/@3blue1brown/about',
+        'info_dict': {
+            'id': 'UCYO_jab_esuFRV4b17AJtAw',
+            'tags': ['Mathematics'],
+            'title': '3Blue1Brown - About',
+            'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
+            'channel_follower_count': int,
+            'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
+            'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
+            'channel': '3Blue1Brown',
+            'uploader': '3Blue1Brown',
+            'view_count': int,
+            'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
+            'description': 'md5:e1384e8a133307dd10edee76e875d62f',
+        },
+        'playlist_count': 0,
     }]
 
     @classmethod

From 5b28cef72db3b531680d89c121631c73ae05354f Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 28 Feb 2023 23:31:02 +0530
Subject: [PATCH 100/871] [cleanup] Misc

---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |   2 +
 .../ISSUE_TEMPLATE/2_site_support_request.yml |   2 +
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |   2 +
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |   2 +
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |   2 +
 .github/ISSUE_TEMPLATE/6_question.yml         |   2 +
 CONTRIBUTING.md                               |   2 +-
 Changelog.md                                  |   4 +-
 README.md                                     |   1 +
 devscripts/make_issue_template.py             |   2 +
 supportedsites.md                             | 366 +++++++++---------
 yt_dlp/YoutubeDL.py                           |   2 +-
 yt_dlp/dependencies/Cryptodome.py             |  18 +-
 yt_dlp/downloader/fragment.py                 |   2 +-
 yt_dlp/extractor/common.py                    |   2 +-
 yt_dlp/extractor/youtube.py                   |   4 +-
 16 files changed, 212 insertions(+), 203 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index e1103fb84..48e8890c5 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -50,6 +50,8 @@ body:
       options:
         - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
           required: true
+        - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead"
+          required: false
         - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
           required: true
   - type: textarea
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index 90d7294ac..d43d62f03 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -62,6 +62,8 @@ body:
       options:
         - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
           required: true
+        - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead"
+          required: false
         - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
           required: true
   - type: textarea
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index 5b59852c7..352b47242 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -58,6 +58,8 @@ body:
       options:
         - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
           required: true
+        - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead"
+          required: false
         - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
           required: true
   - type: textarea
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index bd4695f87..7588b8ed8 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -43,6 +43,8 @@ body:
       options:
         - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
           required: true
+        - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead"
+          required: false
         - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
           required: true
   - type: textarea
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index 8c7f315e9..fdda50b7b 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -40,6 +40,8 @@ body:
       label: Provide verbose output that clearly demonstrates the problem
       options:
         - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
+        - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead"
+          required: false
         - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
   - type: textarea
     id: log
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index 4a1344628..56ce74654 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -46,6 +46,8 @@ body:
       label: Provide verbose output that clearly demonstrates the problem
       options:
         - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
+        - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead"
+          required: false
         - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
   - type: textarea
     id: log
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 551db674e..ae2c45423 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -127,7 +127,7 @@ ### Are you willing to share account details if needed?
 
 ### Is the website primarily used for piracy?
 
-We follow [youtube-dl's policy](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free) to not support services that is primarily used for infringing copyright. Additionally, it has been decided to not to support porn sites that specialize in deep fake. We also cannot support any service that serves only [DRM protected content](https://en.wikipedia.org/wiki/Digital_rights_management). 
+We follow [youtube-dl's policy](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free) to not support services that is primarily used for infringing copyright. Additionally, it has been decided to not to support porn sites that specialize in fakes. We also cannot support any service that serves only [DRM protected content](https://en.wikipedia.org/wiki/Digital_rights_management). 
 
 
 
diff --git a/Changelog.md b/Changelog.md
index 8d3ac089c..24bc8a2e2 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -50,8 +50,8 @@ ### 2023.02.17
 * [extractor/txxx] Add extractors by [chio0hai](https://github.com/chio0hai)
 * [extractor/vocaroo] Add extractor by [SuperSonicHub1](https://github.com/SuperSonicHub1), [qbnu](https://github.com/qbnu)
 * [extractor/wrestleuniverse] Add extractors by [Grub4K](https://github.com/Grub4K), [bashonly](https://github.com/bashonly)
-* [extractor/yappy] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
-* **[extractor/youtube] Fix `uploader_id` extraction** by [bashonly](https://github.com/bashonly)
+* [extractor/yappy] Add extractor by [HobbyistDev](https://github.com/HobbyistDev), [dirkf](https://github.com/dirkf)
+* [extractor/youtube] **Fix `uploader_id` extraction** by [bashonly](https://github.com/bashonly)
 * [extractor/youtube] Add hyperpipe instances by [Generator](https://github.com/Generator)
 * [extractor/youtube] Handle `consent.youtube`
 * [extractor/youtube] Support `/live/` URL
diff --git a/README.md b/README.md
index 9b91775bc..3d3db933a 100644
--- a/README.md
+++ b/README.md
@@ -130,6 +130,7 @@ ### Differences in default behavior
 
 Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc:
 
+* yt-dlp supports only [Python 3.7+](## "Windows 7"), and *may* remove support for more versions as they [become EOL](https://devguide.python.org/versions/#python-release-cycle); while [youtube-dl still supports Python 2.6+ and 3.2+](https://github.com/ytdl-org/youtube-dl/issues/30568#issue-1118238743)
 * The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
 * `avconv` is not supported as an alternative to `ffmpeg`
 * yt-dlp stores config files in slightly different locations to youtube-dl. See [CONFIGURATION](#configuration) for a list of correct locations
diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index 1ee00f2b8..39b95c8da 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -24,6 +24,8 @@
       options:
         - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
           required: true
+        - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead"
+          required: false
         - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
           required: true
   - type: textarea
diff --git a/supportedsites.md b/supportedsites.md
index b545ec540..d7ac6dce5 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -28,14 +28,14 @@ # Supported sites
  - **abcnews:video**
  - **abcotvs**: ABC Owned Television Stations
  - **abcotvs:clips**
- - **AbemaTV**: [<abbr title="netrc machine"><em>abematv</em></abbr>]
+ - **AbemaTV**: [*abematv*](## "netrc machine")
  - **AbemaTVTitle**
  - **AcademicEarth:Course**
  - **acast**
  - **acast:channel**
  - **AcFunBangumi**
  - **AcFunVideo**
- - **ADN**: [<abbr title="netrc machine"><em>animationdigitalnetwork</em></abbr>] Animation Digital Network
+ - **ADN**: [*animationdigitalnetwork*](## "netrc machine") Animation Digital Network
  - **AdobeConnect**
  - **adobetv**
  - **adobetv:channel**
@@ -47,8 +47,8 @@ # Supported sites
  - **aenetworks:collection**
  - **aenetworks:show**
  - **AeonCo**
- - **afreecatv**: [<abbr title="netrc machine"><em>afreecatv</em></abbr>] afreecatv.com
- - **afreecatv:live**: [<abbr title="netrc machine"><em>afreecatv</em></abbr>] afreecatv.com
+ - **afreecatv**: [*afreecatv*](## "netrc machine") afreecatv.com
+ - **afreecatv:live**: [*afreecatv*](## "netrc machine") afreecatv.com
  - **afreecatv:user**
  - **AirMozilla**
  - **AirTV**
@@ -59,8 +59,8 @@ # Supported sites
  - **AlphaPorno**
  - **Alsace20TV**
  - **Alsace20TVEmbed**
- - **Alura**: [<abbr title="netrc machine"><em>alura</em></abbr>]
- - **AluraCourse**: [<abbr title="netrc machine"><em>aluracourse</em></abbr>]
+ - **Alura**: [*alura*](## "netrc machine")
+ - **AluraCourse**: [*aluracourse*](## "netrc machine")
  - **Amara**
  - **AmazonMiniTV**
  - **amazonminitv:season**: Amazon MiniTV Season, "minitv:season:" prefix
@@ -100,7 +100,7 @@ # Supported sites
  - **ArteTVPlaylist**
  - **AsianCrush**
  - **AsianCrushPlaylist**
- - **AtresPlayer**: [<abbr title="netrc machine"><em>atresplayer</em></abbr>]
+ - **AtresPlayer**: [*atresplayer*](## "netrc machine")
  - **AtScaleConfEvent**
  - **ATTTechChannel**
  - **ATVAt**
@@ -128,15 +128,15 @@ # Supported sites
  - **Bandcamp:user**
  - **Bandcamp:weekly**
  - **BannedVideo**
- - **bbc**: [<abbr title="netrc machine"><em>bbc</em></abbr>] BBC
- - **bbc.co.uk**: [<abbr title="netrc machine"><em>bbc</em></abbr>] BBC iPlayer
+ - **bbc**: [*bbc*](## "netrc machine") BBC
+ - **bbc.co.uk**: [*bbc*](## "netrc machine") BBC iPlayer
  - **bbc.co.uk:article**: BBC articles
  - **bbc.co.uk:​iplayer:episodes**
  - **bbc.co.uk:​iplayer:group**
  - **bbc.co.uk:playlist**
- - **BBVTV**: [<abbr title="netrc machine"><em>bbvtv</em></abbr>]
- - **BBVTVLive**: [<abbr title="netrc machine"><em>bbvtv</em></abbr>]
- - **BBVTVRecordings**: [<abbr title="netrc machine"><em>bbvtv</em></abbr>]
+ - **BBVTV**: [*bbvtv*](## "netrc machine")
+ - **BBVTVLive**: [*bbvtv*](## "netrc machine")
+ - **BBVTVRecordings**: [*bbvtv*](## "netrc machine")
  - **BeatBumpPlaylist**
  - **BeatBumpVideo**
  - **Beatport**
@@ -165,8 +165,8 @@ # Supported sites
  - **BilibiliSpaceAudio**
  - **BilibiliSpacePlaylist**
  - **BilibiliSpaceVideo**
- - **BiliIntl**: [<abbr title="netrc machine"><em>biliintl</em></abbr>]
- - **biliIntl:series**: [<abbr title="netrc machine"><em>biliintl</em></abbr>]
+ - **BiliIntl**: [*biliintl*](## "netrc machine")
+ - **biliIntl:series**: [*biliintl*](## "netrc machine")
  - **BiliLive**
  - **BioBioChileTV**
  - **Biography**
@@ -232,7 +232,7 @@ # Supported sites
  - **cbssports:embed**
  - **CCMA**
  - **CCTV**: 央视网
- - **CDA**: [<abbr title="netrc machine"><em>cdapl</em></abbr>]
+ - **CDA**: [*cdapl*](## "netrc machine")
  - **Cellebrite**
  - **CeskaTelevize**
  - **CGTN**
@@ -286,8 +286,8 @@ # Supported sites
  - **CrooksAndLiars**
  - **CrowdBunker**
  - **CrowdBunkerChannel**
- - **crunchyroll**: [<abbr title="netrc machine"><em>crunchyroll</em></abbr>]
- - **crunchyroll:playlist**: [<abbr title="netrc machine"><em>crunchyroll</em></abbr>]
+ - **crunchyroll**: [*crunchyroll*](## "netrc machine")
+ - **crunchyroll:playlist**: [*crunchyroll*](## "netrc machine")
  - **CSpan**: C-SPAN
  - **CSpanCongress**
  - **CtsNews**: 華視新聞
@@ -295,18 +295,18 @@ # Supported sites
  - **CTVNews**
  - **cu.ntv.co.jp**: Nippon Television Network
  - **CultureUnplugged**
- - **curiositystream**: [<abbr title="netrc machine"><em>curiositystream</em></abbr>]
- - **curiositystream:collections**: [<abbr title="netrc machine"><em>curiositystream</em></abbr>]
- - **curiositystream:series**: [<abbr title="netrc machine"><em>curiositystream</em></abbr>]
+ - **curiositystream**: [*curiositystream*](## "netrc machine")
+ - **curiositystream:collections**: [*curiositystream*](## "netrc machine")
+ - **curiositystream:series**: [*curiositystream*](## "netrc machine")
  - **CWTV**
- - **Cybrary**: [<abbr title="netrc machine"><em>cybrary</em></abbr>]
- - **CybraryCourse**: [<abbr title="netrc machine"><em>cybrary</em></abbr>]
+ - **Cybrary**: [*cybrary*](## "netrc machine")
+ - **CybraryCourse**: [*cybrary*](## "netrc machine")
  - **Daftsex**
  - **DagelijkseKost**: dagelijksekost.een.be
  - **DailyMail**
- - **dailymotion**: [<abbr title="netrc machine"><em>dailymotion</em></abbr>]
- - **dailymotion:playlist**: [<abbr title="netrc machine"><em>dailymotion</em></abbr>]
- - **dailymotion:user**: [<abbr title="netrc machine"><em>dailymotion</em></abbr>]
+ - **dailymotion**: [*dailymotion*](## "netrc machine")
+ - **dailymotion:playlist**: [*dailymotion*](## "netrc machine")
+ - **dailymotion:user**: [*dailymotion*](## "netrc machine")
  - **DailyWire**
  - **DailyWirePodcast**
  - **damtomo:record**
@@ -328,7 +328,7 @@ # Supported sites
  - **DeuxMNews**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
  - **Digg**
- - **DigitalConcertHall**: [<abbr title="netrc machine"><em>digitalconcerthall</em></abbr>] DigitalConcertHall extractor
+ - **DigitalConcertHall**: [*digitalconcerthall*](## "netrc machine") DigitalConcertHall extractor
  - **DigitallySpeaking**
  - **Digiteka**
  - **Discovery**
@@ -351,7 +351,7 @@ # Supported sites
  - **DRBonanza**
  - **Drooble**
  - **Dropbox**
- - **Dropout**: [<abbr title="netrc machine"><em>dropout</em></abbr>]
+ - **Dropout**: [*dropout*](## "netrc machine")
  - **DropoutSeason**
  - **DrTuber**
  - **drtv**
@@ -373,9 +373,9 @@ # Supported sites
  - **egghead:lesson**: egghead.io lesson
  - **ehftv**
  - **eHow**
- - **EinsUndEinsTV**: [<abbr title="netrc machine"><em>1und1tv</em></abbr>]
- - **EinsUndEinsTVLive**: [<abbr title="netrc machine"><em>1und1tv</em></abbr>]
- - **EinsUndEinsTVRecordings**: [<abbr title="netrc machine"><em>1und1tv</em></abbr>]
+ - **EinsUndEinsTV**: [*1und1tv*](## "netrc machine")
+ - **EinsUndEinsTVLive**: [*1und1tv*](## "netrc machine")
+ - **EinsUndEinsTVRecordings**: [*1und1tv*](## "netrc machine")
  - **Einthusan**
  - **eitb.tv**
  - **EllenTube**
@@ -390,7 +390,7 @@ # Supported sites
  - **EpiconSeries**
  - **Epoch**
  - **Eporner**
- - **EroProfile**: [<abbr title="netrc machine"><em>eroprofile</em></abbr>]
+ - **EroProfile**: [*eroprofile*](## "netrc machine")
  - **EroProfile:album**
  - **ertflix**: ERTFLIX videos
  - **ertflix:codename**: ERTFLIX videos by codename
@@ -405,20 +405,20 @@ # Supported sites
  - **EuropeanTour**
  - **Eurosport**
  - **EUScreen**
- - **EWETV**: [<abbr title="netrc machine"><em>ewetv</em></abbr>]
- - **EWETVLive**: [<abbr title="netrc machine"><em>ewetv</em></abbr>]
- - **EWETVRecordings**: [<abbr title="netrc machine"><em>ewetv</em></abbr>]
+ - **EWETV**: [*ewetv*](## "netrc machine")
+ - **EWETVLive**: [*ewetv*](## "netrc machine")
+ - **EWETVRecordings**: [*ewetv*](## "netrc machine")
  - **ExpoTV**
  - **Expressen**
  - **ExtremeTube**
  - **EyedoTV**
- - **facebook**: [<abbr title="netrc machine"><em>facebook</em></abbr>]
+ - **facebook**: [*facebook*](## "netrc machine")
  - **facebook:reel**
  - **FacebookPluginsVideo**
- - **fancode:live**: [<abbr title="netrc machine"><em>fancode</em></abbr>]
- - **fancode:vod**: [<abbr title="netrc machine"><em>fancode</em></abbr>]
+ - **fancode:live**: [*fancode*](## "netrc machine")
+ - **fancode:vod**: [*fancode*](## "netrc machine")
  - **faz.net**
- - **fc2**: [<abbr title="netrc machine"><em>fc2</em></abbr>]
+ - **fc2**: [*fc2*](## "netrc machine")
  - **fc2:embed**
  - **fc2:live**
  - **Fczenit**
@@ -452,20 +452,20 @@ # Supported sites
  - **freespeech.org**
  - **freetv:series**
  - **FreeTvMovies**
- - **FrontendMasters**: [<abbr title="netrc machine"><em>frontendmasters</em></abbr>]
- - **FrontendMastersCourse**: [<abbr title="netrc machine"><em>frontendmasters</em></abbr>]
- - **FrontendMastersLesson**: [<abbr title="netrc machine"><em>frontendmasters</em></abbr>]
+ - **FrontendMasters**: [*frontendmasters*](## "netrc machine")
+ - **FrontendMastersCourse**: [*frontendmasters*](## "netrc machine")
+ - **FrontendMastersLesson**: [*frontendmasters*](## "netrc machine")
  - **FujiTVFODPlus7**
- - **Funimation**: [<abbr title="netrc machine"><em>funimation</em></abbr>]
- - **funimation:page**: [<abbr title="netrc machine"><em>funimation</em></abbr>]
- - **funimation:show**: [<abbr title="netrc machine"><em>funimation</em></abbr>]
+ - **Funimation**: [*funimation*](## "netrc machine")
+ - **funimation:page**: [*funimation*](## "netrc machine")
+ - **funimation:show**: [*funimation*](## "netrc machine")
  - **Funk**
  - **Fusion**
  - **Fux**
  - **FuyinTV**
  - **Gab**
  - **GabTV**
- - **Gaia**: [<abbr title="netrc machine"><em>gaia</em></abbr>]
+ - **Gaia**: [*gaia*](## "netrc machine")
  - **GameInformer**
  - **GameJolt**
  - **GameJoltCommunity**
@@ -477,9 +477,9 @@ # Supported sites
  - **GameStar**
  - **Gaskrank**
  - **Gazeta**
- - **GDCVault**: [<abbr title="netrc machine"><em>gdcvault</em></abbr>]
+ - **GDCVault**: [*gdcvault*](## "netrc machine")
  - **GediDigital**
- - **gem.cbc.ca**: [<abbr title="netrc machine"><em>cbcgem</em></abbr>]
+ - **gem.cbc.ca**: [*cbcgem*](## "netrc machine")
  - **gem.cbc.ca:live**
  - **gem.cbc.ca:playlist**
  - **Genius**
@@ -489,11 +489,11 @@ # Supported sites
  - **Gfycat**
  - **GiantBomb**
  - **Giga**
- - **GlattvisionTV**: [<abbr title="netrc machine"><em>glattvisiontv</em></abbr>]
- - **GlattvisionTVLive**: [<abbr title="netrc machine"><em>glattvisiontv</em></abbr>]
- - **GlattvisionTVRecordings**: [<abbr title="netrc machine"><em>glattvisiontv</em></abbr>]
+ - **GlattvisionTV**: [*glattvisiontv*](## "netrc machine")
+ - **GlattvisionTVLive**: [*glattvisiontv*](## "netrc machine")
+ - **GlattvisionTVRecordings**: [*glattvisiontv*](## "netrc machine")
  - **Glide**: Glide mobile video messages (glide.me)
- - **Globo**: [<abbr title="netrc machine"><em>globo</em></abbr>]
+ - **Globo**: [*globo*](## "netrc machine")
  - **GloboArticle**
  - **glomex**: Glomex videos
  - **glomex:embed**: Glomex embedded videos
@@ -507,7 +507,7 @@ # Supported sites
  - **google:​podcasts:feed**
  - **GoogleDrive**
  - **GoogleDrive:Folder**
- - **GoPlay**: [<abbr title="netrc machine"><em>goplay</em></abbr>]
+ - **GoPlay**: [*goplay*](## "netrc machine")
  - **GoPro**
  - **Goshgay**
  - **GoToStage**
@@ -527,7 +527,7 @@ # Supported sites
  - **hgtv.com:show**
  - **HGTVDe**
  - **HGTVUsa**
- - **HiDive**: [<abbr title="netrc machine"><em>hidive</em></abbr>]
+ - **HiDive**: [*hidive*](## "netrc machine")
  - **HistoricFilms**
  - **history:player**
  - **history:topic**: History.com Topic
@@ -544,8 +544,8 @@ # Supported sites
  - **Howcast**
  - **HowStuffWorks**
  - **hrfernsehen**
- - **HRTi**: [<abbr title="netrc machine"><em>hrti</em></abbr>]
- - **HRTiPlaylist**: [<abbr title="netrc machine"><em>hrti</em></abbr>]
+ - **HRTi**: [*hrti*](## "netrc machine")
+ - **HRTiPlaylist**: [*hrti*](## "netrc machine")
  - **HSEProduct**
  - **HSEShow**
  - **html5**
@@ -575,19 +575,19 @@ # Supported sites
  - **Inc**
  - **IndavideoEmbed**
  - **InfoQ**
- - **Instagram**: [<abbr title="netrc machine"><em>instagram</em></abbr>]
- - **instagram:story**: [<abbr title="netrc machine"><em>instagram</em></abbr>]
- - **instagram:tag**: [<abbr title="netrc machine"><em>instagram</em></abbr>] Instagram hashtag search URLs
- - **instagram:user**: [<abbr title="netrc machine"><em>instagram</em></abbr>] Instagram user profile
+ - **Instagram**: [*instagram*](## "netrc machine")
+ - **instagram:story**: [*instagram*](## "netrc machine")
+ - **instagram:tag**: [*instagram*](## "netrc machine") Instagram hashtag search URLs
+ - **instagram:user**: [*instagram*](## "netrc machine") Instagram user profile
  - **InstagramIOS**: IOS instagram:// URL
  - **Internazionale**
  - **InternetVideoArchive**
  - **InvestigationDiscovery**
- - **IPrima**: [<abbr title="netrc machine"><em>iprima</em></abbr>]
+ - **IPrima**: [*iprima*](## "netrc machine")
  - **IPrimaCNN**
  - **iq.com**: International version of iQiyi
  - **iq.com:album**
- - **iqiyi**: [<abbr title="netrc machine"><em>iqiyi</em></abbr>] 爱奇艺
+ - **iqiyi**: [*iqiyi*](## "netrc machine") 爱奇艺
  - **IslamChannel**
  - **IslamChannelSeries**
  - **IsraelNationalNews**
@@ -660,9 +660,9 @@ # Supported sites
  - **LcpPlay**
  - **Le**: 乐视网
  - **Lecture2Go**
- - **Lecturio**: [<abbr title="netrc machine"><em>lecturio</em></abbr>]
- - **LecturioCourse**: [<abbr title="netrc machine"><em>lecturio</em></abbr>]
- - **LecturioDeCourse**: [<abbr title="netrc machine"><em>lecturio</em></abbr>]
+ - **Lecturio**: [*lecturio*](## "netrc machine")
+ - **LecturioCourse**: [*lecturio*](## "netrc machine")
+ - **LecturioDeCourse**: [*lecturio*](## "netrc machine")
  - **LEGO**
  - **Lemonde**
  - **Lenta**
@@ -678,10 +678,10 @@ # Supported sites
  - **limelight:channel_list**
  - **LineLive**
  - **LineLiveChannel**
- - **LinkedIn**: [<abbr title="netrc machine"><em>linkedin</em></abbr>]
- - **linkedin:learning**: [<abbr title="netrc machine"><em>linkedin</em></abbr>]
- - **linkedin:​learning:course**: [<abbr title="netrc machine"><em>linkedin</em></abbr>]
- - **LinuxAcademy**: [<abbr title="netrc machine"><em>linuxacademy</em></abbr>]
+ - **LinkedIn**: [*linkedin*](## "netrc machine")
+ - **linkedin:learning**: [*linkedin*](## "netrc machine")
+ - **linkedin:​learning:course**: [*linkedin*](## "netrc machine")
+ - **LinuxAcademy**: [*linuxacademy*](## "netrc machine")
  - **Liputan6**
  - **ListenNotes**
  - **LiTV**
@@ -696,8 +696,8 @@ # Supported sites
  - **LoveHomePorn**
  - **LRTStream**
  - **LRTVOD**
- - **lynda**: [<abbr title="netrc machine"><em>lynda</em></abbr>] lynda.com videos
- - **lynda:course**: [<abbr title="netrc machine"><em>lynda</em></abbr>] lynda.com online courses
+ - **lynda**: [*lynda*](## "netrc machine") lynda.com videos
+ - **lynda:course**: [*lynda*](## "netrc machine") lynda.com online courses
  - **m6**
  - **MagentaMusik360**
  - **mailru**: Видео@Mail.Ru
@@ -767,13 +767,13 @@ # Supported sites
  - **mixcloud:user**
  - **MLB**
  - **MLBArticle**
- - **MLBTV**: [<abbr title="netrc machine"><em>mlb</em></abbr>]
+ - **MLBTV**: [*mlb*](## "netrc machine")
  - **MLBVideo**
  - **MLSSoccer**
  - **Mnet**
- - **MNetTV**: [<abbr title="netrc machine"><em>mnettv</em></abbr>]
- - **MNetTVLive**: [<abbr title="netrc machine"><em>mnettv</em></abbr>]
- - **MNetTVRecordings**: [<abbr title="netrc machine"><em>mnettv</em></abbr>]
+ - **MNetTV**: [*mnettv*](## "netrc machine")
+ - **MNetTVLive**: [*mnettv*](## "netrc machine")
+ - **MNetTVRecordings**: [*mnettv*](## "netrc machine")
  - **MochaVideo**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
  - **Mofosex**
@@ -852,9 +852,9 @@ # Supported sites
  - **ndr:embed**
  - **ndr:​embed:base**
  - **NDTV**
- - **Nebula**: [<abbr title="netrc machine"><em>watchnebula</em></abbr>]
- - **nebula:channel**: [<abbr title="netrc machine"><em>watchnebula</em></abbr>]
- - **nebula:subscriptions**: [<abbr title="netrc machine"><em>watchnebula</em></abbr>]
+ - **Nebula**: [*watchnebula*](## "netrc machine")
+ - **nebula:channel**: [*watchnebula*](## "netrc machine")
+ - **nebula:subscriptions**: [*watchnebula*](## "netrc machine")
  - **NerdCubedFeed**
  - **netease:album**: 网易云音乐 - 专辑
  - **netease:djradio**: 网易云音乐 - 电台
@@ -863,9 +863,9 @@ # Supported sites
  - **netease:program**: 网易云音乐 - 电台节目
  - **netease:singer**: 网易云音乐 - 歌手
  - **netease:song**: 网易云音乐
- - **NetPlusTV**: [<abbr title="netrc machine"><em>netplus</em></abbr>]
- - **NetPlusTVLive**: [<abbr title="netrc machine"><em>netplus</em></abbr>]
- - **NetPlusTVRecordings**: [<abbr title="netrc machine"><em>netplus</em></abbr>]
+ - **NetPlusTV**: [*netplus*](## "netrc machine")
+ - **NetPlusTVLive**: [*netplus*](## "netrc machine")
+ - **NetPlusTVRecordings**: [*netplus*](## "netrc machine")
  - **Netverse**
  - **NetversePlaylist**
  - **NetverseSearch**: "netsearch:" prefix
@@ -898,7 +898,7 @@ # Supported sites
  - **nickelodeon:br**
  - **nickelodeonru**
  - **nicknight**
- - **niconico**: [<abbr title="netrc machine"><em>niconico</em></abbr>] ニコニコ動画
+ - **niconico**: [*niconico*](## "netrc machine") ニコニコ動画
  - **niconico:history**: NicoNico user history or likes. Requires cookies.
  - **niconico:playlist**
  - **niconico:series**
@@ -911,7 +911,7 @@ # Supported sites
  - **Nitter**
  - **njoy**: N-JOY
  - **njoy:embed**
- - **NJPWWorld**: [<abbr title="netrc machine"><em>njpwworld</em></abbr>] 新日本プロレスワールド
+ - **NJPWWorld**: [*njpwworld*](## "netrc machine") 新日本プロレスワールド
  - **NobelPrize**
  - **NoicePodcast**
  - **NonkTube**
@@ -980,11 +980,11 @@ # Supported sites
  - **orf:iptv**: iptv.ORF.at
  - **orf:radio**
  - **orf:tvthek**: ORF TVthek
- - **OsnatelTV**: [<abbr title="netrc machine"><em>osnateltv</em></abbr>]
- - **OsnatelTVLive**: [<abbr title="netrc machine"><em>osnateltv</em></abbr>]
- - **OsnatelTVRecordings**: [<abbr title="netrc machine"><em>osnateltv</em></abbr>]
+ - **OsnatelTV**: [*osnateltv*](## "netrc machine")
+ - **OsnatelTVLive**: [*osnateltv*](## "netrc machine")
+ - **OsnatelTVRecordings**: [*osnateltv*](## "netrc machine")
  - **OutsideTV**
- - **PacktPub**: [<abbr title="netrc machine"><em>packtpub</em></abbr>]
+ - **PacktPub**: [*packtpub*](## "netrc machine")
  - **PacktPubCourse**
  - **PalcoMP3:artist**
  - **PalcoMP3:song**
@@ -1007,7 +1007,7 @@ # Supported sites
  - **peer.tv**
  - **PeerTube**
  - **PeerTube:Playlist**
- - **peloton**: [<abbr title="netrc machine"><em>peloton</em></abbr>]
+ - **peloton**: [*peloton*](## "netrc machine")
  - **peloton:live**: Peloton Live
  - **People**
  - **PerformGroup**
@@ -1016,7 +1016,7 @@ # Supported sites
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **phoenix.de**
  - **Photobucket**
- - **Piapro**: [<abbr title="netrc machine"><em>piapro</em></abbr>]
+ - **Piapro**: [*piapro*](## "netrc machine")
  - **Picarto**
  - **PicartoVod**
  - **Piksel**
@@ -1027,11 +1027,11 @@ # Supported sites
  - **pixiv:​sketch:user**
  - **Pladform**
  - **PlanetMarathi**
- - **Platzi**: [<abbr title="netrc machine"><em>platzi</em></abbr>]
- - **PlatziCourse**: [<abbr title="netrc machine"><em>platzi</em></abbr>]
+ - **Platzi**: [*platzi*](## "netrc machine")
+ - **PlatziCourse**: [*platzi*](## "netrc machine")
  - **play.fm**
  - **player.sky.it**
- - **PlayPlusTV**: [<abbr title="netrc machine"><em>playplustv</em></abbr>]
+ - **PlayPlusTV**: [*playplustv*](## "netrc machine")
  - **PlayStuff**
  - **PlaysTV**
  - **PlaySuisse**
@@ -1039,7 +1039,7 @@ # Supported sites
  - **Playvid**
  - **PlayVids**
  - **Playwire**
- - **pluralsight**: [<abbr title="netrc machine"><em>pluralsight</em></abbr>]
+ - **pluralsight**: [*pluralsight*](## "netrc machine")
  - **pluralsight:course**
  - **PlutoTV**
  - **PodbayFM**
@@ -1048,8 +1048,8 @@ # Supported sites
  - **podomatic**
  - **Pokemon**
  - **PokemonWatch**
- - **PokerGo**: [<abbr title="netrc machine"><em>pokergo</em></abbr>]
- - **PokerGoCollection**: [<abbr title="netrc machine"><em>pokergo</em></abbr>]
+ - **PokerGo**: [*pokergo*](## "netrc machine")
+ - **PokerGoCollection**: [*pokergo*](## "netrc machine")
  - **PolsatGo**
  - **PolskieRadio**
  - **polskieradio:audition**
@@ -1066,11 +1066,11 @@ # Supported sites
  - **Pornez**
  - **PornFlip**
  - **PornHd**
- - **PornHub**: [<abbr title="netrc machine"><em>pornhub</em></abbr>] PornHub and Thumbzilla
- - **PornHubPagedVideoList**: [<abbr title="netrc machine"><em>pornhub</em></abbr>]
- - **PornHubPlaylist**: [<abbr title="netrc machine"><em>pornhub</em></abbr>]
- - **PornHubUser**: [<abbr title="netrc machine"><em>pornhub</em></abbr>]
- - **PornHubUserVideosUpload**: [<abbr title="netrc machine"><em>pornhub</em></abbr>]
+ - **PornHub**: [*pornhub*](## "netrc machine") PornHub and Thumbzilla
+ - **PornHubPagedVideoList**: [*pornhub*](## "netrc machine")
+ - **PornHubPlaylist**: [*pornhub*](## "netrc machine")
+ - **PornHubUser**: [*pornhub*](## "netrc machine")
+ - **PornHubUserVideosUpload**: [*pornhub*](## "netrc machine")
  - **Pornotube**
  - **PornoVoisines**
  - **PornoXO**
@@ -1098,9 +1098,9 @@ # Supported sites
  - **qqmusic:playlist**: QQ音乐 - 歌单
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
- - **QuantumTV**: [<abbr title="netrc machine"><em>quantumtv</em></abbr>]
- - **QuantumTVLive**: [<abbr title="netrc machine"><em>quantumtv</em></abbr>]
- - **QuantumTVRecordings**: [<abbr title="netrc machine"><em>quantumtv</em></abbr>]
+ - **QuantumTV**: [*quantumtv*](## "netrc machine")
+ - **QuantumTVLive**: [*quantumtv*](## "netrc machine")
+ - **QuantumTVRecordings**: [*quantumtv*](## "netrc machine")
  - **Qub**
  - **R7**
  - **R7Article**
@@ -1157,16 +1157,16 @@ # Supported sites
  - **RICE**
  - **RMCDecouverte**
  - **RockstarGames**
- - **Rokfin**: [<abbr title="netrc machine"><em>rokfin</em></abbr>]
+ - **Rokfin**: [*rokfin*](## "netrc machine")
  - **rokfin:channel**: Rokfin Channels
  - **rokfin:search**: Rokfin Search; "rkfnsearch:" prefix
  - **rokfin:stack**: Rokfin Stacks
- - **RoosterTeeth**: [<abbr title="netrc machine"><em>roosterteeth</em></abbr>]
- - **RoosterTeethSeries**: [<abbr title="netrc machine"><em>roosterteeth</em></abbr>]
+ - **RoosterTeeth**: [*roosterteeth*](## "netrc machine")
+ - **RoosterTeethSeries**: [*roosterteeth*](## "netrc machine")
  - **RottenTomatoes**
  - **Rozhlas**
  - **RozhlasVltava**
- - **RTBF**: [<abbr title="netrc machine"><em>rtbf</em></abbr>]
+ - **RTBF**: [*rtbf*](## "netrc machine")
  - **RTDocumentry**
  - **RTDocumentryPlaylist**
  - **rte**: Raidió Teilifís Éireann TV
@@ -1208,16 +1208,16 @@ # Supported sites
  - **Ruutu**
  - **Ruv**
  - **ruv.is:spila**
- - **safari**: [<abbr title="netrc machine"><em>safari</em></abbr>] safaribooksonline.com online video
- - **safari:api**: [<abbr title="netrc machine"><em>safari</em></abbr>]
- - **safari:course**: [<abbr title="netrc machine"><em>safari</em></abbr>] safaribooksonline.com online courses
+ - **safari**: [*safari*](## "netrc machine") safaribooksonline.com online video
+ - **safari:api**: [*safari*](## "netrc machine")
+ - **safari:course**: [*safari*](## "netrc machine") safaribooksonline.com online courses
  - **Saitosan**
- - **SAKTV**: [<abbr title="netrc machine"><em>saktv</em></abbr>]
- - **SAKTVLive**: [<abbr title="netrc machine"><em>saktv</em></abbr>]
- - **SAKTVRecordings**: [<abbr title="netrc machine"><em>saktv</em></abbr>]
- - **SaltTV**: [<abbr title="netrc machine"><em>salttv</em></abbr>]
- - **SaltTVLive**: [<abbr title="netrc machine"><em>salttv</em></abbr>]
- - **SaltTVRecordings**: [<abbr title="netrc machine"><em>salttv</em></abbr>]
+ - **SAKTV**: [*saktv*](## "netrc machine")
+ - **SAKTVLive**: [*saktv*](## "netrc machine")
+ - **SAKTVRecordings**: [*saktv*](## "netrc machine")
+ - **SaltTV**: [*salttv*](## "netrc machine")
+ - **SaltTVLive**: [*salttv*](## "netrc machine")
+ - **SaltTVRecordings**: [*salttv*](## "netrc machine")
  - **SampleFocus**
  - **Sangiin**: 参議院インターネット審議中継 (archive)
  - **Sapo**: SAPO Vídeos
@@ -1233,8 +1233,8 @@ # Supported sites
  - **ScrippsNetworks**
  - **scrippsnetworks:watch**
  - **Scrolller**
- - **SCTE**: [<abbr title="netrc machine"><em>scte</em></abbr>]
- - **SCTECourse**: [<abbr title="netrc machine"><em>scte</em></abbr>]
+ - **SCTE**: [*scte*](## "netrc machine")
+ - **SCTECourse**: [*scte*](## "netrc machine")
  - **Seeker**
  - **SenateGov**
  - **SenateISVP**
@@ -1243,7 +1243,7 @@ # Supported sites
  - **Sexu**
  - **SeznamZpravy**
  - **SeznamZpravyArticle**
- - **Shahid**: [<abbr title="netrc machine"><em>shahid</em></abbr>]
+ - **Shahid**: [*shahid*](## "netrc machine")
  - **ShahidShow**
  - **Shared**: shared.sx
  - **ShareVideosEmbed**
@@ -1273,16 +1273,16 @@ # Supported sites
  - **Smotrim**
  - **Snotr**
  - **Sohu**
- - **SonyLIV**: [<abbr title="netrc machine"><em>sonyliv</em></abbr>]
+ - **SonyLIV**: [*sonyliv*](## "netrc machine")
  - **SonyLIVSeries**
- - **soundcloud**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
- - **soundcloud:playlist**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
- - **soundcloud:related**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
- - **soundcloud:search**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>] Soundcloud search; "scsearch:" prefix
- - **soundcloud:set**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
- - **soundcloud:trackstation**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
- - **soundcloud:user**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
- - **soundcloud:​user:permalink**: [<abbr title="netrc machine"><em>soundcloud</em></abbr>]
+ - **soundcloud**: [*soundcloud*](## "netrc machine")
+ - **soundcloud:playlist**: [*soundcloud*](## "netrc machine")
+ - **soundcloud:related**: [*soundcloud*](## "netrc machine")
+ - **soundcloud:search**: [*soundcloud*](## "netrc machine") Soundcloud search; "scsearch:" prefix
+ - **soundcloud:set**: [*soundcloud*](## "netrc machine")
+ - **soundcloud:trackstation**: [*soundcloud*](## "netrc machine")
+ - **soundcloud:user**: [*soundcloud*](## "netrc machine")
+ - **soundcloud:​user:permalink**: [*soundcloud*](## "netrc machine")
  - **SoundcloudEmbed**
  - **soundgasm**
  - **soundgasm:profile**
@@ -1349,13 +1349,13 @@ # Supported sites
  - **Tass**
  - **TBS**
  - **TDSLifeway**
- - **Teachable**: [<abbr title="netrc machine"><em>teachable</em></abbr>]
- - **TeachableCourse**: [<abbr title="netrc machine"><em>teachable</em></abbr>]
+ - **Teachable**: [*teachable*](## "netrc machine")
+ - **TeachableCourse**: [*teachable*](## "netrc machine")
  - **teachertube**: teachertube.com videos
  - **teachertube:​user:collection**: teachertube.com user and collection videos
  - **TeachingChannel**
  - **Teamcoco**
- - **TeamTreeHouse**: [<abbr title="netrc machine"><em>teamtreehouse</em></abbr>]
+ - **TeamTreeHouse**: [*teamtreehouse*](## "netrc machine")
  - **TechTalks**
  - **techtv.mit.edu**
  - **TedEmbed**
@@ -1378,8 +1378,8 @@ # Supported sites
  - **TeleTask**
  - **Telewebion**
  - **Tempo**
- - **TennisTV**: [<abbr title="netrc machine"><em>tennistv</em></abbr>]
- - **TenPlay**: [<abbr title="netrc machine"><em>10play</em></abbr>]
+ - **TennisTV**: [*tennistv*](## "netrc machine")
+ - **TenPlay**: [*10play*](## "netrc machine")
  - **TF1**
  - **TFO**
  - **TheHoleTv**
@@ -1417,13 +1417,13 @@ # Supported sites
  - **tokfm:audition**
  - **tokfm:podcast**
  - **ToonGoggles**
- - **tou.tv**: [<abbr title="netrc machine"><em>toutv</em></abbr>]
+ - **tou.tv**: [*toutv*](## "netrc machine")
  - **Toypics**: Toypics video
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict**: (**Currently broken**)
  - **TravelChannel**
- - **Triller**: [<abbr title="netrc machine"><em>triller</em></abbr>]
- - **TrillerUser**: [<abbr title="netrc machine"><em>triller</em></abbr>]
+ - **Triller**: [*triller*](## "netrc machine")
+ - **TrillerUser**: [*triller*](## "netrc machine")
  - **Trilulilu**
  - **Trovo**
  - **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix
@@ -1435,11 +1435,11 @@ # Supported sites
  - **Truth**
  - **TruTV**
  - **Tube8**
- - **TubeTuGraz**: [<abbr title="netrc machine"><em>tubetugraz</em></abbr>] tube.tugraz.at
- - **TubeTuGrazSeries**: [<abbr title="netrc machine"><em>tubetugraz</em></abbr>]
- - **TubiTv**: [<abbr title="netrc machine"><em>tubitv</em></abbr>]
+ - **TubeTuGraz**: [*tubetugraz*](## "netrc machine") tube.tugraz.at
+ - **TubeTuGrazSeries**: [*tubetugraz*](## "netrc machine")
+ - **TubiTv**: [*tubitv*](## "netrc machine")
  - **TubiTvShow**
- - **Tumblr**: [<abbr title="netrc machine"><em>tumblr</em></abbr>]
+ - **Tumblr**: [*tumblr*](## "netrc machine")
  - **tunein:clip**
  - **tunein:program**
  - **tunein:station**
@@ -1489,13 +1489,13 @@ # Supported sites
  - **TwitCasting**
  - **TwitCastingLive**
  - **TwitCastingUser**
- - **twitch:clips**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
- - **twitch:stream**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
- - **twitch:vod**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
- - **TwitchCollection**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
- - **TwitchVideos**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
- - **TwitchVideosClips**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
- - **TwitchVideosCollections**: [<abbr title="netrc machine"><em>twitch</em></abbr>]
+ - **twitch:clips**: [*twitch*](## "netrc machine")
+ - **twitch:stream**: [*twitch*](## "netrc machine")
+ - **twitch:vod**: [*twitch*](## "netrc machine")
+ - **TwitchCollection**: [*twitch*](## "netrc machine")
+ - **TwitchVideos**: [*twitch*](## "netrc machine")
+ - **TwitchVideosClips**: [*twitch*](## "netrc machine")
+ - **TwitchVideosCollections**: [*twitch*](## "netrc machine")
  - **twitter**
  - **twitter:amplify**
  - **twitter:broadcast**
@@ -1503,11 +1503,11 @@ # Supported sites
  - **twitter:shortener**
  - **twitter:spaces**
  - **Txxx**
- - **udemy**: [<abbr title="netrc machine"><em>udemy</em></abbr>]
- - **udemy:course**: [<abbr title="netrc machine"><em>udemy</em></abbr>]
+ - **udemy**: [*udemy*](## "netrc machine")
+ - **udemy:course**: [*udemy*](## "netrc machine")
  - **UDNEmbed**: 聯合影音
- - **UFCArabia**: [<abbr title="netrc machine"><em>ufcarabia</em></abbr>]
- - **UFCTV**: [<abbr title="netrc machine"><em>ufctv</em></abbr>]
+ - **UFCArabia**: [*ufcarabia*](## "netrc machine")
+ - **UFCTV**: [*ufctv*](## "netrc machine")
  - **ukcolumn**
  - **UKTVPlay**
  - **umg:de**: Universal Music Deutschland
@@ -1537,7 +1537,7 @@ # Supported sites
  - **VevoPlaylist**
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
- - **vhx:embed**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
+ - **vhx:embed**: [*vimeo*](## "netrc machine")
  - **Viafree**
  - **vice**
  - **vice:article**
@@ -1560,25 +1560,25 @@ # Supported sites
  - **videomore:season**
  - **videomore:video**
  - **VideoPress**
- - **Vidio**: [<abbr title="netrc machine"><em>vidio</em></abbr>]
- - **VidioLive**: [<abbr title="netrc machine"><em>vidio</em></abbr>]
- - **VidioPremier**: [<abbr title="netrc machine"><em>vidio</em></abbr>]
+ - **Vidio**: [*vidio*](## "netrc machine")
+ - **VidioLive**: [*vidio*](## "netrc machine")
+ - **VidioPremier**: [*vidio*](## "netrc machine")
  - **VidLii**
  - **viewlift**
  - **viewlift:embed**
  - **Viidea**
- - **viki**: [<abbr title="netrc machine"><em>viki</em></abbr>]
- - **viki:channel**: [<abbr title="netrc machine"><em>viki</em></abbr>]
- - **vimeo**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
- - **vimeo:album**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
- - **vimeo:channel**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
- - **vimeo:group**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
- - **vimeo:likes**: [<abbr title="netrc machine"><em>vimeo</em></abbr>] Vimeo user likes
- - **vimeo:ondemand**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
- - **vimeo:pro**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
- - **vimeo:review**: [<abbr title="netrc machine"><em>vimeo</em></abbr>] Review pages on vimeo
- - **vimeo:user**: [<abbr title="netrc machine"><em>vimeo</em></abbr>]
- - **vimeo:watchlater**: [<abbr title="netrc machine"><em>vimeo</em></abbr>] Vimeo watch later list, ":vimeowatchlater" keyword (requires authentication)
+ - **viki**: [*viki*](## "netrc machine")
+ - **viki:channel**: [*viki*](## "netrc machine")
+ - **vimeo**: [*vimeo*](## "netrc machine")
+ - **vimeo:album**: [*vimeo*](## "netrc machine")
+ - **vimeo:channel**: [*vimeo*](## "netrc machine")
+ - **vimeo:group**: [*vimeo*](## "netrc machine")
+ - **vimeo:likes**: [*vimeo*](## "netrc machine") Vimeo user likes
+ - **vimeo:ondemand**: [*vimeo*](## "netrc machine")
+ - **vimeo:pro**: [*vimeo*](## "netrc machine")
+ - **vimeo:review**: [*vimeo*](## "netrc machine") Review pages on vimeo
+ - **vimeo:user**: [*vimeo*](## "netrc machine")
+ - **vimeo:watchlater**: [*vimeo*](## "netrc machine") Vimeo watch later list, ":vimeowatchlater" keyword (requires authentication)
  - **Vimm:recording**
  - **Vimm:stream**
  - **ViMP**
@@ -1588,13 +1588,13 @@ # Supported sites
  - **vine:user**
  - **Viqeo**
  - **Viu**
- - **viu:ott**: [<abbr title="netrc machine"><em>viu</em></abbr>]
+ - **viu:ott**: [*viu*](## "netrc machine")
  - **viu:playlist**
  - **ViuOTTIndonesia**
  - **Vivo**: vivo.sx
- - **vk**: [<abbr title="netrc machine"><em>vk</em></abbr>] VK
- - **vk:uservideos**: [<abbr title="netrc machine"><em>vk</em></abbr>] VK - User's Videos
- - **vk:wallpost**: [<abbr title="netrc machine"><em>vk</em></abbr>]
+ - **vk**: [*vk*](## "netrc machine") VK
+ - **vk:uservideos**: [*vk*](## "netrc machine") VK - User's Videos
+ - **vk:wallpost**: [*vk*](## "netrc machine")
  - **vm.tiktok**
  - **Vocaroo**
  - **Vodlocker**
@@ -1613,14 +1613,14 @@ # Supported sites
  - **vqq:video**
  - **Vrak**
  - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza
- - **VrtNU**: [<abbr title="netrc machine"><em>vrtnu</em></abbr>] VrtNU.be
- - **vrv**: [<abbr title="netrc machine"><em>vrv</em></abbr>]
+ - **VrtNU**: [*vrtnu*](## "netrc machine") VrtNU.be
+ - **vrv**: [*vrv*](## "netrc machine")
  - **vrv:series**
  - **VShare**
  - **VTM**
- - **VTXTV**: [<abbr title="netrc machine"><em>vtxtv</em></abbr>]
- - **VTXTVLive**: [<abbr title="netrc machine"><em>vtxtv</em></abbr>]
- - **VTXTVRecordings**: [<abbr title="netrc machine"><em>vtxtv</em></abbr>]
+ - **VTXTV**: [*vtxtv*](## "netrc machine")
+ - **VTXTVLive**: [*vtxtv*](## "netrc machine")
+ - **VTXTVRecordings**: [*vtxtv*](## "netrc machine")
  - **VuClip**
  - **Vupload**
  - **VVVVID**
@@ -1629,9 +1629,9 @@ # Supported sites
  - **Vzaar**
  - **Wakanim**
  - **Walla**
- - **WalyTV**: [<abbr title="netrc machine"><em>walytv</em></abbr>]
- - **WalyTVLive**: [<abbr title="netrc machine"><em>walytv</em></abbr>]
- - **WalyTVRecordings**: [<abbr title="netrc machine"><em>walytv</em></abbr>]
+ - **WalyTV**: [*walytv*](## "netrc machine")
+ - **WalyTVLive**: [*walytv*](## "netrc machine")
+ - **WalyTVRecordings**: [*walytv*](## "netrc machine")
  - **wasdtv:clip**
  - **wasdtv:record**
  - **wasdtv:stream**
@@ -1743,13 +1743,13 @@ # Supported sites
  - **YoutubeLivestreamEmbed**: YouTube livestream embeds
  - **YoutubeYtBe**: youtu.be
  - **Zapiks**
- - **Zattoo**: [<abbr title="netrc machine"><em>zattoo</em></abbr>]
- - **ZattooLive**: [<abbr title="netrc machine"><em>zattoo</em></abbr>]
- - **ZattooMovies**: [<abbr title="netrc machine"><em>zattoo</em></abbr>]
- - **ZattooRecordings**: [<abbr title="netrc machine"><em>zattoo</em></abbr>]
+ - **Zattoo**: [*zattoo*](## "netrc machine")
+ - **ZattooLive**: [*zattoo*](## "netrc machine")
+ - **ZattooMovies**: [*zattoo*](## "netrc machine")
+ - **ZattooRecordings**: [*zattoo*](## "netrc machine")
  - **ZDF**
  - **ZDFChannel**
- - **Zee5**: [<abbr title="netrc machine"><em>zee5</em></abbr>]
+ - **Zee5**: [*zee5*](## "netrc machine")
  - **zee5:series**
  - **ZeeNews**
  - **ZenYandex**
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index d6c5ce769..00846cd7e 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -614,7 +614,7 @@ def __init__(self, params=None, auto_init=True):
                    '\n                    You will no longer receive updates on this version')
             if current_version < MIN_SUPPORTED:
                 msg = 'Python version %d.%d is no longer supported'
-            self.deprecation_warning(
+            self.deprecated_feature(
                 f'{msg}! Please update to Python %d.%d or above' % (*current_version, *MIN_RECOMMENDED))
 
         if self.params.get('allow_unplayable_formats'):
diff --git a/yt_dlp/dependencies/Cryptodome.py b/yt_dlp/dependencies/Cryptodome.py
index a50bce4d4..74ab6575c 100644
--- a/yt_dlp/dependencies/Cryptodome.py
+++ b/yt_dlp/dependencies/Cryptodome.py
@@ -14,22 +14,14 @@
 try:
     if _parent.__name__ == 'Cryptodome':
         from Cryptodome import __version__
-        from Cryptodome.Cipher import AES
-        from Cryptodome.Cipher import PKCS1_v1_5
-        from Cryptodome.Cipher import Blowfish
-        from Cryptodome.Cipher import PKCS1_OAEP
-        from Cryptodome.Hash import SHA1
-        from Cryptodome.Hash import CMAC
+        from Cryptodome.Cipher import AES, PKCS1_OAEP, Blowfish, PKCS1_v1_5
+        from Cryptodome.Hash import CMAC, SHA1
         from Cryptodome.PublicKey import RSA
     elif _parent.__name__ == 'Crypto':
         from Crypto import __version__
-        from Crypto.Cipher import AES
-        from Crypto.Cipher import PKCS1_v1_5
-        from Crypto.Cipher import Blowfish
-        from Crypto.Cipher import PKCS1_OAEP
-        from Crypto.Hash import SHA1
-        from Crypto.Hash import CMAC
-        from Crypto.PublicKey import RSA
+        from Crypto.Cipher import AES, PKCS1_OAEP, Blowfish, PKCS1_v1_5  # noqa: F401
+        from Crypto.Hash import CMAC, SHA1  # noqa: F401
+        from Crypto.PublicKey import RSA  # noqa: F401
 except ImportError:
     __version__ = f'broken {__version__}'.strip()
 
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 377f138b7..3dc638f52 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -497,7 +497,7 @@ def _download_fragment(fragment):
                 download_fragment(fragment, ctx_copy)
                 return fragment, fragment['frag_index'], ctx_copy.get('fragment_filename_sanitized')
 
-            self.report_warning('The download speed shown is only of one thread. This is a known issue and patches are welcome')
+            self.report_warning('The download speed shown is only of one thread. This is a known issue')
             with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool:
                 try:
                     for fragment, frag_index, frag_filename in pool.map(_download_fragment, fragments):
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 98efe0e9d..8ad63b411 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -3527,7 +3527,7 @@ def description(cls, *, markdown=True, search_examples=None):
         desc = ''
         if cls._NETRC_MACHINE:
             if markdown:
-                desc += f' [<abbr title="netrc machine"><em>{cls._NETRC_MACHINE}</em></abbr>]'
+                desc += f' [*{cls._NETRC_MACHINE}*](## "netrc machine")'
             else:
                 desc += f' [{cls._NETRC_MACHINE}]'
         if cls.IE_DESC is False:
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index d1696349a..44e932293 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -956,7 +956,7 @@ def _extract_response(self, item_id, query, note='Downloading API JSON', headers
 
     @staticmethod
     def is_music_url(url):
-        return re.match(r'https?://music\.youtube\.com/', url) is not None
+        return re.match(r'(https?://)?music\.youtube\.com/', url) is not None
 
     def _extract_video(self, renderer):
         video_id = renderer.get('videoId')
@@ -6211,6 +6211,8 @@ def _real_extract(self, url, smuggled_data):
         original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
             url = f'{pre}/videos{post}'
+        if smuggled_data.get('is_music_url'):
+            self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}')
 
         # Handle both video/playlist URLs
         qs = parse_qs(url)

From 7f51861b1820c37b157a239b1fe30628d907c034 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Wed, 1 Mar 2023 07:56:53 +0000
Subject: [PATCH 101/871] [extractor/youtube] Detect and break on looping
 comments (#6301)

Fixes https://github.com/yt-dlp/yt-dlp/issues/6290

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 44e932293..b02e0153a 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3341,6 +3341,13 @@ def extract_thread(contents):
                 comment = self._extract_comment(comment_renderer, parent)
                 if not comment:
                     continue
+                # Sometimes YouTube may break and give us infinite looping comments.
+                # See: https://github.com/yt-dlp/yt-dlp/issues/6290
+                if comment['id'] in tracker['seen_comment_ids']:
+                    self.report_warning('Detected YouTube comments looping. Stopping comment extraction as we probably cannot get any more.')
+                    yield
+                else:
+                    tracker['seen_comment_ids'].add(comment['id'])
 
                 tracker['running_total'] += 1
                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
@@ -3365,7 +3372,8 @@ def extract_thread(contents):
                 est_total=0,
                 current_page_thread=0,
                 total_parent_comments=0,
-                total_reply_comments=0)
+                total_reply_comments=0,
+                seen_comment_ids=set())
 
         # TODO: Deprecated
         # YouTube comments have a max depth of 2

From b38cae49e6f4849c8ee2a774bdc3c1c647ae5f0e Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Wed, 1 Mar 2023 06:38:02 -0600
Subject: [PATCH 102/871] [extractor/generic] Detect manifest links via
 extension

Authored by: bashonly
---
 yt_dlp/extractor/generic.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index d76ef3e31..49aa5a1f5 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -2393,14 +2393,15 @@ def _real_extract(self, url):
             self.report_detected('direct video link')
             headers = smuggled_data.get('http_headers', {})
             format_id = str(m.group('format_id'))
+            ext = determine_ext(url)
             subtitles = {}
-            if format_id.endswith('mpegurl'):
+            if format_id.endswith('mpegurl') or ext == 'm3u8':
                 formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4', headers=headers)
                 info_dict.update(self._fragment_query(url))
-            elif format_id.endswith('mpd') or format_id.endswith('dash+xml'):
+            elif format_id.endswith('mpd') or format_id.endswith('dash+xml') or ext == 'mpd':
                 formats, subtitles = self._extract_mpd_formats_and_subtitles(url, video_id, headers=headers)
                 info_dict.update(self._fragment_query(url))
-            elif format_id == 'f4m':
+            elif format_id == 'f4m' or ext == 'f4m':
                 formats = self._extract_f4m_formats(url, video_id, headers=headers)
             else:
                 formats = [{

From 9fddc12ab022a31754e0eaa358fc4e1dfa974587 Mon Sep 17 00:00:00 2001
From: std-move <26625259+std-move@users.noreply.github.com>
Date: Thu, 2 Mar 2023 19:33:33 +0100
Subject: [PATCH 103/871] [extractor/iprima] Fix extractor (#6291)

Authored by: std-move
Closes #6187
---
 yt_dlp/extractor/iprima.py | 41 +++++++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/iprima.py b/yt_dlp/extractor/iprima.py
index 181820542..e58e9c2ee 100644
--- a/yt_dlp/extractor/iprima.py
+++ b/yt_dlp/extractor/iprima.py
@@ -7,7 +7,8 @@
     js_to_json,
     urlencode_postdata,
     ExtractorError,
-    parse_qs
+    parse_qs,
+    traverse_obj
 )
 
 
@@ -15,8 +16,7 @@ class IPrimaIE(InfoExtractor):
     _VALID_URL = r'https?://(?!cnn)(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _GEO_BYPASS = False
     _NETRC_MACHINE = 'iprima'
-    _LOGIN_URL = 'https://auth.iprima.cz/oauth2/login'
-    _TOKEN_URL = 'https://auth.iprima.cz/oauth2/token'
+    _AUTH_ROOT = 'https://auth.iprima.cz'
     access_token = None
 
     _TESTS = [{
@@ -67,7 +67,7 @@ def _perform_login(self, username, password):
             return
 
         login_page = self._download_webpage(
-            self._LOGIN_URL, None, note='Downloading login page',
+            f'{self._AUTH_ROOT}/oauth2/login', None, note='Downloading login page',
             errnote='Downloading login page failed')
 
         login_form = self._hidden_inputs(login_page)
@@ -76,11 +76,20 @@ def _perform_login(self, username, password):
             '_email': username,
             '_password': password})
 
-        _, login_handle = self._download_webpage_handle(
-            self._LOGIN_URL, None, data=urlencode_postdata(login_form),
+        profile_select_html, login_handle = self._download_webpage_handle(
+            f'{self._AUTH_ROOT}/oauth2/login', None, data=urlencode_postdata(login_form),
             note='Logging in')
 
-        code = parse_qs(login_handle.geturl()).get('code')[0]
+        # a profile may need to be selected first, even when there is only a single one
+        if '/profile-select' in login_handle.geturl():
+            profile_id = self._search_regex(
+                r'data-identifier\s*=\s*["\']?(\w+)', profile_select_html, 'profile id')
+
+            login_handle = self._request_webpage(
+                f'{self._AUTH_ROOT}/user/profile-select-perform/{profile_id}', None,
+                query={'continueUrl': '/user/login?redirect_uri=/user/'}, note='Selecting profile')
+
+        code = traverse_obj(login_handle.geturl(), ({parse_qs}, 'code', 0))
         if not code:
             raise ExtractorError('Login failed', expected=True)
 
@@ -89,10 +98,10 @@ def _perform_login(self, username, password):
             'client_id': 'prima_sso',
             'grant_type': 'authorization_code',
             'code': code,
-            'redirect_uri': 'https://auth.iprima.cz/sso/auth-check'}
+            'redirect_uri': f'{self._AUTH_ROOT}/sso/auth-check'}
 
         token_data = self._download_json(
-            self._TOKEN_URL, None,
+            f'{self._AUTH_ROOT}/oauth2/token', None,
             note='Downloading token', errnote='Downloading token failed',
             data=urlencode_postdata(token_request_data))
 
@@ -115,14 +124,22 @@ def _real_extract(self, url):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_meta(
+        title = self._html_extract_title(webpage) or self._html_search_meta(
             ['og:title', 'twitter:title'],
             webpage, 'title', default=None)
 
         video_id = self._search_regex((
             r'productId\s*=\s*([\'"])(?P<id>p\d+)\1',
-            r'pproduct_id\s*=\s*([\'"])(?P<id>p\d+)\1'),
-            webpage, 'real id', group='id')
+            r'pproduct_id\s*=\s*([\'"])(?P<id>p\d+)\1',
+        ), webpage, 'real id', group='id', default=None)
+
+        if not video_id:
+            nuxt_data = self._search_nuxt_data(webpage, video_id, traverse='data')
+            video_id = traverse_obj(
+                nuxt_data, (..., 'content', 'additionals', 'videoPlayId', {str}), get_all=False)
+
+        if not video_id:
+            self.raise_no_formats('Unable to extract video ID from webpage')
 
         metadata = self._download_json(
             f'https://api.play-backend.iprima.cz/api/v1//products/id-{video_id}/play',

From 77d6d136468d0c23c8e79bc937898747804f585a Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 3 Mar 2023 03:34:56 -0600
Subject: [PATCH 104/871] [extractor/ntvru] Extract HLS and DASH formats
 (#6403)

Closes #5915
Authored by: bashonly
---
 yt_dlp/extractor/ntvru.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/yt_dlp/extractor/ntvru.py b/yt_dlp/extractor/ntvru.py
index 8d5877daa..91b7724eb 100644
--- a/yt_dlp/extractor/ntvru.py
+++ b/yt_dlp/extractor/ntvru.py
@@ -21,6 +21,7 @@ class NTVRuIE(InfoExtractor):
             'description': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
             'thumbnail': r're:^http://.*\.jpg',
             'duration': 136,
+            'view_count': int,
         },
     }, {
         'url': 'http://www.ntv.ru/video/novosti/750370/',
@@ -32,6 +33,7 @@ class NTVRuIE(InfoExtractor):
             'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
             'thumbnail': r're:^http://.*\.jpg',
             'duration': 172,
+            'view_count': int,
         },
     }, {
         'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
@@ -43,6 +45,7 @@ class NTVRuIE(InfoExtractor):
             'description': '«Сегодня». 21 марта 2014 года. 16:00',
             'thumbnail': r're:^http://.*\.jpg',
             'duration': 1496,
+            'view_count': int,
         },
     }, {
         'url': 'https://www.ntv.ru/kino/Koma_film/m70281/o336036/video/',
@@ -54,6 +57,7 @@ class NTVRuIE(InfoExtractor):
             'description': 'Остросюжетный фильм «Кома»',
             'thumbnail': r're:^http://.*\.jpg',
             'duration': 5592,
+            'view_count': int,
         },
     }, {
         'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
@@ -65,6 +69,7 @@ class NTVRuIE(InfoExtractor):
             'description': '«Дело врачей»: «Деревце жизни»',
             'thumbnail': r're:^http://.*\.jpg',
             'duration': 2590,
+            'view_count': int,
         },
     }, {
         # Schemeless file URL
@@ -115,6 +120,14 @@ def _real_extract(self, url):
                 'url': file_,
                 'filesize': int_or_none(xpath_text(video, './%ssize' % format_id)),
             })
+        hls_manifest = xpath_text(video, './playback/hls')
+        if hls_manifest:
+            formats.extend(self._extract_m3u8_formats(
+                hls_manifest, video_id, m3u8_id='hls', fatal=False))
+        dash_manifest = xpath_text(video, './playback/dash')
+        if dash_manifest:
+            formats.extend(self._extract_mpd_formats(
+                dash_manifest, video_id, mpd_id='dash', fatal=False))
 
         return {
             'id': xpath_text(video, './id'),

From 2d5a8c5db2bd4ff1c2e45e00cd890a10f8ffca9e Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 3 Mar 2023 03:37:23 -0600
Subject: [PATCH 105/871] [extractor/mediastream] Improve WinSports support
 (#6401)

Closes #6360
Authored by: bashonly
---
 yt_dlp/extractor/mediastream.py | 41 +++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/yt_dlp/extractor/mediastream.py b/yt_dlp/extractor/mediastream.py
index 4d3949527..e8d427a31 100644
--- a/yt_dlp/extractor/mediastream.py
+++ b/yt_dlp/extractor/mediastream.py
@@ -1,7 +1,13 @@
 import re
 
 from .common import InfoExtractor
-from ..utils import clean_html, get_element_html_by_class
+from ..utils import (
+    remove_end,
+    str_or_none,
+    strip_or_none,
+    traverse_obj,
+    urljoin,
+)
 
 
 class MediaStreamIE(InfoExtractor):
@@ -117,39 +123,56 @@ def _real_extract(self, url):
 
 
 class WinSportsVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.winsports\.co/videos/(?P<display_id>[\w-]+)-(?P<id>\d+)'
+    _VALID_URL = r'https?://www\.winsports\.co/videos/(?P<id>[\w-]+)'
 
     _TESTS = [{
         'url': 'https://www.winsports.co/videos/siempre-castellanos-gran-atajada-del-portero-cardenal-para-evitar-la-caida-de-su-arco-60536',
         'info_dict': {
             'id': '62dc8357162c4b0821fcfb3c',
-            'display_id': 'siempre-castellanos-gran-atajada-del-portero-cardenal-para-evitar-la-caida-de-su-arco',
+            'display_id': 'siempre-castellanos-gran-atajada-del-portero-cardenal-para-evitar-la-caida-de-su-arco-60536',
             'title': '¡Siempre Castellanos! Gran atajada del portero \'cardenal\' para evitar la caída de su arco',
             'description': 'md5:eb811b2b2882bdc59431732c06b905f2',
             'thumbnail': r're:^https?://[^?#]+62dc8357162c4b0821fcfb3c',
             'ext': 'mp4',
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://www.winsports.co/videos/observa-aqui-los-goles-del-empate-entre-tolima-y-nacional-60548',
         'info_dict': {
             'id': '62dcb875ef12a5526790b552',
-            'display_id': 'observa-aqui-los-goles-del-empate-entre-tolima-y-nacional',
+            'display_id': 'observa-aqui-los-goles-del-empate-entre-tolima-y-nacional-60548',
             'title': 'Observa aquí los goles del empate entre Tolima y Nacional',
             'description': 'md5:b19402ba6e46558b93fd24b873eea9c9',
             'thumbnail': r're:^https?://[^?#]+62dcb875ef12a5526790b552',
             'ext': 'mp4',
         },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.winsports.co/videos/equidad-vuelve-defender-su-arco-de-remates-de-junior',
+        'info_dict': {
+            'id': '63fa7eca72f1741ad3a4d515',
+            'display_id': 'equidad-vuelve-defender-su-arco-de-remates-de-junior',
+            'title': '⚽ Equidad vuelve a defender su arco de remates de Junior',
+            'description': 'Remate de Sierra',
+            'thumbnail': r're:^https?://[^?#]+63fa7eca72f1741ad3a4d515',
+            'ext': 'mp4',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):
-        display_id, video_id = self._match_valid_url(url).group('display_id', 'id')
+        display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-
+        json_ld = self._search_json_ld(webpage, display_id, expected_type='VideoObject', default={})
         media_setting_json = self._search_json(
             r'<script\s*[^>]+data-drupal-selector="drupal-settings-json">', webpage, 'drupal-setting-json', display_id)
 
-        mediastream_id = media_setting_json['settings']['mediastream_formatter'][video_id]['mediastream_id']
+        mediastream_id = traverse_obj(
+            media_setting_json, ('settings', 'mediastream_formatter', ..., 'mediastream_id', {str_or_none}),
+            get_all=False) or json_ld.get('url')
+        if not mediastream_id:
+            self.raise_no_formats('No MediaStream embed found in webpage')
 
         return self.url_result(
-            f'https://mdstrm.com/embed/{mediastream_id}', MediaStreamIE, video_id, url_transparent=True,
-            display_id=display_id, video_title=clean_html(get_element_html_by_class('title-news', webpage)))
+            urljoin('https://mdstrm.com/embed/', mediastream_id), MediaStreamIE, display_id, url_transparent=True,
+            display_id=display_id, video_title=strip_or_none(remove_end(json_ld.get('title'), '| Win Sports')))

From 40d77d89027cd0e0ce31d22aec81db3e1d433900 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 3 Mar 2023 03:42:54 -0600
Subject: [PATCH 106/871] [extractor/yle_areena] Extract non-Kaltura videos
 (#6402)

Closes #6066
Authored by: bashonly
---
 yt_dlp/extractor/yle_areena.py | 37 ++++++++++++++++++++++++++++------
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/yle_areena.py b/yt_dlp/extractor/yle_areena.py
index 98d3b1949..c5b45f0cb 100644
--- a/yt_dlp/extractor/yle_areena.py
+++ b/yt_dlp/extractor/yle_areena.py
@@ -61,7 +61,22 @@ class YleAreenaIE(InfoExtractor):
                 'age_limit': 0,
                 'webpage_url': 'https://areena.yle.fi/1-2158940'
             }
-        }
+        },
+        {
+            'url': 'https://areena.yle.fi/1-64829589',
+            'info_dict': {
+                'id': '1-64829589',
+                'ext': 'mp4',
+                'title': 'HKO & Mälkki & Tanner',
+                'description': 'md5:b4f1b1af2c6569b33f75179a86eea156',
+                'series': 'Helsingin kaupunginorkesterin konsertteja',
+                'thumbnail': r're:^https?://.+\.jpg$',
+                'release_date': '20230120',
+            },
+            'params': {
+                'skip_download': 'm3u8',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -91,12 +106,22 @@ def _real_extract(self, url):
                     'name': sub.get('kind'),
                 })
 
+        kaltura_id = traverse_obj(video_data, ('data', 'ongoing_ondemand', 'kaltura', 'id'), expected_type=str)
+        if kaltura_id:
+            info_dict = {
+                '_type': 'url_transparent',
+                'url': smuggle_url(f'kaltura:1955031:{kaltura_id}', {'source_url': url}),
+                'ie_key': KalturaIE.ie_key(),
+            }
+        else:
+            info_dict = {
+                'id': video_id,
+                'formats': self._extract_m3u8_formats(
+                    video_data['data']['ongoing_ondemand']['manifest_url'], video_id, 'mp4', m3u8_id='hls'),
+            }
+
         return {
-            '_type': 'url_transparent',
-            'url': smuggle_url(
-                f'kaltura:1955031:{video_data["data"]["ongoing_ondemand"]["kaltura"]["id"]}',
-                {'source_url': url}),
-            'ie_key': KalturaIE.ie_key(),
+            **info_dict,
             'title': (traverse_obj(video_data, ('data', 'ongoing_ondemand', 'title', 'fin'), expected_type=str)
                       or episode or info.get('title')),
             'description': description,

From 9acf1ee25f7ad3920ede574a9de95b8c18626af4 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 3 Mar 2023 16:48:54 +0530
Subject: [PATCH 107/871] [jsinterp] Handle `Date` at epoch 0

Closes #6400
---
 test/test_youtube_signature.py | 4 ++++
 yt_dlp/jsinterp.py             | 6 +++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 3203538bb..336e80291 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -66,6 +66,10 @@
 ]
 
 _NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/7862ca1f/player_ias.vflset/en_US/base.js',
+        'X_LCxVDjAavgE5t', 'yxJ1dM6iz5ogUg',
+    ),
     (
         'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
         'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index c2d056aa1..31ab204d7 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -355,11 +355,11 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             obj = expr[4:]
             if obj.startswith('Date('):
                 left, right = self._separate_at_paren(obj[4:])
-                expr = unified_timestamp(
+                date = unified_timestamp(
                     self.interpret_expression(left, local_vars, allow_recursion), False)
-                if not expr:
+                if date is None:
                     raise self.Exception(f'Failed to parse date {left!r}', expr)
-                expr = self._dump(int(expr * 1000), local_vars) + right
+                expr = self._dump(int(date * 1000), local_vars) + right
             else:
                 raise self.Exception(f'Unsupported object {obj}', expr)
 

From d400e261cf029a3f20d364113b14de973be75404 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Fri, 3 Mar 2023 22:31:41 +0530
Subject: [PATCH 108/871] [devscripts] Script to generate changelog (#6220)

Authored by: Grub4K
---
 README.md                                 |   7 +-
 devscripts/changelog_override.json        |   1 +
 devscripts/changelog_override.schema.json |  96 +++++
 devscripts/make_changelog.py              | 491 ++++++++++++++++++++++
 4 files changed, 593 insertions(+), 2 deletions(-)
 create mode 100644 devscripts/changelog_override.json
 create mode 100644 devscripts/changelog_override.schema.json
 create mode 100644 devscripts/make_changelog.py

diff --git a/README.md b/README.md
index 3d3db933a..ddd71eeeb 100644
--- a/README.md
+++ b/README.md
@@ -311,10 +311,13 @@ ### Standalone Py2Exe Builds (Windows)
 
 ### Related scripts
 
-* **`devscripts/update-version.py [revision]`** - Update the version number based on current date
-* **`devscripts/set-variant.py variant [-M update_message]`** - Set the build variant of the executable
+* **`devscripts/update-version.py`** - Update the version number based on current date.
+* **`devscripts/set-variant.py`** - Set the build variant of the executable.
+* **`devscripts/make_changelog.py`** - Create a markdown changelog using short commit messages and update `CONTRIBUTORS` file.
 * **`devscripts/make_lazy_extractors.py`** - Create lazy extractors. Running this before building the binaries (any variant) will improve their startup performance. Set the environment variable `YTDLP_NO_LAZY_EXTRACTORS=1` if you wish to forcefully disable lazy extractor loading.
 
+Note: See their `--help` for more info.
+
 You can also fork the project on GitHub and run your fork's [build workflow](.github/workflows/build.yml) to automatically build a full release
 
 # USAGE AND OPTIONS
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/devscripts/changelog_override.json
@@ -0,0 +1 @@
+{}
diff --git a/devscripts/changelog_override.schema.json b/devscripts/changelog_override.schema.json
new file mode 100644
index 000000000..9bd747b70
--- /dev/null
+++ b/devscripts/changelog_override.schema.json
@@ -0,0 +1,96 @@
+{
+    "$schema": "http://json-schema.org/draft/2020-12/schema",
+    "type": "array",
+    "uniqueItems": true,
+    "items": {
+        "type": "object",
+        "oneOf": [
+            {
+                "type": "object",
+                "properties": {
+                    "action": {
+                        "enum": [
+                            "add"
+                        ]
+                    },
+                    "when": {
+                        "type": "string",
+                        "pattern": "^([0-9a-f]{40}|\\d{4}\\.\\d{2}\\.\\d{2})$"
+                    },
+                    "hash": {
+                        "type": "string",
+                        "pattern": "^[0-9a-f]{40}$"
+                    },
+                    "short": {
+                        "type": "string"
+                    },
+                    "authors": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }
+                },
+                "required": [
+                    "action",
+                    "short"
+                ]
+            },
+            {
+                "type": "object",
+                "properties": {
+                    "action": {
+                        "enum": [
+                            "remove"
+                        ]
+                    },
+                    "when": {
+                        "type": "string",
+                        "pattern": "^([0-9a-f]{40}|\\d{4}\\.\\d{2}\\.\\d{2})$"
+                    },
+                    "hash": {
+                        "type": "string",
+                        "pattern": "^[0-9a-f]{40}$"
+                    }
+                },
+                "required": [
+                    "action",
+                    "hash"
+                ]
+            },
+            {
+                "type": "object",
+                "properties": {
+                    "action": {
+                        "enum": [
+                            "change"
+                        ]
+                    },
+                    "when": {
+                        "type": "string",
+                        "pattern": "^([0-9a-f]{40}|\\d{4}\\.\\d{2}\\.\\d{2})$"
+                    },
+                    "hash": {
+                        "type": "string",
+                        "pattern": "^[0-9a-f]{40}$"
+                    },
+                    "short": {
+                        "type": "string"
+                    },
+                    "authors": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }
+                },
+                "required": [
+                    "action",
+                    "hash",
+                    "short",
+                    "authors"
+                ]
+            }
+        ]
+    }
+}
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
new file mode 100644
index 000000000..b66181b53
--- /dev/null
+++ b/devscripts/make_changelog.py
@@ -0,0 +1,491 @@
+from __future__ import annotations
+
+import enum
+import itertools
+import json
+import logging
+import re
+import subprocess
+import sys
+from collections import defaultdict
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+BASE_URL = 'https://github.com'
+LOCATION_PATH = Path(__file__).parent
+
+logger = logging.getLogger(__name__)
+
+
+class CommitGroup(enum.Enum):
+    UPSTREAM = None
+    PRIORITY = 'Important'
+    CORE = 'Core'
+    EXTRACTOR = 'Extractor'
+    DOWNLOADER = 'Downloader'
+    POSTPROCESSOR = 'Postprocessor'
+    MISC = 'Misc.'
+
+    @classmethod
+    @lru_cache
+    def commit_lookup(cls):
+        return {
+            name: group
+            for group, names in {
+                cls.PRIORITY: {''},
+                cls.UPSTREAM: {'upstream'},
+                cls.CORE: {
+                    'aes',
+                    'cache',
+                    'compat_utils',
+                    'compat',
+                    'cookies',
+                    'core',
+                    'dependencies',
+                    'jsinterp',
+                    'outtmpl',
+                    'plugins',
+                    'update',
+                    'utils',
+                },
+                cls.MISC: {
+                    'build',
+                    'cleanup',
+                    'devscripts',
+                    'docs',
+                    'misc',
+                    'test',
+                },
+                cls.EXTRACTOR: {'extractor', 'extractors'},
+                cls.DOWNLOADER: {'downloader'},
+                cls.POSTPROCESSOR: {'postprocessor'},
+            }.items()
+            for name in names
+        }
+
+    @classmethod
+    def get(cls, value):
+        result = cls.commit_lookup().get(value)
+        if result:
+            logger.debug(f'Mapped {value!r} => {result.name}')
+        return result
+
+
+@dataclass
+class Commit:
+    hash: str | None
+    short: str
+    authors: list[str]
+
+    def __str__(self):
+        result = f'{self.short!r}'
+
+        if self.hash:
+            result += f' ({self.hash[:7]})'
+
+        if self.authors:
+            authors = ', '.join(self.authors)
+            result += f' by {authors}'
+
+        return result
+
+
+@dataclass
+class CommitInfo:
+    details: str | None
+    sub_details: tuple[str, ...]
+    message: str
+    issues: list[str]
+    commit: Commit
+    fixes: list[Commit]
+
+    def key(self):
+        return ((self.details or '').lower(), self.sub_details, self.message)
+
+
+class Changelog:
+    MISC_RE = re.compile(r'(?:^|\b)(?:lint(?:ing)?|misc|format(?:ting)?|fixes)(?:\b|$)', re.IGNORECASE)
+
+    def __init__(self, groups, repo):
+        self._groups = groups
+        self._repo = repo
+
+    def __str__(self):
+        return '\n'.join(self._format_groups(self._groups)).replace('\t', '    ')
+
+    def _format_groups(self, groups):
+        for item in CommitGroup:
+            group = groups[item]
+            if group:
+                yield self.format_module(item.value, group)
+
+    def format_module(self, name, group):
+        result = f'\n#### {name} changes\n' if name else '\n'
+        return result + '\n'.join(self._format_group(group))
+
+    def _format_group(self, group):
+        sorted_group = sorted(group, key=CommitInfo.key)
+        detail_groups = itertools.groupby(sorted_group, lambda item: (item.details or '').lower())
+        for details, items in detail_groups:
+            if not details:
+                indent = ''
+            else:
+                yield f'- {details}'
+                indent = '\t'
+
+            if details == 'cleanup':
+                items, cleanup_misc_items = self._filter_cleanup_misc_items(items)
+
+            sub_detail_groups = itertools.groupby(items, lambda item: item.sub_details)
+            for sub_details, entries in sub_detail_groups:
+                if not sub_details:
+                    for entry in entries:
+                        yield f'{indent}- {self.format_single_change(entry)}'
+                    continue
+
+                prefix = f'{indent}- {", ".join(sub_details)}'
+                entries = list(entries)
+                if len(entries) == 1:
+                    yield f'{prefix}: {self.format_single_change(entries[0])}'
+                    continue
+
+                yield prefix
+                for entry in entries:
+                    yield f'{indent}\t- {self.format_single_change(entry)}'
+
+            if details == 'cleanup' and cleanup_misc_items:
+                yield from self._format_cleanup_misc_sub_group(cleanup_misc_items)
+
+    def _filter_cleanup_misc_items(self, items):
+        cleanup_misc_items = defaultdict(list)
+        non_misc_items = []
+        for item in items:
+            if self.MISC_RE.search(item.message):
+                cleanup_misc_items[tuple(item.commit.authors)].append(item)
+            else:
+                non_misc_items.append(item)
+
+        return non_misc_items, cleanup_misc_items
+
+    def _format_cleanup_misc_sub_group(self, group):
+        prefix = '\t- Miscellaneous'
+        if len(group) == 1:
+            yield f'{prefix}: {next(self._format_cleanup_misc_items(group))}'
+            return
+
+        yield prefix
+        for message in self._format_cleanup_misc_items(group):
+            yield f'\t\t- {message}'
+
+    def _format_cleanup_misc_items(self, group):
+        for authors, infos in group.items():
+            message = ', '.join(
+                self._format_message_link(None, info.commit.hash)
+                for info in sorted(infos, key=lambda item: item.commit.hash or ''))
+            yield f'{message} by {self._format_authors(authors)}'
+
+    def format_single_change(self, info):
+        message = self._format_message_link(info.message, info.commit.hash)
+        if info.issues:
+            message = f'{message} ({self._format_issues(info.issues)})'
+
+        if info.commit.authors:
+            message = f'{message} by {self._format_authors(info.commit.authors)}'
+
+        if info.fixes:
+            fix_message = ', '.join(f'{self._format_message_link(None, fix.hash)}' for fix in info.fixes)
+
+            authors = sorted({author for fix in info.fixes for author in fix.authors}, key=str.casefold)
+            if authors != info.commit.authors:
+                fix_message = f'{fix_message} by {self._format_authors(authors)}'
+
+            message = f'{message} (With fixes in {fix_message})'
+
+        return message
+
+    def _format_message_link(self, message, hash):
+        assert message or hash, 'Improperly defined commit message or override'
+        message = message if message else hash[:7]
+        return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message
+
+    def _format_issues(self, issues):
+        return ', '.join(f'[#{issue}]({self.repo_url}/issues/{issue})' for issue in issues)
+
+    @staticmethod
+    def _format_authors(authors):
+        return ', '.join(f'[{author}]({BASE_URL}/{author})' for author in authors)
+
+    @property
+    def repo_url(self):
+        return f'{BASE_URL}/{self._repo}'
+
+
+class CommitRange:
+    COMMAND = 'git'
+    COMMIT_SEPARATOR = '-----'
+
+    AUTHOR_INDICATOR_RE = re.compile(r'Authored by:? ', re.IGNORECASE)
+    MESSAGE_RE = re.compile(r'''
+        (?:\[
+            (?P<prefix>[^\]\/:,]+)
+            (?:/(?P<details>[^\]:,]+))?
+            (?:[:,](?P<sub_details>[^\]]+))?
+        \]\ )?
+        (?:`?(?P<sub_details_alt>[^:`]+)`?: )?
+        (?P<message>.+?)
+        (?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
+        ''', re.VERBOSE | re.DOTALL)
+    EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE)
+    FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+for)?|Revert)\s+([\da-f]{40})')
+    UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)')
+
+    def __init__(self, start, end, default_author=None) -> None:
+        self._start = start
+        self._end = end
+        self._commits, self._fixes = self._get_commits_and_fixes(default_author)
+        self._commits_added = []
+
+    @classmethod
+    def from_single(cls, commitish='HEAD', default_author=None):
+        start_commitish = cls.get_prev_tag(commitish)
+        end_commitish = cls.get_next_tag(commitish)
+        if start_commitish == end_commitish:
+            start_commitish = cls.get_prev_tag(f'{commitish}~')
+        logger.info(f'Determined range from {commitish!r}: {start_commitish}..{end_commitish}')
+        return cls(start_commitish, end_commitish, default_author)
+
+    @classmethod
+    def get_prev_tag(cls, commitish):
+        command = [cls.COMMAND, 'describe', '--tags', '--abbrev=0', '--exclude=*[^0-9.]*', commitish]
+        return subprocess.check_output(command, text=True).strip()
+
+    @classmethod
+    def get_next_tag(cls, commitish):
+        result = subprocess.run(
+            [cls.COMMAND, 'describe', '--contains', '--abbrev=0', commitish],
+            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
+        if result.returncode:
+            return 'HEAD'
+
+        return result.stdout.partition('~')[0].strip()
+
+    def __iter__(self):
+        return iter(itertools.chain(self._commits.values(), self._commits_added))
+
+    def __len__(self):
+        return len(self._commits) + len(self._commits_added)
+
+    def __contains__(self, commit):
+        if isinstance(commit, Commit):
+            if not commit.hash:
+                return False
+            commit = commit.hash
+
+        return commit in self._commits
+
+    def _is_ancestor(self, commitish):
+        return bool(subprocess.call(
+            [self.COMMAND, 'merge-base', '--is-ancestor', commitish, self._start]))
+
+    def _get_commits_and_fixes(self, default_author):
+        result = subprocess.check_output([
+            self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}',
+            f'{self._start}..{self._end}'], text=True)
+
+        commits = {}
+        fixes = defaultdict(list)
+        lines = iter(result.splitlines(False))
+        for line in lines:
+            commit_hash = line
+            short = next(lines)
+            skip = short.startswith('Release ') or short == '[version] update'
+
+            authors = [default_author] if default_author else []
+            for line in iter(lambda: next(lines), self.COMMIT_SEPARATOR):
+                match = self.AUTHOR_INDICATOR_RE.match(line)
+                if match:
+                    authors = sorted(map(str.strip, line[match.end():].split(',')), key=str.casefold)
+
+            commit = Commit(commit_hash, short, authors)
+            if skip:
+                logger.debug(f'Skipped commit: {commit}')
+                continue
+
+            fix_match = self.FIXES_RE.search(commit.short)
+            if fix_match:
+                commitish = fix_match.group(1)
+                fixes[commitish].append(commit)
+
+            commits[commit.hash] = commit
+
+        for commitish, fix_commits in fixes.items():
+            if commitish in commits:
+                hashes = ', '.join(commit.hash[:7] for commit in fix_commits)
+                logger.info(f'Found fix(es) for {commitish[:7]}: {hashes}')
+                for fix_commit in fix_commits:
+                    del commits[fix_commit.hash]
+            else:
+                logger.debug(f'Commit with fixes not in changes: {commitish[:7]}')
+
+        return commits, fixes
+
+    def apply_overrides(self, overrides):
+        for override in overrides:
+            when = override.get('when')
+            if when and when not in self and when != self._start:
+                logger.debug(f'Ignored {when!r}, not in commits {self._start!r}')
+                continue
+
+            override_hash = override.get('hash')
+            if override['action'] == 'add':
+                commit = Commit(override.get('hash'), override['short'], override.get('authors') or [])
+                logger.info(f'ADD    {commit}')
+                self._commits_added.append(commit)
+
+            elif override['action'] == 'remove':
+                if override_hash in self._commits:
+                    logger.info(f'REMOVE {self._commits[override_hash]}')
+                    del self._commits[override_hash]
+
+            elif override['action'] == 'change':
+                if override_hash not in self._commits:
+                    continue
+                commit = Commit(override_hash, override['short'], override['authors'])
+                logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}')
+                self._commits[commit.hash] = commit
+
+        self._commits = {key: value for key, value in reversed(self._commits.items())}
+
+    def groups(self):
+        groups = defaultdict(list)
+        for commit in self:
+            upstream_re = self.UPSTREAM_MERGE_RE.match(commit.short)
+            if upstream_re:
+                commit.short = f'[upstream] Merge up to youtube-dl {upstream_re.group(1)}'
+
+            match = self.MESSAGE_RE.fullmatch(commit.short)
+            if not match:
+                logger.error(f'Error parsing short commit message: {commit.short!r}')
+                continue
+
+            prefix, details, sub_details, sub_details_alt, message, issues = match.groups()
+            group = None
+            if prefix:
+                if prefix == 'priority':
+                    prefix, _, details = (details or '').partition('/')
+                    logger.debug(f'Priority: {message!r}')
+                    group = CommitGroup.PRIORITY
+
+                if not details and prefix:
+                    if prefix not in ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream'):
+                        logger.debug(f'Replaced details with {prefix!r}')
+                        details = prefix or None
+
+                if details == 'common':
+                    details = None
+
+                if details:
+                    details = details.strip()
+
+            else:
+                group = CommitGroup.CORE
+
+            sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.lower().replace(':', ',')
+            sub_details = tuple(filter(None, map(str.strip, sub_details.split(','))))
+
+            issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else []
+
+            if not group:
+                group = CommitGroup.get(prefix.lower())
+                if not group:
+                    if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
+                        group = CommitGroup.EXTRACTOR
+                    else:
+                        group = CommitGroup.POSTPROCESSOR
+                    logger.warning(f'Failed to map {commit.short!r}, selected {group.name}')
+
+            commit_info = CommitInfo(
+                details, sub_details, message.strip(),
+                issues, commit, self._fixes[commit.hash])
+            logger.debug(f'Resolved {commit.short!r} to {commit_info!r}')
+            groups[group].append(commit_info)
+
+        return groups
+
+
+def get_new_contributors(contributors_path, commits):
+    contributors = set()
+    if contributors_path.exists():
+        with contributors_path.open() as file:
+            for line in filter(None, map(str.strip, file)):
+                author, _, _ = line.partition(' (')
+                authors = author.split('/')
+                contributors.update(map(str.casefold, authors))
+
+    new_contributors = set()
+    for commit in commits:
+        for author in commit.authors:
+            author_folded = author.casefold()
+            if author_folded not in contributors:
+                contributors.add(author_folded)
+                new_contributors.add(author)
+
+    return sorted(new_contributors, key=str.casefold)
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description='Create a changelog markdown from a git commit range')
+    parser.add_argument(
+        'commitish', default='HEAD', nargs='?',
+        help='The commitish to create the range from (default: %(default)s)')
+    parser.add_argument(
+        '-v', '--verbosity', action='count', default=0,
+        help='increase verbosity (can be used twice)')
+    parser.add_argument(
+        '-c', '--contributors', action='store_true',
+        help='update CONTRIBUTORS file (default: %(default)s)')
+    parser.add_argument(
+        '--contributors-path', type=Path, default=LOCATION_PATH.parent / 'CONTRIBUTORS',
+        help='path to the CONTRIBUTORS file')
+    parser.add_argument(
+        '--no-override', action='store_true',
+        help='skip override json in commit generation (default: %(default)s)')
+    parser.add_argument(
+        '--override-path', type=Path, default=LOCATION_PATH / 'changelog_override.json',
+        help='path to the changelog_override.json file')
+    parser.add_argument(
+        '--default-author', default='pukkandan',
+        help='the author to use without a author indicator (default: %(default)s)')
+    parser.add_argument(
+        '--repo', default='yt-dlp/yt-dlp',
+        help='the github repository to use for the operations (default: %(default)s)')
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}',
+        level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr)
+
+    commits = CommitRange.from_single(args.commitish, args.default_author)
+
+    if not args.no_override:
+        if args.override_path.exists():
+            with args.override_path.open() as file:
+                overrides = json.load(file)
+            commits.apply_overrides(overrides)
+        else:
+            logger.warning(f'File {args.override_path.as_posix()} does not exist')
+
+    logger.info(f'Loaded {len(commits)} commits')
+
+    new_contributors = get_new_contributors(args.contributors_path, commits)
+    if new_contributors:
+        if args.contributors:
+            with args.contributors_path.open('a') as file:
+                file.writelines(f'{contributor}\n' for contributor in new_contributors)
+        logger.info(f'New contributors: {", ".join(new_contributors)}')
+
+    print(Changelog(commits.groups(), args.repo))

From 29cb20bd563c02671b31dd840139e93dd37150a1 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Fri, 3 Mar 2023 22:33:12 +0530
Subject: [PATCH 109/871] [build] Automated builds and nightly releases (#6220)

Closes #1839
Authored by: Grub4K, bashonly

Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
---
 .github/workflows/build.yml           | 541 ++++++++++++--------------
 .github/workflows/publish.yml         |  80 ++++
 .github/workflows/release-nightly.yml |  49 +++
 .github/workflows/release.yml         | 125 ++++++
 Changelog.md                          |   8 +-
 README.md                             |   8 +-
 devscripts/make_readme.py             |  22 +-
 devscripts/update-version.py          |  46 ++-
 yt_dlp/YoutubeDL.py                   |   6 +-
 9 files changed, 552 insertions(+), 333 deletions(-)
 create mode 100644 .github/workflows/publish.yml
 create mode 100644 .github/workflows/release-nightly.yml
 create mode 100644 .github/workflows/release.yml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 6041376a4..2183903ea 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -1,393 +1,338 @@
-name: Build
-on: workflow_dispatch
+name: Build Artifacts
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+      channel:
+        required: false
+        default: stable
+        type: string
+      unix:
+        default: true
+        type: boolean
+      linux_arm:
+        default: true
+        type: boolean
+      macos:
+        default: true
+        type: boolean
+      macos_legacy:
+        default: true
+        type: boolean
+      windows:
+        default: true
+        type: boolean
+      windows32:
+        default: true
+        type: boolean
+      meta_files:
+        default: true
+        type: boolean
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Version tag (YYYY.MM.DD[.REV])
+        required: true
+        type: string
+      channel:
+        description: Update channel (stable/nightly)
+        required: true
+        default: stable
+        type: string
+      unix:
+        description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip
+        default: true
+        type: boolean
+      linux_arm:
+        description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l
+        default: true
+        type: boolean
+      macos:
+        description: yt-dlp_macos, yt-dlp_macos.zip
+        default: true
+        type: boolean
+      macos_legacy:
+        description: yt-dlp_macos_legacy
+        default: true
+        type: boolean
+      windows:
+        description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip
+        default: true
+        type: boolean
+      windows32:
+        description: yt-dlp_x86.exe
+        default: true
+        type: boolean
+      meta_files:
+        description: SHA2-256SUMS, SHA2-512SUMS, _update_spec
+        default: true
+        type: boolean
+
 permissions:
   contents: read
 
 jobs:
-  prepare:
-    permissions:
-      contents: write  # for push_release
+  unix:
+    if: inputs.unix
     runs-on: ubuntu-latest
-    outputs:
-      version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
-      ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
-      head_sha: ${{ steps.push_release.outputs.head_sha }}
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@v4
-      with:
-          python-version: '3.10'
-
-    - name: Set version suffix
-      id: version_suffix
-      env:
-        PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
-      if: "env.PUSH_VERSION_COMMIT == ''"
-      run: echo "version_suffix=$(date -u +"%H%M%S")" >> "$GITHUB_OUTPUT"
-    - name: Bump version
-      id: bump_version
-      run: |
-        python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }}
-        make issuetemplates
-
-    - name: Push to release
-      id: push_release
-      run: |
-        git config --global user.name github-actions
-        git config --global user.email github-actions@example.com
-        git add -u
-        git commit -m "[version] update" -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
-        git push origin --force ${{ github.event.ref }}:release
-        echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
-    - name: Update master
-      env:
-        PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
-      if: "env.PUSH_VERSION_COMMIT != ''"
-      run: git push origin ${{ github.event.ref }}
-
-
-  build_unix:
-    needs: prepare
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:
-          python-version: '3.10'
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
           miniforge-variant: Mambaforge
           use-mamba: true
           channels: conda-forge
           auto-update-conda: true
-          activate-environment: ''
+          activate-environment: ""
           auto-activate-base: false
-    - name: Install Requirements
-      run: |
+      - name: Install Requirements
+        run: |
           sudo apt-get -y install zip pandoc man sed
-          python -m pip install -U pip setuptools wheel twine
+          python -m pip install -U pip setuptools wheel
           python -m pip install -U Pyinstaller -r requirements.txt
           reqs=$(mktemp)
           echo -e 'python=3.10.*\npyinstaller' >$reqs
           sed 's/^brotli.*/brotli-python/' <requirements.txt >>$reqs
           mamba create -n build --file $reqs
 
-    - name: Prepare
-      run: |
-          python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+      - name: Prepare
+        run: |
+          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           python devscripts/make_lazy_extractors.py
-    - name: Build Unix platform-independent binary
-      run: |
+      - name: Build Unix platform-independent binary
+        run: |
           make all tar
-    - name: Build Unix standalone binary
-      shell: bash -l {0}
-      run: |
+      - name: Build Unix standalone binary
+        shell: bash -l {0}
+        run: |
           unset LD_LIBRARY_PATH  # Harmful; set by setup-python
           conda activate build
           python pyinst.py --onedir
           (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
           python pyinst.py
+          mv ./dist/yt-dlp_linux ./yt-dlp_linux
+          mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip
 
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        path: |
-          yt-dlp
-          yt-dlp.tar.gz
-          dist/yt-dlp_linux
-          dist/yt-dlp_linux.zip
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          path: |
+            yt-dlp
+            yt-dlp.tar.gz
+            yt-dlp_linux
+            yt-dlp_linux.zip
 
-    - name: Build and publish on PyPi
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-      if: "env.TWINE_PASSWORD != ''"
-      run: |
-        rm -rf dist/*
-        python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
-
-    - name: Install SSH private key for Homebrew
-      env:
-        BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
-      if: "env.BREW_TOKEN != ''"
-      uses: yt-dlp/ssh-agent@v0.5.3
-      with:
-          ssh-private-key: ${{ env.BREW_TOKEN }}
-    - name: Update Homebrew Formulae
-      env:
-        BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
-      if: "env.BREW_TOKEN != ''"
-      run: |
-        git clone git@github.com:yt-dlp/homebrew-taps taps/
-        python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.ytdlp_version }}"
-        git -C taps/ config user.name github-actions
-        git -C taps/ config user.email github-actions@example.com
-        git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.ytdlp_version }}'
-        git -C taps/ push
-
-
-  build_linux_arm:
+  linux_arm:
+    if: inputs.linux_arm
     permissions:
-      packages: write  # for Creating cache
+      contents: read
+      packages: write # for creating cache
     runs-on: ubuntu-latest
-    needs: prepare
     strategy:
       matrix:
         architecture:
-        - armv7
-        - aarch64
+          - armv7
+          - aarch64
 
     steps:
-    - uses: actions/checkout@v3
-      with:
-        path: ./repo
-    - name: Virtualized Install, Prepare & Build
-      uses: yt-dlp/run-on-arch-action@v2
-      with:
-        githubToken: ${{ github.token }}  # To cache image
-        arch: ${{ matrix.architecture }}
-        distro: ubuntu18.04  # Standalone executable should be built on minimum supported OS
-        dockerRunArgs: --volume "${PWD}/repo:/repo"
-        install: |  # Installing Python 3.10 from the Deadsnakes repo raises errors
-          apt update
-          apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
-          python3.8 -m pip install -U pip setuptools wheel
-          # Cannot access requirements.txt from the repo directory at this stage
-          python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi
+      - uses: actions/checkout@v3
+        with:
+          path: ./repo
+      - name: Virtualized Install, Prepare & Build
+        uses: yt-dlp/run-on-arch-action@v2
+        with:
+          # Ref: https://github.com/uraimo/run-on-arch-action/issues/55
+          env: |
+            GITHUB_WORKFLOW: build
+          githubToken: ${{ github.token }} # To cache image
+          arch: ${{ matrix.architecture }}
+          distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS
+          dockerRunArgs: --volume "${PWD}/repo:/repo"
+          install: | # Installing Python 3.10 from the Deadsnakes repo raises errors
+            apt update
+            apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
+            python3.8 -m pip install -U pip setuptools wheel
+            # Cannot access requirements.txt from the repo directory at this stage
+            python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi
 
-        run: |
-          cd repo
-          python3.8 -m pip install -U Pyinstaller -r requirements.txt  # Cached version may be out of date
-          python3.8 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
-          python3.8 devscripts/make_lazy_extractors.py
-          python3.8 pyinst.py
+          run: |
+            cd repo
+            python3.8 -m pip install -U Pyinstaller -r requirements.txt  # Cached version may be out of date
+            python3.8 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+            python3.8 devscripts/make_lazy_extractors.py
+            python3.8 pyinst.py
 
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        path: |  # run-on-arch-action designates armv7l as armv7
-          repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          path: | # run-on-arch-action designates armv7l as armv7
+            repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
 
-
-  build_macos:
+  macos:
+    if: inputs.macos
     runs-on: macos-11
-    needs: prepare
 
     steps:
-    - uses: actions/checkout@v3
-    # NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used
-    - name: Install Requirements
-      run: |
+      - uses: actions/checkout@v3
+      # NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used
+      - name: Install Requirements
+        run: |
           brew install coreutils
           /usr/bin/python3 -m pip install -U --user pip Pyinstaller -r requirements.txt
 
-    - name: Prepare
-      run: |
-          /usr/bin/python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+      - name: Prepare
+        run: |
+          /usr/bin/python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           /usr/bin/python3 devscripts/make_lazy_extractors.py
-    - name: Build
-      run: |
+      - name: Build
+        run: |
           /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir
           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
           /usr/bin/python3 pyinst.py --target-architecture universal2
 
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        path: |
-          dist/yt-dlp_macos
-          dist/yt-dlp_macos.zip
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          path: |
+            dist/yt-dlp_macos
+            dist/yt-dlp_macos.zip
 
-
-  build_macos_legacy:
+  macos_legacy:
+    if: inputs.macos_legacy
     runs-on: macos-latest
-    needs: prepare
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Install Python
-      # We need the official Python, because the GA ones only support newer macOS versions
-      env:
-        PYTHON_VERSION: 3.10.5
-        MACOSX_DEPLOYMENT_TARGET: 10.9  # Used up by the Python build tools
-      run: |
+      - uses: actions/checkout@v3
+      - name: Install Python
+        # We need the official Python, because the GA ones only support newer macOS versions
+        env:
+          PYTHON_VERSION: 3.10.5
+          MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools
+        run: |
           # Hack to get the latest patch version. Uncomment if needed
           #brew install python@3.10
           #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
           curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg"
           sudo installer -pkg python.pkg -target /
           python3 --version
-    - name: Install Requirements
-      run: |
+      - name: Install Requirements
+        run: |
           brew install coreutils
           python3 -m pip install -U --user pip Pyinstaller -r requirements.txt
 
-    - name: Prepare
-      run: |
-          python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+      - name: Prepare
+        run: |
+          python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           python3 devscripts/make_lazy_extractors.py
-    - name: Build
-      run: |
+      - name: Build
+        run: |
           python3 pyinst.py
           mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
 
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        path: |
-          dist/yt-dlp_macos_legacy
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          path: |
+            dist/yt-dlp_macos_legacy
 
-
-  build_windows:
+  windows:
+    if: inputs.windows
     runs-on: windows-latest
-    needs: prepare
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:  # 3.8 is used for Win7 support
-          python-version: '3.8'
-    - name: Install Requirements
-      run: |  # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with: # 3.8 is used for Win7 support
+          python-version: "3.8"
+      - name: Install Requirements
+        run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
           python -m pip install -U pip setuptools wheel py2exe
           pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
 
-    - name: Prepare
-      run: |
-          python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+      - name: Prepare
+        run: |
+          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           python devscripts/make_lazy_extractors.py
-    - name: Build
-      run: |
+      - name: Build
+        run: |
           python setup.py py2exe
           Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
           python pyinst.py
           python pyinst.py --onedir
           Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
 
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        path: |
-          dist/yt-dlp.exe
-          dist/yt-dlp_min.exe
-          dist/yt-dlp_win.zip
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          path: |
+            dist/yt-dlp.exe
+            dist/yt-dlp_min.exe
+            dist/yt-dlp_win.zip
 
-
-  build_windows32:
+  windows32:
+    if: inputs.windows32
     runs-on: windows-latest
-    needs: prepare
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:  # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390
-          python-version: '3.7'
-          architecture: 'x86'
-    - name: Install Requirements
-      run: |
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390
+          python-version: "3.7"
+          architecture: "x86"
+      - name: Install Requirements
+        run: |
           python -m pip install -U pip setuptools wheel
           pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
 
-    - name: Prepare
-      run: |
-          python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+      - name: Prepare
+        run: |
+          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           python devscripts/make_lazy_extractors.py
-    - name: Build
-      run: |
+      - name: Build
+        run: |
           python pyinst.py
 
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        path: |
-          dist/yt-dlp_x86.exe
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          path: |
+            dist/yt-dlp_x86.exe
 
-
-  publish_release:
-    permissions:
-      contents: write  # for action-gh-release
+  meta_files:
+    if: inputs.meta_files && always()
+    needs:
+      - unix
+      - linux_arm
+      - macos
+      - macos_legacy
+      - windows
+      - windows32
     runs-on: ubuntu-latest
-    needs: [prepare, build_unix, build_linux_arm, build_windows, build_windows32, build_macos, build_macos_legacy]
-
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v3
 
-    - name: Get Changelog
-      run: |
-        changelog=$(grep -oPz '(?s)(?<=### ${{ needs.prepare.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)' Changelog.md) || true
-        echo "changelog<<EOF" >> $GITHUB_ENV
-        echo "$changelog" >> $GITHUB_ENV
-        echo "EOF" >> $GITHUB_ENV
-    - name: Make Update spec
-      run: |
-        echo "# This file is used for regulating self-update" >> _update_spec
-        echo "lock 2022.07.18 .+ Python 3.6" >> _update_spec
-    - name: Make SHA2-SUMS files
-      run: |
-          sha256sum artifact/yt-dlp | awk '{print $1 "  yt-dlp"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp.tar.gz | awk '{print $1 "  yt-dlp.tar.gz"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp.exe | awk '{print $1 "  yt-dlp.exe"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_win.zip | awk '{print $1 "  yt-dlp_win.zip"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_min.exe | awk '{print $1 "  yt-dlp_min.exe"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_x86.exe | awk '{print $1 "  yt-dlp_x86.exe"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_macos | awk '{print $1 "  yt-dlp_macos"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_macos.zip | awk '{print $1 "  yt-dlp_macos.zip"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_macos_legacy | awk '{print $1 "  yt-dlp_macos_legacy"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_linux_armv7l | awk '{print $1 "  yt-dlp_linux_armv7l"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_linux_aarch64 | awk '{print $1 "  yt-dlp_linux_aarch64"}' >> SHA2-256SUMS
-          sha256sum artifact/dist/yt-dlp_linux | awk '{print $1 "  yt-dlp_linux"}' >> SHA2-256SUMS
-          sha256sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 "  yt-dlp_linux.zip"}' >> SHA2-256SUMS
-          sha512sum artifact/yt-dlp | awk '{print $1 "  yt-dlp"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp.tar.gz | awk '{print $1 "  yt-dlp.tar.gz"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp.exe | awk '{print $1 "  yt-dlp.exe"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_win.zip | awk '{print $1 "  yt-dlp_win.zip"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_min.exe | awk '{print $1 "  yt-dlp_min.exe"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_x86.exe | awk '{print $1 "  yt-dlp_x86.exe"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_macos | awk '{print $1 "  yt-dlp_macos"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_macos.zip | awk '{print $1 "  yt-dlp_macos.zip"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_macos_legacy | awk '{print $1 "  yt-dlp_macos_legacy"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_linux_armv7l | awk '{print $1 "  yt-dlp_linux_armv7l"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_linux_aarch64 | awk '{print $1 "  yt-dlp_linux_aarch64"}' >> SHA2-512SUMS
-          sha512sum artifact/dist/yt-dlp_linux | awk '{print $1 "  yt-dlp_linux"}' >> SHA2-512SUMS
-          sha512sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 "  yt-dlp_linux.zip"}' >> SHA2-512SUMS
+      - name: Make SHA2-SUMS files
+        run: |
+          cd ./artifact/
+          sha256sum * > ../SHA2-256SUMS
+          sha512sum * > ../SHA2-512SUMS
 
-    - name: Publish Release
-      uses: yt-dlp/action-gh-release@v1
-      with:
-        tag_name: ${{ needs.prepare.outputs.ytdlp_version }}
-        name: yt-dlp ${{ needs.prepare.outputs.ytdlp_version }}
-        target_commitish: ${{ needs.prepare.outputs.head_sha }}
-        body: |
-          #### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
+      - name: Make Update spec
+        run: |
+          cat >> _update_spec << EOF
+          # This file is used for regulating self-update
+          lock 2022.08.18.36 .+ Python 3.6
+          EOF
 
-          ---
-          <details open><summary><h3>Changelog</summary>
-          <p>
-
-          ${{ env.changelog }}
-
-          </p>
-          </details>
-        files: |
-          SHA2-256SUMS
-          SHA2-512SUMS
-          artifact/yt-dlp
-          artifact/yt-dlp.tar.gz
-          artifact/yt-dlp.exe
-          artifact/yt-dlp_win.zip
-          artifact/yt-dlp_min.exe
-          artifact/yt-dlp_x86.exe
-          artifact/yt-dlp_macos
-          artifact/yt-dlp_macos.zip
-          artifact/yt-dlp_macos_legacy
-          artifact/yt-dlp_linux_armv7l
-          artifact/yt-dlp_linux_aarch64
-          artifact/dist/yt-dlp_linux
-          artifact/dist/yt-dlp_linux.zip
-          _update_spec
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          path: |
+            SHA*SUMS*
+            _update_spec
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
new file mode 100644
index 000000000..42e66a29c
--- /dev/null
+++ b/.github/workflows/publish.yml
@@ -0,0 +1,80 @@
+name: Publish
+on:
+  workflow_call:
+    inputs:
+      nightly:
+        default: false
+        required: false
+        type: boolean
+      version:
+        required: true
+        type: string
+      target_commitish:
+        required: true
+        type: string
+    secrets:
+      ARCHIVE_REPO_TOKEN:
+        required: false
+
+permissions:
+  contents: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/download-artifact@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Generate release notes
+        run: |
+          cat >> ./RELEASE_NOTES << EOF
+          #### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files)
+          ---
+          <details><summary><h3>Changelog</h3></summary>
+          $(python ./devscripts/make_changelog.py -vv)
+          </details>
+          EOF
+          echo "**This is an automated nightly pre-release build**" >> ./PRERELEASE_NOTES
+          cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES
+          echo "Generated from: https://github.com/${{ github.repository }}/commit/${{ inputs.target_commitish }}" >> ./ARCHIVE_NOTES
+          cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES
+
+      - name: Archive nightly release
+        env:
+          GH_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
+          GH_REPO: ${{ vars.ARCHIVE_REPO }}
+        if: |
+          inputs.nightly && env.GH_TOKEN != '' && env.GH_REPO != ''
+        run: |
+          gh release create \
+            --notes-file ARCHIVE_NOTES \
+            --title "Build ${{ inputs.version }}" \
+            ${{ inputs.version }} \
+            artifact/*
+
+      - name: Prune old nightly release
+        if: inputs.nightly
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release delete --yes --cleanup-tag "nightly" || true
+          git tag --delete "nightly" || true
+          sleep 5  # Enough time to cover deletion race condition
+
+      - name: Publish release${{ inputs.nightly && ' (nightly)' || '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create \
+            --notes-file ${{ inputs.nightly && 'PRE' || '' }}RELEASE_NOTES \
+            --target ${{ inputs.target_commitish }} \
+            --title "yt-dlp ${{ inputs.nightly && 'nightly ' || '' }}${{ inputs.version }}" \
+            ${{ inputs.nightly && '--prerelease "nightly"' || inputs.version }} \
+            artifact/*
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
new file mode 100644
index 000000000..ec079b8d0
--- /dev/null
+++ b/.github/workflows/release-nightly.yml
@@ -0,0 +1,49 @@
+name: Release (nightly)
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "**.py"
+      - "!yt_dlp/version.py"
+concurrency:
+  group: release-nightly
+  cancel-in-progress: true
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    if: vars.BUILD_NIGHTLY != ''
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get_version.outputs.version }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get version
+        id: get_version
+        run: |
+          python devscripts/update-version.py "$(date -u +"%H%M%S")" | grep -Po "version=\d+(\.\d+){3}" >> "$GITHUB_OUTPUT"
+
+  build:
+    needs: prepare
+    uses: ./.github/workflows/build.yml
+    with:
+      version: ${{ needs.prepare.outputs.version }}
+      channel: nightly
+    permissions:
+      contents: read
+      packages: write # For package cache
+
+  publish:
+    needs: [prepare, build]
+    uses: ./.github/workflows/publish.yml
+    secrets:
+      ARCHIVE_REPO_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
+    permissions:
+      contents: write
+    with:
+      nightly: true
+      version: ${{ needs.prepare.outputs.version }}
+      target_commitish: ${{ github.sha }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 000000000..c97cd1f4a
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,125 @@
+name: Release
+on: workflow_dispatch
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.update_version.outputs.version }}
+      head_sha: ${{ steps.push_release.outputs.head_sha }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Update version
+        id: update_version
+        run: |
+          python devscripts/update-version.py ${{ vars.PUSH_VERSION_COMMIT == '' && '"$(date -u +"%H%M%S")"' || '' }} | \
+            grep -Po "version=\d+\.\d+\.\d+(\.\d+)?" >> "$GITHUB_OUTPUT"
+
+      - name: Update documentation
+        run: |
+          make doc
+          sed '/### /Q' Changelog.md >> ./CHANGELOG
+          echo '### ${{ steps.update_version.outputs.version }}' >> ./CHANGELOG
+          python ./devscripts/make_changelog.py -vv -c >> ./CHANGELOG
+          echo >> ./CHANGELOG
+          grep -Poz '(?s)### \d+\.\d+\.\d+.+' 'Changelog.md' | head -n -1 >> ./CHANGELOG
+          cat ./CHANGELOG > Changelog.md
+
+      - name: Push to release
+        id: push_release
+        run: |
+          git config --global user.name github-actions
+          git config --global user.email github-actions@example.com
+          git add -u
+          git commit -m "Release ${{ steps.update_version.outputs.version }}" \
+            -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
+          git push origin --force ${{ github.event.ref }}:release
+          echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Update master
+        if: vars.PUSH_VERSION_COMMIT != ''
+        run: git push origin ${{ github.event.ref }}
+
+  publish_pypi_homebrew:
+    needs: prepare
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Requirements
+        run: |
+          python -m pip install -U pip setuptools wheel twine
+          python -m pip install -U -r requirements.txt
+
+      - name: Prepare
+        run: |
+          python devscripts/update-version.py ${{ needs.prepare.outputs.version }}
+          python devscripts/make_lazy_extractors.py
+
+      - name: Build and publish on PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        if: env.TWINE_PASSWORD != ''
+        run: |
+          rm -rf dist/*
+          python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
+          python setup.py sdist bdist_wheel
+          twine upload dist/*
+
+      - name: Checkout Homebrew repository
+        env:
+          BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != ''
+        uses: actions/checkout@v3
+        with:
+          repository: yt-dlp/homebrew-taps
+          path: taps
+          ssh-key: ${{ secrets.BREW_TOKEN }}
+
+      - name: Update Homebrew Formulae
+        env:
+          BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != ''
+        run: |
+          python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.version }}"
+          git -C taps/ config user.name github-actions
+          git -C taps/ config user.email github-actions@example.com
+          git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.version }}'
+          git -C taps/ push
+
+  build:
+    needs: prepare
+    uses: ./.github/workflows/build.yml
+    with:
+      version: ${{ needs.prepare.outputs.version }}
+    permissions:
+      contents: read
+      packages: write # For package cache
+
+  publish:
+    needs: [prepare, build]
+    uses: ./.github/workflows/publish.yml
+    permissions:
+      contents: write
+    with:
+      version: ${{ needs.prepare.outputs.version }}
+      target_commitish: ${{ needs.prepare.outputs.head_sha }}
diff --git a/Changelog.md b/Changelog.md
index 24bc8a2e2..60bd99f72 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -1,13 +1,7 @@
 # Changelog
 
 <!--
-# Instuctions for creating release
-
-* Run `make doc`
-* Update Changelog.md and CONTRIBUTORS
-* Change "Based on ytdl" version in Readme.md if needed
-* Commit as `Release <version>` and push to master
-* Dispatch the workflow https://github.com/yt-dlp/yt-dlp/actions/workflows/build.yml on master
+# To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
 ### 2023.02.17
diff --git a/README.md b/README.md
index ddd71eeeb..e6e95b147 100644
--- a/README.md
+++ b/README.md
@@ -318,7 +318,8 @@ ### Related scripts
 
 Note: See their `--help` for more info.
 
-You can also fork the project on GitHub and run your fork's [build workflow](.github/workflows/build.yml) to automatically build a full release
+### Forking the project
+If you fork the project on GitHub, you can run your fork's [build workflow](.github/workflows/build.yml) to automatically build the selected version(s) as artifacts. Alternatively, you can run the [release workflow](.github/workflows/release.yml) or enable the [nightly workflow](.github/workflows/release-nightly.yml) to create full (pre-)releases.
 
 # USAGE AND OPTIONS
 
@@ -460,9 +461,8 @@ ## Video Selection:
     --date DATE                     Download only videos uploaded on this date.
                                     The date can be "YYYYMMDD" or in the format 
                                     [now|today|yesterday][-N[day|week|month|year]].
-                                    E.g. "--date today-2weeks" downloads
-                                    only videos uploaded on the same day two
-                                    weeks ago
+                                    E.g. "--date today-2weeks" downloads only
+                                    videos uploaded on the same day two weeks ago
     --datebefore DATE               Download only videos uploaded on or before
                                     this date. The date formats accepted is the
                                     same as --date
diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py
index fad993a19..2270b31d3 100755
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@@ -45,33 +45,43 @@ def apply_patch(text, patch):
 delim = f'\n{" " * switch_col_width}'
 
 PATCHES = (
-    (   # Standardize update message
+    (   # Standardize `--update` message
         r'(?m)^(    -U, --update\s+).+(\n    \s.+)*$',
         r'\1Update this program to the latest version',
     ),
-    (  # Headings
+    (   # Headings
         r'(?m)^  (\w.+\n)(    (?=\w))?',
         r'## \1'
     ),
-    (  # Do not split URLs
+    (   # Fixup `--date` formatting
+        rf'(?m)(    --date DATE.+({delim}[^\[]+)*)\[.+({delim}.+)*$',
+        (rf'\1[now|today|yesterday][-N[day|week|month|year]].{delim}'
+         f'E.g. "--date today-2weeks" downloads only{delim}'
+         'videos uploaded on the same day two weeks ago'),
+    ),
+    (   # Do not split URLs
         rf'({delim[:-1]})? (?P<label>\[\S+\] )?(?P<url>https?({delim})?:({delim})?/({delim})?/(({delim})?\S+)+)\s',
         lambda mobj: ''.join((delim, mobj.group('label') or '', re.sub(r'\s+', '', mobj.group('url')), '\n'))
     ),
-    (  # Do not split "words"
+    (   # Do not split "words"
         rf'(?m)({delim}\S+)+$',
         lambda mobj: ''.join((delim, mobj.group(0).replace(delim, '')))
     ),
-    (  # Allow overshooting last line
+    (   # Allow overshooting last line
         rf'(?m)^(?P<prev>.+)${delim}(?P<current>.+)$(?!{delim})',
         lambda mobj: (mobj.group().replace(delim, ' ')
                       if len(mobj.group()) - len(delim) + 1 <= max_width + ALLOWED_OVERSHOOT
                       else mobj.group())
     ),
-    (  # Avoid newline when a space is available b/w switch and description
+    (   # Avoid newline when a space is available b/w switch and description
         DISABLE_PATCH,  # This creates issues with prepare_manpage
         r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
         r'\1 '
     ),
+    (   # Replace brackets with a Markdown link
+        r'SponsorBlock API \((http.+)\)',
+        r'[SponsorBlock API](\1)'
+    ),
 )
 
 readme = read_file(README_FILE)
diff --git a/devscripts/update-version.py b/devscripts/update-version.py
index 9cf8b42e6..00c2d54cf 100644
--- a/devscripts/update-version.py
+++ b/devscripts/update-version.py
@@ -7,6 +7,7 @@
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
+import argparse
 import contextlib
 import subprocess
 import sys
@@ -15,8 +16,9 @@
 from devscripts.utils import read_version, write_file
 
 
-def get_new_version(revision):
-    version = datetime.utcnow().strftime('%Y.%m.%d')
+def get_new_version(version, revision):
+    if not version:
+        version = datetime.utcnow().strftime('%Y.%m.%d')
 
     if revision:
         assert revision.isdigit(), 'Revision must be a number'
@@ -30,27 +32,41 @@ def get_new_version(revision):
 
 def get_git_head():
     with contextlib.suppress(Exception):
-        sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE)
-        return sp.communicate()[0].decode().strip() or None
+        return subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() or None
 
 
-VERSION = get_new_version((sys.argv + [''])[1])
-GIT_HEAD = get_git_head()
-
-VERSION_FILE = f'''\
+VERSION_TEMPLATE = '''\
 # Autogenerated by devscripts/update-version.py
 
-__version__ = {VERSION!r}
+__version__ = {version!r}
 
-RELEASE_GIT_HEAD = {GIT_HEAD!r}
+RELEASE_GIT_HEAD = {git_head!r}
 
 VARIANT = None
 
 UPDATE_HINT = None
+
+CHANNEL = "{channel!r}"
 '''
 
-write_file('yt_dlp/version.py', VERSION_FILE)
-github_output = os.getenv('GITHUB_OUTPUT')
-if github_output:
-    write_file(github_output, f'ytdlp_version={VERSION}\n', 'a')
-print(f'\nVersion = {VERSION}, Git HEAD = {GIT_HEAD}')
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Update the version.py file')
+    parser.add_argument(
+        '-c', '--channel', choices=['stable', 'nightly'], default='stable',
+        help='Select update channel (default: %(default)s)')
+    parser.add_argument(
+        '-o', '--output', default='yt_dlp/version.py',
+        help='The output file to write to (default: %(default)s)')
+    parser.add_argument(
+        'version', nargs='?', default=None,
+        help='A version or revision to use instead of generating one')
+    args = parser.parse_args()
+
+    git_head = get_git_head()
+    version = (
+        args.version if args.version and '.' in args.version
+        else get_new_version(None, args.version))
+    write_file(args.output, VERSION_TEMPLATE.format(
+        version=version, git_head=git_head, channel=args.channel))
+
+    print(f'version={version} ({args.channel}), head={git_head}')
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 00846cd7e..1b77e12b8 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -150,7 +150,7 @@
     write_json_file,
     write_string,
 )
-from .version import RELEASE_GIT_HEAD, VARIANT, __version__
+from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
 
 if compat_os_name == 'nt':
     import ctypes
@@ -3768,8 +3768,8 @@ def get_encoding(stream):
         klass = type(self)
         write_debug(join_nonempty(
             f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
-            __version__,
-            f'[{RELEASE_GIT_HEAD}]' if RELEASE_GIT_HEAD else '',
+            __version__ + {'stable': '', 'nightly': '*'}.get(CHANNEL, f' <{CHANNEL}>'),
+            f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '',
             '' if source == 'unknown' else f'({source})',
             '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}',
             delim=' '))

From 77df20f14cc9ed41dfe3a1fe2d77fd27f5365a94 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Fri, 3 Mar 2023 22:36:11 +0530
Subject: [PATCH 110/871] [update] Add option `--update-to`, including to
 nightly (#6220)

* By default, stable will only update to stable, and nightly to nightly

Authored by: Grub4K, bashonly, pukkandan

Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
---
 Collaborators.md   |   2 +
 README.md          |  23 ++++++-
 yt_dlp/__init__.py |   2 +-
 yt_dlp/options.py  |  13 +++-
 yt_dlp/update.py   | 151 +++++++++++++++++++++++++++++++++------------
 yt_dlp/version.py  |   2 +
 6 files changed, 150 insertions(+), 43 deletions(-)

diff --git a/Collaborators.md b/Collaborators.md
index 83dfbe389..71baf5080 100644
--- a/Collaborators.md
+++ b/Collaborators.md
@@ -56,6 +56,7 @@ ## [Lesmiscore](https://github.com/Lesmiscore) <sub><sup>(nao20010128nao)</sup><
 
 ## [bashonly](https://github.com/bashonly)
 
+* `--update-to`, automated release, nightly builds
 * `--cookies-from-browser` support for Firefox containers
 * Added support for new websites Genius, Kick, NBCStations, Triller, VideoKen etc
 * Improved/fixed support for Anvato, Brightcove, Instagram, ParamountPlus, Reddit, SlidesLive, TikTok, Twitter, Vimeo etc
@@ -65,5 +66,6 @@ ## [Grub4K](https://github.com/Grub4K)
 
 [![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/Grub4K) [![gh-sponsor](https://img.shields.io/badge/_-Github-red.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/Grub4K)
 
+* `--update-to`, automated release, nightly builds
 * Rework internals like `traverse_obj`, various core refactors and bugs fixes
 * Helped fix crunchyroll, Twitter, wrestleuniverse, wistia, slideslive etc
diff --git a/README.md b/README.md
index e6e95b147..1acd71fd3 100644
--- a/README.md
+++ b/README.md
@@ -120,7 +120,9 @@ # NEW FEATURES
 
 * **Plugins**: Extractors and PostProcessors can be loaded from an external file. See [plugins](#plugins) for details
 
-* **Self-updater**: The releases can be updated using `yt-dlp -U`
+* **Self updater**: The releases can be updated using `yt-dlp -U`, and downgraded using `--update-to` if required
+
+* **Nightly builds**: [Automated nightly builds](#update-channels) can be used with `--update-to nightly`
 
 See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/commits) for the full list of changes
 
@@ -187,6 +189,20 @@ ## UPDATE
 
 For other third-party package managers, see [the wiki](https://github.com/yt-dlp/yt-dlp/wiki/Installation#third-party-package-managers) or refer their documentation
 
+<a id="update-channels"/>
+
+There are currently two release channels for binaries, `stable` and `nightly`.
+`stable` releases are what the program will update to by default, and have had many of their changes tested by users of the master branch.
+`nightly` releases are built after each push to the master branch, and will have the most recent fixes and additions, but also have the potential for bugs.
+The latest `nightly` is available as a [pre-release from this repository](https://github.com/yt-dlp/yt-dlp/releases/tag/nightly), and all `nightly` releases are [archived in their own repo](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases).
+
+When using `--update`/`-U`, a release binary will only update to its current channel.
+This release channel can be changed by using the `--update-to` option. `--update-to` can also be used to upgrade or downgrade to specific tags from a channel.
+
+Example usage:
+* `yt-dlp --update-to nightly` change to `nightly` channel and update to its latest release
+* `yt-dlp --update-to stable@2023.02.17` upgrade/downgrade to release to `stable` channel tag `2023.02.17`
+* `yt-dlp --update-to 2023.01.06` upgrade/downgrade to tag `2023.01.06` if it exists on the current channel
 
 <!-- MANPAGE: BEGIN EXCLUDED SECTION -->
 ## RELEASE FILES
@@ -335,6 +351,11 @@ ## General Options:
     --version                       Print program version and exit
     -U, --update                    Update this program to the latest version
     --no-update                     Do not check for updates (default)
+    --update-to [CHANNEL]@[TAG]     Upgrade/downgrade to a specific version.
+                                    CHANNEL and TAG defaults to "stable" and
+                                    "latest" respectively if ommited; See
+                                    "UPDATE" for details. Supported channels:
+                                    stable, nightly
     -i, --ignore-errors             Ignore download and postprocessing errors.
                                     The download will be considered successful
                                     even if the postprocessing fails
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index fb44303a2..15c6e997f 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -931,7 +931,7 @@ def _real_main(argv=None):
         if opts.rm_cachedir:
             ydl.cache.remove()
 
-        updater = Updater(ydl)
+        updater = Updater(ydl, opts.update_self if isinstance(opts.update_self, str) else None)
         if opts.update_self and updater.update() and actual_use:
             if updater.cmd:
                 return updater.restart()
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index fd60ff55f..b81e14966 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -20,7 +20,7 @@
     SponsorBlockPP,
 )
 from .postprocessor.modify_chapters import DEFAULT_SPONSORBLOCK_CHAPTER_TITLE
-from .update import detect_variant, is_non_updateable
+from .update import UPDATE_SOURCES, detect_variant, is_non_updateable
 from .utils import (
     OUTTMPL_TYPES,
     POSTPROCESS_WHEN,
@@ -36,7 +36,7 @@
     remove_end,
     write_string,
 )
-from .version import __version__
+from .version import CHANNEL, __version__
 
 
 def parseOpts(overrideArguments=None, ignore_config_files='if_override'):
@@ -326,11 +326,18 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         action='store_true', dest='update_self',
         help=format_field(
             is_non_updateable(), None, 'Check if updates are available. %s',
-            default='Update this program to the latest version'))
+            default=f'Update this program to the latest {CHANNEL} version'))
     general.add_option(
         '--no-update',
         action='store_false', dest='update_self',
         help='Do not check for updates (default)')
+    general.add_option(
+        '--update-to',
+        action='store', dest='update_self', metavar='[CHANNEL]@[TAG]',
+        help=(
+            'Upgrade/downgrade to a specific version. CHANNEL and TAG defaults to '
+            f'"{CHANNEL}" and "latest" respectively if ommited; See "UPDATE" for details. '
+            f'Supported channels: {", ".join(UPDATE_SOURCES)}'))
     general.add_option(
         '-i', '--ignore-errors',
         action='store_true', dest='ignoreerrors',
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index dad273267..297539bb6 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -7,6 +7,7 @@
 import re
 import subprocess
 import sys
+import urllib.error
 from zipimport import zipimporter
 
 from .compat import functools  # isort: split
@@ -16,15 +17,26 @@
     cached_method,
     deprecation_warning,
     remove_end,
+    remove_start,
+    sanitized_Request,
     shell_quote,
     system_identifier,
-    traverse_obj,
     version_tuple,
 )
-from .version import UPDATE_HINT, VARIANT, __version__
+from .version import CHANNEL, UPDATE_HINT, VARIANT, __version__
 
-REPOSITORY = 'yt-dlp/yt-dlp'
-API_URL = f'https://api.github.com/repos/{REPOSITORY}/releases'
+UPDATE_SOURCES = {
+    'stable': 'yt-dlp/yt-dlp',
+    'nightly': 'yt-dlp/yt-dlp-nightly-builds',
+}
+
+_VERSION_RE = re.compile(r'(\d+\.)*\d+')
+
+API_BASE_URL = 'https://api.github.com/repos'
+
+# Backwards compatibility variables for the current channel
+REPOSITORY = UPDATE_SOURCES[CHANNEL]
+API_URL = f'{API_BASE_URL}/{REPOSITORY}/releases'
 
 
 @functools.cache
@@ -110,49 +122,99 @@ def _sha256_file(path):
 
 
 class Updater:
-    def __init__(self, ydl):
+    _exact = True
+
+    def __init__(self, ydl, target=None):
         self.ydl = ydl
 
+        self.target_channel, sep, self.target_tag = (target or CHANNEL).rpartition('@')
+        if not sep and self.target_tag in UPDATE_SOURCES:  # stable => stable@latest
+            self.target_channel, self.target_tag = self.target_tag, None
+        elif not self.target_channel:
+            self.target_channel = CHANNEL
+
+        if not self.target_tag:
+            self.target_tag, self._exact = 'latest', False
+        elif self.target_tag != 'latest':
+            self.target_tag = f'tags/{self.target_tag}'
+
+    @property
+    def _target_repo(self):
+        try:
+            return UPDATE_SOURCES[self.target_channel]
+        except KeyError:
+            return self._report_error(
+                f'Invalid update channel {self.target_channel!r} requested. '
+                f'Valid channels are {", ".join(UPDATE_SOURCES)}', True)
+
+    def _version_compare(self, a, b, channel=CHANNEL):
+        if channel != self.target_channel:
+            return False
+
+        if _VERSION_RE.fullmatch(f'{a}.{b}'):
+            a, b = version_tuple(a), version_tuple(b)
+            return a == b if self._exact else a >= b
+        return a == b
+
     @functools.cached_property
     def _tag(self):
-        if version_tuple(__version__) >= version_tuple(self.latest_version):
-            return 'latest'
+        if self._version_compare(self.current_version, self.latest_version):
+            return self.target_tag
 
-        identifier = f'{detect_variant()} {system_identifier()}'
+        identifier = f'{detect_variant()} {self.target_channel} {system_identifier()}'
         for line in self._download('_update_spec', 'latest').decode().splitlines():
             if not line.startswith('lock '):
                 continue
             _, tag, pattern = line.split(' ', 2)
             if re.match(pattern, identifier):
-                return f'tags/{tag}'
-        return 'latest'
+                if not self._exact:
+                    return f'tags/{tag}'
+                elif self.target_tag == 'latest' or not self._version_compare(
+                        tag, self.target_tag[5:], channel=self.target_channel):
+                    self._report_error(
+                        f'yt-dlp cannot be updated above {tag} since you are on an older Python version', True)
+                    return f'tags/{self.current_version}'
+        return self.target_tag
 
     @cached_method
     def _get_version_info(self, tag):
-        self.ydl.write_debug(f'Fetching release info: {API_URL}/{tag}')
-        return json.loads(self.ydl.urlopen(f'{API_URL}/{tag}').read().decode())
+        url = f'{API_BASE_URL}/{self._target_repo}/releases/{tag}'
+        self.ydl.write_debug(f'Fetching release info: {url}')
+        return json.loads(self.ydl.urlopen(sanitized_Request(url, headers={
+            'Accept': 'application/vnd.github+json',
+            'User-Agent': 'yt-dlp',
+            'X-GitHub-Api-Version': '2022-11-28',
+        })).read().decode())
 
     @property
     def current_version(self):
         """Current version"""
         return __version__
 
+    @staticmethod
+    def _label(channel, tag):
+        """Label for a given channel and tag"""
+        return f'{channel}@{remove_start(tag, "tags/")}'
+
+    def _get_actual_tag(self, tag):
+        if tag.startswith('tags/'):
+            return tag[5:]
+        return self._get_version_info(tag)['tag_name']
+
     @property
     def new_version(self):
         """Version of the latest release we can update to"""
-        if self._tag.startswith('tags/'):
-            return self._tag[5:]
-        return self._get_version_info(self._tag)['tag_name']
+        return self._get_actual_tag(self._tag)
 
     @property
     def latest_version(self):
-        """Version of the latest release"""
-        return self._get_version_info('latest')['tag_name']
+        """Version of the target release"""
+        return self._get_actual_tag(self.target_tag)
 
     @property
     def has_update(self):
         """Whether there is an update available"""
-        return version_tuple(__version__) < version_tuple(self.new_version)
+        return not self._version_compare(self.current_version, self.new_version)
 
     @functools.cached_property
     def filename(self):
@@ -160,10 +222,8 @@ def filename(self):
         return compat_realpath(_get_variant_and_executable_path()[1])
 
     def _download(self, name, tag):
-        url = traverse_obj(self._get_version_info(tag), (
-            'assets', lambda _, v: v['name'] == name, 'browser_download_url'), get_all=False)
-        if not url:
-            raise Exception('Unable to find download URL')
+        slug = 'latest/download' if tag == 'latest' else f'download/{tag[5:]}'
+        url = f'https://github.com/{self._target_repo}/releases/{slug}/{name}'
         self.ydl.write_debug(f'Downloading {name} from {url}')
         return self.ydl.urlopen(url).read()
 
@@ -186,24 +246,32 @@ def _report_permission_error(self, file):
         self._report_error(f'Unable to write to {file}; Try running as administrator', True)
 
     def _report_network_error(self, action, delim=';'):
-        self._report_error(f'Unable to {action}{delim} Visit  https://github.com/{REPOSITORY}/releases/latest', True)
+        self._report_error(
+            f'Unable to {action}{delim} visit  '
+            f'https://github.com/{self._target_repo}/releases/{self.target_tag.replace("tags/", "tag/")}', True)
 
     def check_update(self):
         """Report whether there is an update available"""
+        if not self._target_repo:
+            return False
         try:
-            self.ydl.to_screen(
-                f'Latest version: {self.latest_version}, Current version: {self.current_version}')
-            if not self.has_update:
-                if self._tag == 'latest':
-                    return self.ydl.to_screen(f'yt-dlp is up to date ({__version__})')
-                return self.ydl.report_warning(
-                    'yt-dlp cannot be updated any further since you are on an older Python version')
+            self.ydl.to_screen((
+                f'Available version: {self._label(self.target_channel, self.latest_version)}, ' if self.target_tag == 'latest' else ''
+            ) + f'Current version: {self._label(CHANNEL, self.current_version)}')
         except Exception:
             return self._report_network_error('obtain version info', delim='; Please try again later or')
 
         if not is_non_updateable():
-            self.ydl.to_screen(f'Current Build Hash {_sha256_file(self.filename)}')
-        return True
+            self.ydl.to_screen(f'Current Build Hash: {_sha256_file(self.filename)}')
+
+        if self.has_update:
+            return True
+
+        if self.target_tag == self._tag:
+            self.ydl.to_screen(f'yt-dlp is up to date ({self._label(CHANNEL, self.current_version)})')
+        elif not self._exact:
+            self.ydl.report_warning('yt-dlp cannot be updated any further since you are on an older Python version')
+        return False
 
     def update(self):
         """Update yt-dlp executable to the latest version"""
@@ -212,7 +280,10 @@ def update(self):
         err = is_non_updateable()
         if err:
             return self._report_error(err, True)
-        self.ydl.to_screen(f'Updating to version {self.new_version} ...')
+        self.ydl.to_screen(f'Updating to {self._label(self.target_channel, self.new_version)} ...')
+        if (_VERSION_RE.fullmatch(self.target_tag[5:])
+                and version_tuple(self.target_tag[5:]) < (2023, 3, 2)):
+            self.ydl.report_warning('You are downgrading to a version without --update-to')
 
         directory = os.path.dirname(self.filename)
         if not os.access(self.filename, os.W_OK):
@@ -232,10 +303,11 @@ def update(self):
 
         try:
             newcontent = self._download(self.release_name, self._tag)
-        except OSError:
-            return self._report_network_error('download latest version')
-        except Exception:
-            return self._report_network_error('fetch updates')
+        except Exception as e:
+            if isinstance(e, urllib.error.HTTPError) and e.code == 404:
+                return self._report_error(
+                    f'The requested tag {self._label(self.target_channel, self.target_tag)} does not exist', True)
+            return self._report_network_error(f'fetch updates: {e}')
 
         try:
             expected_hash = self.release_hash
@@ -280,7 +352,7 @@ def update(self):
                 return self._report_error(
                     f'Unable to set permissions. Run: sudo chmod a+rx {compat_shlex_quote(self.filename)}')
 
-        self.ydl.to_screen(f'Updated yt-dlp to version {self.new_version}')
+        self.ydl.to_screen(f'Updated yt-dlp to {self._label(self.target_channel, self.new_version)}')
         return True
 
     @functools.cached_property
@@ -346,3 +418,6 @@ def urlopen(self, url):
             return opener.open(url)
 
     return run_update(FakeYDL())
+
+
+__all__ = ['Updater']
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index 3c92a85e1..f477a927a 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -7,3 +7,5 @@
 VARIANT = None
 
 UPDATE_HINT = None
+
+CHANNEL = 'stable'

From 12647e03d417feaa9ea6a458bea5ebd747494a53 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Fri, 3 Mar 2023 22:36:30 +0530
Subject: [PATCH 111/871] [build] Sign SHA files and release public key

Closes #6344
Authored by: Grub4K
---
 .github/workflows/build.yml           | 13 ++++++++++++
 .github/workflows/release-nightly.yml |  2 ++
 .github/workflows/release.yml         |  2 ++
 README.md                             | 11 +++++++++-
 public.key                            | 29 +++++++++++++++++++++++++++
 5 files changed, 56 insertions(+), 1 deletion(-)
 create mode 100644 public.key

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2183903ea..4c0c6837e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -30,6 +30,9 @@ on:
       meta_files:
         default: true
         type: boolean
+    secrets:
+      GPG_SIGNING_KEY:
+        required: false
 
   workflow_dispatch:
     inputs:
@@ -330,6 +333,16 @@ jobs:
           lock 2022.08.18.36 .+ Python 3.6
           EOF
 
+      - name: Sign checksum files
+        env:
+          GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+        if: env.GPG_SIGNING_KEY != ''
+        run: |
+          gpg --batch --import <<< "${{ secrets.GPG_SIGNING_KEY }}"
+          for signfile in ./SHA*SUMS; do
+            gpg --batch --detach-sign "$signfile"
+          done
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index ec079b8d0..b0634fa5a 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -35,6 +35,8 @@ jobs:
     permissions:
       contents: read
       packages: write # For package cache
+    secrets:
+      GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
 
   publish:
     needs: [prepare, build]
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c97cd1f4a..329d49af8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -114,6 +114,8 @@ jobs:
     permissions:
       contents: read
       packages: write # For package cache
+    secrets:
+      GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
 
   publish:
     needs: [prepare, build]
diff --git a/README.md b/README.md
index 1acd71fd3..a1e915778 100644
--- a/README.md
+++ b/README.md
@@ -235,9 +235,18 @@ #### Misc
 :---|:---
 [yt-dlp.tar.gz](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)|Source tarball
 [SHA2-512SUMS](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-512SUMS)|GNU-style SHA512 sums
+[SHA2-512SUMS.sig](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-512SUMS.sig)|GPG signature file for SHA512 sums
 [SHA2-256SUMS](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-256SUMS)|GNU-style SHA256 sums
-<!-- MANPAGE: END EXCLUDED SECTION -->
+[SHA2-256SUMS.sig](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-256SUMS.sig)|GPG signature file for SHA256 sums
 
+The public key that can be used to verify the GPG signatures is [available here](https://github.com/yt-dlp/yt-dlp/blob/master/public.key)
+Example usage:
+```
+curl -L https://github.com/yt-dlp/yt-dlp/raw/master/public.key | gpg --import
+gpg --verify SHA2-256SUMS.sig SHA2-256SUMS
+gpg --verify SHA2-512SUMS.sig SHA2-512SUMS
+```
+<!-- MANPAGE: END EXCLUDED SECTION -->
 
 **Note**: The manpages, shell completion files etc. are available in the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
 
diff --git a/public.key b/public.key
new file mode 100644
index 000000000..b3af31e9a
--- /dev/null
+++ b/public.key
@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGP78C4BEAD0rF9zjGPAt0thlt5C1ebzccAVX7Nb1v+eqQjk+WEZdTETVCg3
+WAM5ngArlHdm/fZqzUgO+pAYrB60GKeg7ffUDf+S0XFKEZdeRLYeAaqqKhSibVal
+DjvOBOztu3W607HLETQAqA7wTPuIt2WqmpL60NIcyr27LxqmgdN3mNvZ2iLO+bP0
+nKR/C+PgE9H4ytywDa12zMx6PmZCnVOOOu6XZEFmdUxxdQ9fFDqd9LcBKY2LDOcS
+Yo1saY0YWiZWHtzVoZu1kOzjnS5Fjq/yBHJLImDH7pNxHm7s/PnaurpmQFtDFruk
+t+2lhDnpKUmGr/I/3IHqH/X+9nPoS4uiqQ5HpblB8BK+4WfpaiEg75LnvuOPfZIP
+KYyXa/0A7QojMwgOrD88ozT+VCkKkkJ+ijXZ7gHNjmcBaUdKK7fDIEOYI63Lyc6Q
+WkGQTigFffSUXWHDCO9aXNhP3ejqFWgGMtCUsrbkcJkWuWY7q5ARy/05HbSM3K4D
+U9eqtnxmiV1WQ8nXuI9JgJQRvh5PTkny5LtxqzcmqvWO9TjHBbrs14BPEO9fcXxK
+L/CFBbzXDSvvAgArdqqlMoncQ/yicTlfL6qzJ8EKFiqW14QMTdAn6SuuZTodXCTi
+InwoT7WjjuFPKKdvfH1GP4bnqdzTnzLxCSDIEtfyfPsIX+9GI7Jkk/zZjQARAQAB
+tDdTaW1vbiBTYXdpY2tpICh5dC1kbHAgc2lnbmluZyBrZXkpIDxjb250YWN0QGdy
+dWI0ay54eXo+iQJOBBMBCgA4FiEErAy75oSNaoc0ZK9OV89lkztadYEFAmP78C4C
+GwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQV89lkztadYEVqQ//cW7TxhXg
+7Xbh2EZQzXml0egn6j8QaV9KzGragMiShrlvTO2zXfLXqyizrFP4AspgjSn/4NrI
+8mluom+Yi+qr7DXT4BjQqIM9y3AjwZPdywe912Lxcw52NNoPZCm24I9T7ySc8lmR
+FQvZC0w4H/VTNj/2lgJ1dwMflpwvNRiWa5YzcFGlCUeDIPskLx9++AJE+xwU3LYm
+jQQsPBqpHHiTBEJzMLl+rfd9Fg4N+QNzpFkTDW3EPerLuvJniSBBwZthqxeAtw4M
+UiAXh6JvCc2hJkKCoygRfM281MeolvmsGNyQm+axlB0vyldiPP6BnaRgZlx+l6MU
+cPqgHblb7RW5j9lfr6OYL7SceBIHNv0CFrt1OnkGo/tVMwcs8LH3Ae4a7UJlIceL
+V54aRxSsZU7w4iX+PB79BWkEsQzwKrUuJVOeL4UDwWajp75OFaUqbS/slDDVXvK5
+OIeuth3mA/adjdvgjPxhRQjA3l69rRWIJDrqBSHldmRsnX6cvXTDy8wSXZgy51lP
+m4IVLHnCy9m4SaGGoAsfTZS0cC9FgjUIyTyrq9M67wOMpUxnuB0aRZgJE1DsI23E
+qdvcSNVlO+39xM/KPWUEh6b83wMn88QeW+DCVGWACQq5N3YdPnAJa50617fGbY6I
+gXIoRHXkDqe23PZ/jURYCv0sjVtjPoVC+bg=
+=bJkn
+-----END PGP PUBLIC KEY BLOCK-----

From 776d1c3f0c9b00399896dd2e40e78e9a43218109 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 3 Mar 2023 22:00:03 +0530
Subject: [PATCH 112/871] [build] Add `cffi` as a dependency for `yt_dlp_linux`

Closes #6394
Authored by: bashonly
---
 .github/workflows/build.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4c0c6837e..93668a7bf 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -100,8 +100,13 @@ jobs:
           python -m pip install -U pip setuptools wheel
           python -m pip install -U Pyinstaller -r requirements.txt
           reqs=$(mktemp)
-          echo -e 'python=3.10.*\npyinstaller' >$reqs
-          sed 's/^brotli.*/brotli-python/' <requirements.txt >>$reqs
+          cat > $reqs << EOF
+          python=3.10.*
+          pyinstaller
+          cffi
+          brotli-python
+          EOF
+          sed '/^brotli.*/d' requirements.txt >> $reqs
           mamba create -n build --file $reqs
 
       - name: Prepare

From 4815bbfc41cf641e4a0650289dbff968cb3bde76 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 3 Mar 2023 22:40:16 +0530
Subject: [PATCH 113/871] [cleanup] Misc

---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |  4 ++--
 .github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml |  4 ++--
 Changelog.md                                  |  6 +++---
 README.md                                     |  8 ++++----
 devscripts/changelog_override.json            | 13 ++++++++++++-
 yt_dlp/options.py                             |  2 +-
 6 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index 48e8890c5..2eff3c8d8 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -1,5 +1,5 @@
 name: Broken site
-description: Report broken or misfunctioning site
+description: Report error in a supported site
 labels: [triage, site-bug]
 body:
   - type: checkboxes
@@ -16,7 +16,7 @@ body:
       description: |
         Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
       options:
-        - label: I'm reporting a broken site
+        - label: I'm reporting that a **supported** site is broken
           required: true
         - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
index 85900e92e..1f6f92634 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
@@ -1,5 +1,5 @@
 name: Broken site
-description: Report broken or misfunctioning site
+description: Report error in a supported site
 labels: [triage, site-bug]
 body:
   %(no_skip)s
@@ -10,7 +10,7 @@ body:
       description: |
         Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
       options:
-        - label: I'm reporting a broken site
+        - label: I'm reporting that a **supported** site is broken
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
diff --git a/Changelog.md b/Changelog.md
index 60bd99f72..b3cba7d73 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -97,7 +97,7 @@ ### 2023.02.17
 
 ### 2023.01.06
 
-* Fix config locations by [Grub4k](https://github.com/Grub4k), [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+* Fix config locations by [Grub4K](https://github.com/Grub4K), [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
 * [downloader/aria2c] Disable native progress
 * [utils] `mimetype2ext`: `weba` is not standard
 * [utils] `windows_enable_vt_mode`: Better error handling
@@ -124,7 +124,7 @@ ### 2023.01.02
 * Add `--compat-options 2021,2022`
     * This allows devs to change defaults and make other potentially breaking changes more easily. If you need everything to work exactly as-is, put Use `--compat 2022` in your config to guard against future compat changes.
 * [downloader/aria2c] Native progress for aria2c via RPC by [Lesmiscore](https://github.com/Lesmiscore), [pukkandan](https://github.com/pukkandan)
-* Merge youtube-dl: Upto [commit/195f22f](https://github.com/ytdl-org/youtube-dl/commit/195f22f6) by [Grub4k](https://github.com/Grub4k), [pukkandan](https://github.com/pukkandan)
+* Merge youtube-dl: Upto [commit/195f22f](https://github.com/ytdl-org/youtube-dl/commit/195f22f6) by [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan)
 * Add pre-processor stage `video`
 * Let `--parse/replace-in-metadata` run at any post-processing stage
 * Add `--enable-file-urls` by [coletdjnz](https://github.com/coletdjnz)
@@ -239,7 +239,7 @@ ### 2023.01.02
 * [extractor/udemy] Fix lectures that have no URL and detect DRM
 * [extractor/unsupported] Add more URLs
 * [extractor/urplay] Support for audio-only formats by [barsnick](https://github.com/barsnick)
-* [extractor/wistia] Improve extension detection by [Grub4k](https://github.com/Grub4k), [bashonly](https://github.com/bashonly), [pukkandan](https://github.com/pukkandan)
+* [extractor/wistia] Improve extension detection by [Grub4K](https://github.com/Grub4K), [bashonly](https://github.com/bashonly), [pukkandan](https://github.com/pukkandan)
 * [extractor/yle_areena] Support restricted videos by [docbender](https://github.com/docbender)
 * [extractor/youku] Fix extractor by [KurtBestor](https://github.com/KurtBestor)
 * [extractor/youporn] Fix metadata by [marieell](https://github.com/marieell)
diff --git a/README.md b/README.md
index a1e915778..045521b80 100644
--- a/README.md
+++ b/README.md
@@ -183,7 +183,7 @@ # INSTALLATION
 
 
 ## UPDATE
-You can use `yt-dlp -U` to update if you are [using the release binaries](#release-files)
+You can use `yt-dlp -U` to update if you are using the [release binaries](#release-files)
 
 If you [installed with PIP](https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip), simply re-run the same command that was used to install the program
 
@@ -248,7 +248,7 @@ #### Misc
 ```
 <!-- MANPAGE: END EXCLUDED SECTION -->
 
-**Note**: The manpages, shell completion files etc. are available in the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
+**Note**: The manpages, shell completion files etc. are available inside the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
 
 ## DEPENDENCIES
 Python versions 3.7+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly.
@@ -362,7 +362,7 @@ ## General Options:
     --no-update                     Do not check for updates (default)
     --update-to [CHANNEL]@[TAG]     Upgrade/downgrade to a specific version.
                                     CHANNEL and TAG defaults to "stable" and
-                                    "latest" respectively if ommited; See
+                                    "latest" respectively if omitted; See
                                     "UPDATE" for details. Supported channels:
                                     stable, nightly
     -i, --ignore-errors             Ignore download and postprocessing errors.
@@ -1261,7 +1261,7 @@ # OUTPUT TEMPLATE
 
 Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`, `pl_video`. E.g. `-o "%(title)s.%(ext)s" -o "thumbnail:%(title)s\%(title)s.%(ext)s"`  will put the thumbnails in a folder with the same name as the video. If any of the templates is empty, that type of file will not be written. E.g. `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
 
-<a id="outtmpl-postprocess-note"></a>
+<a id="outtmpl-postprocess-note"/>
 
 **Note**: Due to post-processing (i.e. merging etc.), the actual output filename might differ. Use `--print after_move:filepath` to get the name after all post-processing is complete.
 
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index 0967ef424..a5872d4b4 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -1 +1,12 @@
-{}
+[
+    {
+        "action": "add",
+        "when": "2023.02.17",
+        "short": "[priority] **A new release type has been added!**\n    * [`nightly`](https://github.com/yt-dlp/yt-dlp/releases/tag/nightly) builds will be made after each push, containing the latest fixes (but also possibly bugs).\n    * When using `--update`/`-U`, a release binary will only update to its current channel (either `stable` or `nightly`).\n    * The `--update-to` option has been added allowing the user more control over program upgrades (or downgrades).\n    * `--update-to` can change the release channel (`stable`, `nightly`) and also upgrade or downgrade to specific tags.\n    * **Usage**: `--update-to CHANNEL`, `--update-to TAG`, `--update-to CHANNEL@TAG`"
+    },
+    {
+        "action": "add",
+        "when": "2023.02.17",
+        "short": "[priority] **YouTube throttling fixes!**"
+    }
+]
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index b81e14966..cb807740d 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -336,7 +336,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         action='store', dest='update_self', metavar='[CHANNEL]@[TAG]',
         help=(
             'Upgrade/downgrade to a specific version. CHANNEL and TAG defaults to '
-            f'"{CHANNEL}" and "latest" respectively if ommited; See "UPDATE" for details. '
+            f'"{CHANNEL}" and "latest" respectively if omitted; See "UPDATE" for details. '
             f'Supported channels: {", ".join(UPDATE_SOURCES)}'))
     general.add_option(
         '-i', '--ignore-errors',

From b2e0343ba0fc5d8702e90f6ba2b71358e2677e0b Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 3 Mar 2023 23:24:50 +0530
Subject: [PATCH 114/871] [cleanup, jsinterp] Give functions names to help
 debugging

---
 yt_dlp/jsinterp.py | 15 ++++++++++-----
 yt_dlp/utils.py    |  6 ++++--
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index 31ab204d7..db6526009 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -9,6 +9,7 @@
 from .utils import (
     NO_DEFAULT,
     ExtractorError,
+    function_with_repr,
     js_to_json,
     remove_quotes,
     truncate_string,
@@ -184,7 +185,8 @@ def interpret_statement(self, stmt, local_vars, allow_recursion, *args, **kwargs
                     cls.write('=> Raises:', e, '<-|', stmt, level=allow_recursion)
                 raise
             if cls.ENABLED and stmt.strip():
-                cls.write(['->', '=>'][should_ret], repr(ret), '<-|', stmt, level=allow_recursion)
+                if should_ret or not repr(ret) == stmt:
+                    cls.write(['->', '=>'][should_ret], repr(ret), '<-|', stmt, level=allow_recursion)
             return ret, should_ret
         return interpret_statement
 
@@ -205,8 +207,6 @@ class JSInterpreter:
         'y': 4096,  # Perform a "sticky" search that matches starting at the current position in the target string
     }
 
-    _EXC_NAME = '__yt_dlp_exception__'
-
     def __init__(self, code, objects=None):
         self.code, self._functions = code, {}
         self._objects = {} if objects is None else objects
@@ -220,6 +220,8 @@ def __init__(self, msg, expr=None, *args, **kwargs):
     def _named_object(self, namespace, obj):
         self.__named_object_counter += 1
         name = f'__yt_dlp_jsinterp_obj{self.__named_object_counter}'
+        if callable(obj) and not isinstance(obj, function_with_repr):
+            obj = function_with_repr(obj, f'F<{self.__named_object_counter}>')
         namespace[name] = obj
         return name
 
@@ -784,7 +786,8 @@ def extract_object(self, objname):
             fields)
         for f in fields_m:
             argnames = f.group('args').split(',')
-            obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))
+            name = remove_quotes(f.group('key'))
+            obj[name] = function_with_repr(self.build_function(argnames, f.group('code')), f'F<{name}>')
 
         return obj
 
@@ -806,7 +809,9 @@ def extract_function_code(self, funcname):
         return [x.strip() for x in func_m.group('args').split(',')], code
 
     def extract_function(self, funcname):
-        return self.extract_function_from_code(*self.extract_function_code(funcname))
+        return function_with_repr(
+            self.extract_function_from_code(*self.extract_function_code(funcname)),
+            f'F<{funcname}>')
 
     def extract_function_from_code(self, argnames, code, *global_stack):
         local_vars = {}
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 9ff096433..19c140483 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -6057,14 +6057,16 @@ def __get__(self, _, cls):
 
 
 class function_with_repr:
-    def __init__(self, func):
+    def __init__(self, func, repr_=None):
         functools.update_wrapper(self, func)
-        self.func = func
+        self.func, self.__repr = func, repr_
 
     def __call__(self, *args, **kwargs):
         return self.func(*args, **kwargs)
 
     def __repr__(self):
+        if self.__repr:
+            return self.__repr
         return f'{self.func.__module__}.{self.func.__qualname__}'
 
 

From d21056f4cf0a1623daa107f9181074f5725ac436 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 3 Mar 2023 23:59:00 +0530
Subject: [PATCH 115/871] Fix `--break-on-existing` with `--lazy-playlist`

Closes #6399
---
 yt_dlp/utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 19c140483..bf2eb6b38 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3042,8 +3042,10 @@ def get_requested_items(self):
                 if not entry:
                     continue
                 try:
-                    # TODO: Add auto-generated fields
-                    self.ydl._match_entry(entry, incomplete=True, silent=True)
+                    # The item may have just been added to archive. Don't break due to it
+                    if not self.ydl.params.get('lazy_playlist'):
+                        # TODO: Add auto-generated fields
+                        self.ydl._match_entry(entry, incomplete=True, silent=True)
                 except (ExistingVideoReached, RejectedVideoReached):
                     return
 

From fe2ce85aff0aa03735fc0152bb8cb9c3d4ef0753 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 4 Mar 2023 01:13:05 +0530
Subject: [PATCH 116/871] Add option `--break-match-filters`

* Deprecates `--break-on-reject`

Closes #5962
---
 README.md           | 10 +++++----
 yt_dlp/YoutubeDL.py | 50 ++++++++++++++++++++++++++++++---------------
 yt_dlp/__init__.py  |  2 +-
 yt_dlp/options.py   | 16 +++++++++++----
 yt_dlp/utils.py     | 15 +++++++++-----
 5 files changed, 62 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index 045521b80..b49059484 100644
--- a/README.md
+++ b/README.md
@@ -114,7 +114,7 @@ # NEW FEATURES
 
 * **Output template improvements**: Output templates can now have date-time formatting, numeric offsets, object traversal etc. See [output template](#output-template) for details. Even more advanced operations can also be done with the help of `--parse-metadata` and `--replace-in-metadata`
 
-* **Other new options**: Many new options have been added such as `--alias`, `--print`, `--concat-playlist`, `--wait-for-video`, `--retry-sleep`, `--sleep-requests`, `--convert-thumbnails`, `--force-download-archive`, `--force-overwrites`, `--break-on-reject` etc
+* **Other new options**: Many new options have been added such as `--alias`, `--print`, `--concat-playlist`, `--wait-for-video`, `--retry-sleep`, `--sleep-requests`, `--convert-thumbnails`, `--force-download-archive`, `--force-overwrites`, `--break-match-filter` etc
 
 * **Improvements**: Regex and other operators in `--format`/`--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection), merge multi-video/audio, multiple `--config-locations`, `--exec` at different stages, etc
 
@@ -519,7 +519,10 @@ ## Video Selection:
                                     dogs" (caseless). Use "--match-filter -" to
                                     interactively ask whether to download each
                                     video
-    --no-match-filter               Do not use generic video filter (default)
+    --no-match-filter               Do not use any --match-filter (default)
+    --break-match-filters FILTER    Same as "--match-filters" but stops the
+                                    download process when a video is rejected
+    --no-break-match-filters        Do not use any --break-match-filters (default)
     --no-playlist                   Download only the video, if the URL refers
                                     to a video and a playlist
     --yes-playlist                  Download the playlist, if the URL refers to
@@ -533,8 +536,6 @@ ## Video Selection:
     --max-downloads NUMBER          Abort after downloading NUMBER files
     --break-on-existing             Stop the download process when encountering
                                     a file that is in the archive
-    --break-on-reject               Stop the download process when encountering
-                                    a file that has been filtered out
     --break-per-input               Alters --max-downloads, --break-on-existing,
                                     --break-on-reject, and autonumber to reset
                                     per input URL
@@ -2133,6 +2134,7 @@ #### Redundant options
     --reject-title REGEX             --match-filter "title !~= (?i)REGEX"
     --min-views COUNT                --match-filter "view_count >=? COUNT"
     --max-views COUNT                --match-filter "view_count <=? COUNT"
+    --break-on-reject                Use --break-match-filter
     --user-agent UA                  --add-header "User-Agent:UA"
     --referer URL                    --add-header "Referer:URL"
     --playlist-start NUMBER          -I NUMBER:
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 1b77e12b8..5d21b43cf 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -300,8 +300,6 @@ class YoutubeDL:
                        Videos already present in the file are not downloaded again.
     break_on_existing: Stop the download process after attempting to download a
                        file that is in the archive.
-    break_on_reject:   Stop the download process when encountering a video that
-                       has been filtered out.
     break_per_url:     Whether break_on_reject and break_on_existing
                        should act on each input URL as opposed to for the entire queue
     cookiefile:        File name or text stream from where cookies should be read and dumped to
@@ -414,6 +412,8 @@ class YoutubeDL:
                        - If it returns None, the video is downloaded.
                        - If it returns utils.NO_DEFAULT, the user is interactively
                          asked whether to download the video.
+                       - Raise utils.DownloadCancelled(msg) to abort remaining
+                         downloads when a video is rejected.
                        match_filter_func in utils.py is one example for this.
     no_color:          Do not emit color codes in output.
     geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
@@ -483,6 +483,9 @@ class YoutubeDL:
 
     The following options are deprecated and may be removed in the future:
 
+    break_on_reject:   Stop the download process when encountering a video that
+                       has been filtered out.
+                       - `raise DownloadCancelled(msg)` in match_filter instead
     force_generic_extractor: Force downloader to use the generic extractor
                        - Use allowed_extractors = ['generic', 'default']
     playliststart:     - Use playlist_items
@@ -1407,31 +1410,44 @@ def check_filter():
                 return 'Skipping "%s" because it is age restricted' % video_title
 
             match_filter = self.params.get('match_filter')
-            if match_filter is not None:
+            if match_filter is None:
+                return None
+
+            cancelled = None
+            try:
                 try:
                     ret = match_filter(info_dict, incomplete=incomplete)
                 except TypeError:
                     # For backward compatibility
                     ret = None if incomplete else match_filter(info_dict)
-                if ret is NO_DEFAULT:
-                    while True:
-                        filename = self._format_screen(self.prepare_filename(info_dict), self.Styles.FILENAME)
-                        reply = input(self._format_screen(
-                            f'Download "{filename}"? (Y/n): ', self.Styles.EMPHASIS)).lower().strip()
-                        if reply in {'y', ''}:
-                            return None
-                        elif reply == 'n':
-                            return f'Skipping {video_title}'
-                elif ret is not None:
-                    return ret
-            return None
+            except DownloadCancelled as err:
+                if err.msg is not NO_DEFAULT:
+                    raise
+                ret, cancelled = err.msg, err
+
+            if ret is NO_DEFAULT:
+                while True:
+                    filename = self._format_screen(self.prepare_filename(info_dict), self.Styles.FILENAME)
+                    reply = input(self._format_screen(
+                        f'Download "{filename}"? (Y/n): ', self.Styles.EMPHASIS)).lower().strip()
+                    if reply in {'y', ''}:
+                        return None
+                    elif reply == 'n':
+                        if cancelled:
+                            raise type(cancelled)(f'Skipping {video_title}')
+                        return f'Skipping {video_title}'
+            return ret
 
         if self.in_download_archive(info_dict):
             reason = '%s has already been recorded in the archive' % video_title
             break_opt, break_err = 'break_on_existing', ExistingVideoReached
         else:
-            reason = check_filter()
-            break_opt, break_err = 'break_on_reject', RejectedVideoReached
+            try:
+                reason = check_filter()
+            except DownloadCancelled as e:
+                reason, break_opt, break_err = e.msg, 'match_filter', type(e)
+            else:
+                break_opt, break_err = 'break_on_reject', RejectedVideoReached
         if reason is not None:
             if not silent:
                 self.to_screen('[download] ' + reason)
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 15c6e997f..9ef31601c 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -403,7 +403,7 @@ def metadataparser_actions(f):
         except Exception:
             raise ValueError('unsupported geo-bypass country or ip-block')
 
-    opts.match_filter = match_filter_func(opts.match_filter)
+    opts.match_filter = match_filter_func(opts.match_filter, opts.breaking_match_filter)
 
     if opts.download_archive is not None:
         opts.download_archive = expand_path(opts.download_archive)
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index cb807740d..84aeda7f1 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -613,8 +613,16 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
             'Use "--match-filter -" to interactively ask whether to download each video'))
     selection.add_option(
         '--no-match-filter',
-        metavar='FILTER', dest='match_filter', action='store_const', const=None,
-        help='Do not use generic video filter (default)')
+        dest='match_filter', action='store_const', const=None,
+        help='Do not use any --match-filter (default)')
+    selection.add_option(
+        '--break-match-filters',
+        metavar='FILTER', dest='breaking_match_filter', action='append',
+        help='Same as "--match-filters" but stops the download process when a video is rejected')
+    selection.add_option(
+        '--no-break-match-filters',
+        dest='breaking_match_filter', action='store_const', const=None,
+        help='Do not use any --break-match-filters (default)')
     selection.add_option(
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
@@ -646,11 +654,11 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
     selection.add_option(
         '--break-on-reject',
         action='store_true', dest='break_on_reject', default=False,
-        help='Stop the download process when encountering a file that has been filtered out')
+        help=optparse.SUPPRESS_HELP)
     selection.add_option(
         '--break-per-input',
         action='store_true', dest='break_per_url', default=False,
-        help='Alters --max-downloads, --break-on-existing, --break-on-reject, and autonumber to reset per input URL')
+        help='Alters --max-downloads, --break-on-existing, --break-match-filter, and autonumber to reset per input URL')
     selection.add_option(
         '--no-break-per-input',
         action='store_false', dest='break_per_url',
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index bf2eb6b38..e9b889447 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -1230,8 +1230,8 @@ class ExistingVideoReached(DownloadCancelled):
 
 
 class RejectedVideoReached(DownloadCancelled):
-    """ --break-on-reject triggered """
-    msg = 'Encountered a video that did not match filter, stopping due to --break-on-reject'
+    """ --break-match-filter triggered """
+    msg = 'Encountered a video that did not match filter, stopping due to --break-match-filter'
 
 
 class MaxDownloadsReached(DownloadCancelled):
@@ -3911,16 +3911,21 @@ def match_str(filter_str, dct, incomplete=False):
         for filter_part in re.split(r'(?<!\\)&', filter_str))
 
 
-def match_filter_func(filters):
-    if not filters:
+def match_filter_func(filters, breaking_filters=None):
+    if not filters and not breaking_filters:
         return None
-    filters = set(variadic(filters))
+    breaking_filters = match_filter_func(breaking_filters) or (lambda _, __: None)
+    filters = set(variadic(filters or []))
 
     interactive = '-' in filters
     if interactive:
         filters.remove('-')
 
     def _match_func(info_dict, incomplete=False):
+        ret = breaking_filters(info_dict, incomplete)
+        if ret is not None:
+            raise RejectedVideoReached(ret)
+
         if not filters or any(match_str(f, info_dict, incomplete) for f in filters):
             return NO_DEFAULT if interactive and not incomplete else None
         else:

From bfc861a91ee65c9b0ac169754f512e052c6827cf Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 4 Mar 2023 01:24:22 +0530
Subject: [PATCH 117/871] Fix bug in 29cb20bd563c02671b31dd840139e93dd37150a1

---
 devscripts/update-version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devscripts/update-version.py b/devscripts/update-version.py
index 00c2d54cf..60ebcff62 100644
--- a/devscripts/update-version.py
+++ b/devscripts/update-version.py
@@ -46,7 +46,7 @@ def get_git_head():
 
 UPDATE_HINT = None
 
-CHANNEL = "{channel!r}"
+CHANNEL = {channel!r}
 '''
 
 if __name__ == '__main__':

From 93449642815a6973a4b09b289982ca7e1f961b5f Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Fri, 3 Mar 2023 22:39:09 +0100
Subject: [PATCH 118/871] Fix d400e261cf029a3f20d364113b14de973be75404

Authored by: Grub4K
---
 devscripts/make_changelog.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index b66181b53..07aa3285b 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -127,7 +127,9 @@ def format_module(self, name, group):
     def _format_group(self, group):
         sorted_group = sorted(group, key=CommitInfo.key)
         detail_groups = itertools.groupby(sorted_group, lambda item: (item.details or '').lower())
-        for details, items in detail_groups:
+        for _, items in detail_groups:
+            items = list(items)
+            details = items[0].details
             if not details:
                 indent = ''
             else:
@@ -137,15 +139,15 @@ def _format_group(self, group):
             if details == 'cleanup':
                 items, cleanup_misc_items = self._filter_cleanup_misc_items(items)
 
-            sub_detail_groups = itertools.groupby(items, lambda item: item.sub_details)
+            sub_detail_groups = itertools.groupby(items, lambda item: tuple(map(str.lower, item.sub_details)))
             for sub_details, entries in sub_detail_groups:
                 if not sub_details:
                     for entry in entries:
                         yield f'{indent}- {self.format_single_change(entry)}'
                     continue
 
-                prefix = f'{indent}- {", ".join(sub_details)}'
                 entries = list(entries)
+                prefix = f'{indent}- {", ".join(entries[0].sub_details)}'
                 if len(entries) == 1:
                     yield f'{prefix}: {self.format_single_change(entries[0])}'
                     continue
@@ -232,12 +234,12 @@ class CommitRange:
             (?:/(?P<details>[^\]:,]+))?
             (?:[:,](?P<sub_details>[^\]]+))?
         \]\ )?
-        (?:`?(?P<sub_details_alt>[^:`]+)`?: )?
+        (?:(?P<sub_details_alt>`?[^:`]+`?): )?
         (?P<message>.+?)
         (?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
         ''', re.VERBOSE | re.DOTALL)
     EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE)
-    FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+for)?|Revert)\s+([\da-f]{40})')
+    FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})')
     UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)')
 
     def __init__(self, start, end, default_author=None) -> None:
@@ -391,7 +393,7 @@ def groups(self):
             else:
                 group = CommitGroup.CORE
 
-            sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.lower().replace(':', ',')
+            sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.replace(':', ',')
             sub_details = tuple(filter(None, map(str.strip, sub_details.split(','))))
 
             issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else []

From 354d5fca7a533f60a6eee14e854b9b4c132a7314 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@example.com>
Date: Fri, 3 Mar 2023 21:41:45 +0000
Subject: [PATCH 119/871] Release 2023.03.03

Created by: Grub4K

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |  8 +--
 .../ISSUE_TEMPLATE/2_site_support_request.yml |  8 +--
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |  8 +--
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |  8 +--
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |  8 +--
 .github/ISSUE_TEMPLATE/6_question.yml         |  8 +--
 CONTRIBUTORS                                  |  1 +
 Changelog.md                                  | 60 +++++++++++++++++++
 README.md                                     |  4 +-
 yt_dlp/version.py                             |  4 +-
 10 files changed, 89 insertions(+), 28 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index 2eff3c8d8..c273c952e 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that a **supported** site is broken
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -64,7 +64,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -72,8 +72,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.02.17, Current version: 2023.02.17
-        yt-dlp is up to date (2023.02.17)
+        Latest version: 2023.03.03, Current version: 2023.03.03
+        yt-dlp is up to date (2023.03.03)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index d43d62f03..bfb362e56 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -76,7 +76,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -84,8 +84,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.02.17, Current version: 2023.02.17
-        yt-dlp is up to date (2023.02.17)
+        Latest version: 2023.03.03, Current version: 2023.03.03
+        yt-dlp is up to date (2023.03.03)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index 352b47242..164f7fa9f 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -72,7 +72,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -80,8 +80,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.02.17, Current version: 2023.02.17
-        yt-dlp is up to date (2023.02.17)
+        Latest version: 2023.03.03, Current version: 2023.03.03
+        yt-dlp is up to date (2023.03.03)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index 7588b8ed8..3cfcb1c03 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -57,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -65,8 +65,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.02.17, Current version: 2023.02.17
-        yt-dlp is up to date (2023.02.17)
+        Latest version: 2023.03.03, Current version: 2023.03.03
+        yt-dlp is up to date (2023.03.03)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index fdda50b7b..35204ca57 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -53,7 +53,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -61,7 +61,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.02.17, Current version: 2023.02.17
-        yt-dlp is up to date (2023.02.17)
+        Latest version: 2023.03.03, Current version: 2023.03.03
+        yt-dlp is up to date (2023.03.03)
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index 56ce74654..dc2ccf1d4 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.02.17** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -59,7 +59,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.02.17 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -67,7 +67,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.02.17, Current version: 2023.02.17
-        yt-dlp is up to date (2023.02.17)
+        Latest version: 2023.03.03, Current version: 2023.03.03
+        yt-dlp is up to date (2023.03.03)
         <more lines>
       render: shell
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 10fb5775b..bd5c6ca97 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -405,3 +405,4 @@ road-master
 rohieb
 sdht0
 seproDev
+Hill-98
diff --git a/Changelog.md b/Changelog.md
index b3cba7d73..580ff958b 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,66 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.03.03
+
+#### Important changes
+- **A new release type has been added!**
+    * [`nightly`](https://github.com/yt-dlp/yt-dlp/releases/tag/nightly) builds will be made after each push, containing the latest fixes (but also possibly bugs).
+    * When using `--update`/`-U`, a release binary will only update to its current channel (either `stable` or `nightly`).
+    * The `--update-to` option has been added allowing the user more control over program upgrades (or downgrades).
+    * `--update-to` can change the release channel (`stable`, `nightly`) and also upgrade or downgrade to specific tags.
+    * **Usage**: `--update-to CHANNEL`, `--update-to TAG`, `--update-to CHANNEL@TAG`
+- **YouTube throttling fixes!**
+
+#### Core changes
+- [Add option `--break-match-filters`](https://github.com/yt-dlp/yt-dlp/commit/fe2ce85aff0aa03735fc0152bb8cb9c3d4ef0753) by [pukkandan](https://github.com/pukkandan)
+- [Fix `--break-on-existing` with `--lazy-playlist`](https://github.com/yt-dlp/yt-dlp/commit/d21056f4cf0a1623daa107f9181074f5725ac436) by [pukkandan](https://github.com/pukkandan)
+- dependencies
+    - [Simplify `Cryptodome`](https://github.com/yt-dlp/yt-dlp/commit/65f6e807804d2af5e00f2aecd72bfc43af19324a) by [pukkandan](https://github.com/pukkandan)
+- jsinterp
+    - [Handle `Date` at epoch 0](https://github.com/yt-dlp/yt-dlp/commit/9acf1ee25f7ad3920ede574a9de95b8c18626af4) by [pukkandan](https://github.com/pukkandan)
+- plugins
+    - [Don't look in `.egg` directories](https://github.com/yt-dlp/yt-dlp/commit/b059188383eee4fa336ef728dda3ff4bb7335625) by [pukkandan](https://github.com/pukkandan)
+- update
+    - [Add option `--update-to`, including to nightly](https://github.com/yt-dlp/yt-dlp/commit/77df20f14cc9ed41dfe3a1fe2d77fd27f5365a94) ([#6220](https://github.com/yt-dlp/yt-dlp/issues/6220)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan)
+- utils
+    - `LenientJSONDecoder`: [Parse unclosed objects](https://github.com/yt-dlp/yt-dlp/commit/cc09083636ce21e58ff74f45eac2dbda507462b0) by [pukkandan](https://github.com/pukkandan)
+    - `Popen`: [Shim undocumented `text_mode` property](https://github.com/yt-dlp/yt-dlp/commit/da8e2912b165005f76779a115a071cd6132ceedf) by [Grub4K](https://github.com/Grub4K)
+
+#### Extractor changes
+- [Fix DRM detection in m3u8](https://github.com/yt-dlp/yt-dlp/commit/43a3eaf96393b712d60cbcf5c6cb1e90ed7f42f5) by [pukkandan](https://github.com/pukkandan)
+- generic
+    - [Detect manifest links via extension](https://github.com/yt-dlp/yt-dlp/commit/b38cae49e6f4849c8ee2a774bdc3c1c647ae5f0e) by [bashonly](https://github.com/bashonly)
+    - [Handle basic-auth when checking redirects](https://github.com/yt-dlp/yt-dlp/commit/8e9fe43cd393e69fa49b3d842aa3180c1d105b8f) by [pukkandan](https://github.com/pukkandan)
+- GoogleDrive
+    - [Fix some audio](https://github.com/yt-dlp/yt-dlp/commit/4d248e29d20d983ededab0b03d4fe69dff9eb4ed) by [pukkandan](https://github.com/pukkandan)
+- iprima
+    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/9fddc12ab022a31754e0eaa358fc4e1dfa974587) ([#6291](https://github.com/yt-dlp/yt-dlp/issues/6291)) by [std-move](https://github.com/std-move)
+- mediastream
+    - [Improve WinSports support](https://github.com/yt-dlp/yt-dlp/commit/2d5a8c5db2bd4ff1c2e45e00cd890a10f8ffca9e) ([#6401](https://github.com/yt-dlp/yt-dlp/issues/6401)) by [bashonly](https://github.com/bashonly)
+- ntvru
+    - [Extract HLS and DASH formats](https://github.com/yt-dlp/yt-dlp/commit/77d6d136468d0c23c8e79bc937898747804f585a) ([#6403](https://github.com/yt-dlp/yt-dlp/issues/6403)) by [bashonly](https://github.com/bashonly)
+- tencent
+    - [Add more formats and info](https://github.com/yt-dlp/yt-dlp/commit/18d295c9e0f95adc179eef345b7af64d6372db78) ([#5950](https://github.com/yt-dlp/yt-dlp/issues/5950)) by [Hill-98](https://github.com/Hill-98)
+- yle_areena
+    - [Extract non-Kaltura videos](https://github.com/yt-dlp/yt-dlp/commit/40d77d89027cd0e0ce31d22aec81db3e1d433900) ([#6402](https://github.com/yt-dlp/yt-dlp/issues/6402)) by [bashonly](https://github.com/bashonly)
+- youtube
+    - [Construct dash formats with `range` query](https://github.com/yt-dlp/yt-dlp/commit/5038f6d713303e0967d002216e7a88652401c22a) by [pukkandan](https://github.com/pukkandan) (With fixes in [f34804b](https://github.com/yt-dlp/yt-dlp/commit/f34804b2f920f62a6e893a14a9e2a2144b14dd23) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz))
+    - [Detect and break on looping comments](https://github.com/yt-dlp/yt-dlp/commit/7f51861b1820c37b157a239b1fe30628d907c034) ([#6301](https://github.com/yt-dlp/yt-dlp/issues/6301)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Extract channel `view_count` when `/about` tab is passed](https://github.com/yt-dlp/yt-dlp/commit/31e183557fcd1b937582f9429f29207c1261f501) by [pukkandan](https://github.com/pukkandan)
+
+#### Misc. changes
+- build
+    - [Add `cffi` as a dependency for `yt_dlp_linux`](https://github.com/yt-dlp/yt-dlp/commit/776d1c3f0c9b00399896dd2e40e78e9a43218109) by [bashonly](https://github.com/bashonly)
+    - [Automated builds and nightly releases](https://github.com/yt-dlp/yt-dlp/commit/29cb20bd563c02671b31dd840139e93dd37150a1) ([#6220](https://github.com/yt-dlp/yt-dlp/issues/6220)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K) (With fixes in [bfc861a](https://github.com/yt-dlp/yt-dlp/commit/bfc861a91ee65c9b0ac169754f512e052c6827cf) by [pukkandan](https://github.com/pukkandan))
+    - [Sign SHA files and release public key](https://github.com/yt-dlp/yt-dlp/commit/12647e03d417feaa9ea6a458bea5ebd747494a53) by [Grub4K](https://github.com/Grub4K)
+- cleanup
+    - [Fix `Changelog`](https://github.com/yt-dlp/yt-dlp/commit/17ca19ab60a6a13eb8a629c51442b5248b0d8394) by [pukkandan](https://github.com/pukkandan)
+    - jsinterp: [Give functions names to help debugging](https://github.com/yt-dlp/yt-dlp/commit/b2e0343ba0fc5d8702e90f6ba2b71358e2677e0b) by [pukkandan](https://github.com/pukkandan)
+    - Miscellaneous: [4815bbf](https://github.com/yt-dlp/yt-dlp/commit/4815bbfc41cf641e4a0650289dbff968cb3bde76), [5b28cef](https://github.com/yt-dlp/yt-dlp/commit/5b28cef72db3b531680d89c121631c73ae05354f) by [pukkandan](https://github.com/pukkandan)
+- devscripts
+    - [Script to generate changelog](https://github.com/yt-dlp/yt-dlp/commit/d400e261cf029a3f20d364113b14de973be75404) ([#6220](https://github.com/yt-dlp/yt-dlp/issues/6220)) by [Grub4K](https://github.com/Grub4K) (With fixes in [9344964](https://github.com/yt-dlp/yt-dlp/commit/93449642815a6973a4b09b289982ca7e1f961b5f))
+
 ### 2023.02.17
 
 * Merge youtube-dl: Upto [commit/2dd6c6e](https://github.com/ytdl-org/youtube-dl/commit/2dd6c6e)
diff --git a/README.md b/README.md
index b49059484..5788e9c0e 100644
--- a/README.md
+++ b/README.md
@@ -537,8 +537,8 @@ ## Video Selection:
     --break-on-existing             Stop the download process when encountering
                                     a file that is in the archive
     --break-per-input               Alters --max-downloads, --break-on-existing,
-                                    --break-on-reject, and autonumber to reset
-                                    per input URL
+                                    --break-match-filter, and autonumber to
+                                    reset per input URL
     --no-break-per-input            --break-on-existing and similar options
                                     terminates the entire download queue
     --skip-playlist-after-errors N  Number of allowed failures until the rest of
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index f477a927a..7b27fcabb 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.02.17'
+__version__ = '2023.03.03'
 
-RELEASE_GIT_HEAD = 'a0a7c0154'
+RELEASE_GIT_HEAD = '93449642815a6973a4b09b289982ca7e1f961b5f'
 
 VARIANT = None
 

From 55676fe498345a389a2539d8baaba958d6d61c3e Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Fri, 3 Mar 2023 21:54:20 -0600
Subject: [PATCH 120/871] [build] Fix publishing to PyPI and homebrew

Closes #6411
Authored by: bashonly
---
 .github/workflows/release.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 329d49af8..e07fc0c07 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -64,6 +64,7 @@ jobs:
 
       - name: Install Requirements
         run: |
+          sudo apt-get -y install pandoc man
           python -m pip install -U pip setuptools wheel twine
           python -m pip install -U -r requirements.txt
 
@@ -79,6 +80,7 @@ jobs:
         if: env.TWINE_PASSWORD != ''
         run: |
           rm -rf dist/*
+          make pypi-files
           python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
           python setup.py sdist bdist_wheel
           twine upload dist/*

From 640c934823fc2d1ec77ec932566078014058635f Mon Sep 17 00:00:00 2001
From: Venkata Krishna S <100816677+venkata-krishnas@users.noreply.github.com>
Date: Sat, 4 Mar 2023 01:57:30 -0500
Subject: [PATCH 121/871] [extractor/ESPNcricinfo] Handle new URL pattern
 (#6321)

Authored by: venkata-krishnas
Closes #6164
---
 yt_dlp/extractor/espn.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/espn.py b/yt_dlp/extractor/espn.py
index f4b0134ab..7ed824ca8 100644
--- a/yt_dlp/extractor/espn.py
+++ b/yt_dlp/extractor/espn.py
@@ -240,7 +240,7 @@ def _real_extract(self, url):
 
 
 class ESPNCricInfoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?espncricinfo\.com/video/[^#$&?/]+-(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?espncricinfo\.com/(?:cricket-)?videos?/[^#$&?/]+-(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.espncricinfo.com/video/finch-chasing-comes-with-risks-despite-world-cup-trend-1289135',
         'info_dict': {
@@ -252,6 +252,17 @@ class ESPNCricInfoIE(InfoExtractor):
             'duration': 96,
         },
         'params': {'skip_download': True}
+    }, {
+        'url': 'https://www.espncricinfo.com/cricket-videos/daryl-mitchell-mitchell-santner-is-one-of-the-best-white-ball-spinners-india-vs-new-zealand-1356225',
+        'info_dict': {
+            'id': '1356225',
+            'ext': 'mp4',
+            'description': '"Santner has done it for a long time for New Zealand - we\'re lucky to have him"',
+            'upload_date': '20230128',
+            'title': 'Mitchell: \'Santner is one of the best white-ball spinners at the moment\'',
+            'duration': 87,
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):

From 4a6272c6d1bff89969b67cd22b26ebe6d7e72279 Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Sat, 4 Mar 2023 01:01:30 -0600
Subject: [PATCH 122/871] [extractor/twitch] Update for GraphQL API changes
 (#6318)

Authored by: elyse0
Closes #6308
---
 yt_dlp/extractor/twitch.py | 46 ++++++++++++++++++++++++++++++--------
 1 file changed, 37 insertions(+), 9 deletions(-)

diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index c59d1cf17..efc7db2c9 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -48,12 +48,12 @@ class TwitchBaseIE(InfoExtractor):
         'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
         'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
         'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
-        'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
-        'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
+        'ChannelCollectionsContent': '447aec6a0cc1e8d0a8d7732d47eb0762c336a2294fdb009e9c9d854e49d484b9',
+        'StreamMetadata': 'a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962',
         'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
         'VideoAccessToken_Clip': '36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11',
         'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
-        'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
+        'VideoMetadata': '49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad',
         'VideoPlayer_ChapterSelectButtonVideo': '8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41',
         'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
     }
@@ -380,13 +380,14 @@ def _download_info(self, item_id):
             }],
             'Downloading stream metadata GraphQL')
 
-        video = traverse_obj(data, (0, 'data', 'video'))
-        video['moments'] = traverse_obj(data, (1, 'data', 'video', 'moments', 'edges', ..., 'node'))
-        video['storyboard'] = traverse_obj(data, (2, 'data', 'video', 'seekPreviewsURL'), expected_type=url_or_none)
-
+        video = traverse_obj(data, (..., 'data', 'video'), get_all=False)
         if video is None:
-            raise ExtractorError(
-                'Video %s does not exist' % item_id, expected=True)
+            raise ExtractorError(f'Video {item_id} does not exist', expected=True)
+
+        video['moments'] = traverse_obj(data, (..., 'data', 'video', 'moments', 'edges', ..., 'node'))
+        video['storyboard'] = traverse_obj(
+            data, (..., 'data', 'video', 'seekPreviewsURL', {url_or_none}), get_all=False)
+
         return video
 
     def _extract_info(self, info):
@@ -854,6 +855,13 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
             'title': 'spamfish - Collections',
         },
         'playlist_mincount': 3,
+    }, {
+        'url': 'https://www.twitch.tv/monstercat/videos?filter=collections',
+        'info_dict': {
+            'id': 'monstercat',
+            'title': 'monstercat - Collections',
+        },
+        'playlist_mincount': 13,
     }]
 
     _OPERATION_NAME = 'ChannelCollectionsContent'
@@ -922,6 +930,7 @@ class TwitchStreamIE(TwitchBaseIE):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'User does not exist',
     }, {
         'url': 'http://www.twitch.tv/miracle_doto#profile-0',
         'only_matching': True,
@@ -934,6 +943,25 @@ class TwitchStreamIE(TwitchBaseIE):
     }, {
         'url': 'https://m.twitch.tv/food',
         'only_matching': True,
+    }, {
+        'url': 'https://www.twitch.tv/monstercat',
+        'info_dict': {
+            'id': '40500071752',
+            'display_id': 'monstercat',
+            'title': 're:Monstercat',
+            'description': 'md5:0945ad625e615bc8f0469396537d87d9',
+            'is_live': True,
+            'timestamp': 1677107190,
+            'upload_date': '20230222',
+            'uploader': 'Monstercat',
+            'uploader_id': 'monstercat',
+            'live_status': 'is_live',
+            'thumbnail': 're:https://.*.jpg',
+            'ext': 'mp4',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
     }]
 
     @classmethod

From 08ff6d59f97b5f5f0128f6bf6fbef56fd836cc52 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 4 Mar 2023 14:18:24 +0100
Subject: [PATCH 123/871] [build] Only archive if `vars.ARCHIVE_REPO` is set

Authored by: Grub4K
---
 .github/workflows/publish.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 42e66a29c..8a1bd9a01 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -55,12 +55,12 @@ jobs:
         run: |
           gh release create \
             --notes-file ARCHIVE_NOTES \
-            --title "Build ${{ inputs.version }}" \
+            --title "yt-dlp nightly ${{ inputs.version }}" \
             ${{ inputs.version }} \
             artifact/*
 
       - name: Prune old nightly release
-        if: inputs.nightly
+        if: inputs.nightly && !vars.ARCHIVE_REPO
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
@@ -71,6 +71,7 @@ jobs:
       - name: Publish release${{ inputs.nightly && ' (nightly)' || '' }}
         env:
           GH_TOKEN: ${{ github.token }}
+        if: (inputs.nightly && !vars.ARCHIVE_REPO) || !inputs.nightly
         run: |
           gh release create \
             --notes-file ${{ inputs.nightly && 'PRE' || '' }}RELEASE_NOTES \

From 22ccd5420b3eb0782776071f12cccd1fedaa1fd0 Mon Sep 17 00:00:00 2001
From: mushbite <mushbite@users.noreply.github.com>
Date: Sat, 4 Mar 2023 15:33:17 +0200
Subject: [PATCH 124/871] [extractor/rutube] Extract chapters from description
 (#6345)

Authored by: mushbite
---
 yt_dlp/extractor/common.py  | 32 ++++++++++++++++++++++++++++
 yt_dlp/extractor/rutube.py  | 34 +++++++++++++++++++++++++-----
 yt_dlp/extractor/youtube.py | 42 +++++--------------------------------
 3 files changed, 66 insertions(+), 42 deletions(-)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 8ad63b411..2091df7fa 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -3649,6 +3649,38 @@ def _generic_title(self, url='', webpage='', *, default=None):
                 or urllib.parse.unquote(os.path.splitext(url_basename(url))[0])
                 or default)
 
+    def _extract_chapters_helper(self, chapter_list, start_function, title_function, duration, strict=True):
+        if not duration:
+            return
+        chapter_list = [{
+            'start_time': start_function(chapter),
+            'title': title_function(chapter),
+        } for chapter in chapter_list or []]
+        if not strict:
+            chapter_list.sort(key=lambda c: c['start_time'] or 0)
+
+        chapters = [{'start_time': 0}]
+        for idx, chapter in enumerate(chapter_list):
+            if chapter['start_time'] is None:
+                self.report_warning(f'Incomplete chapter {idx}')
+            elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
+                chapters.append(chapter)
+            elif chapter not in chapters:
+                self.report_warning(
+                    f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
+        return chapters[1:]
+
+    def _extract_chapters_from_description(self, description, duration):
+        duration_re = r'(?:\d+:)?\d{1,2}:\d{2}'
+        sep_re = r'(?m)^\s*(%s)\b\W*\s(%s)\s*$'
+        return self._extract_chapters_helper(
+            re.findall(sep_re % (duration_re, r'.+?'), description or ''),
+            start_function=lambda x: parse_duration(x[0]), title_function=lambda x: x[1],
+            duration=duration, strict=False) or self._extract_chapters_helper(
+            re.findall(sep_re % (r'.+?', duration_re), description or ''),
+            start_function=lambda x: parse_duration(x[1]), title_function=lambda x: x[0],
+            duration=duration, strict=False)
+
     @staticmethod
     def _availability(is_private=None, needs_premium=None, needs_subscription=None, needs_auth=None, is_unlisted=None):
         all_known = all(map(
diff --git a/yt_dlp/extractor/rutube.py b/yt_dlp/extractor/rutube.py
index 97e6354b4..08d9b9257 100644
--- a/yt_dlp/extractor/rutube.py
+++ b/yt_dlp/extractor/rutube.py
@@ -25,8 +25,7 @@ def _download_api_info(self, video_id, query=None):
             video_id, 'Downloading video JSON',
             'Unable to download video JSON', query=query)
 
-    @staticmethod
-    def _extract_info(video, video_id=None, require_title=True):
+    def _extract_info(self, video, video_id=None, require_title=True):
         title = video['title'] if require_title else video.get('title')
 
         age_limit = video.get('is_adult')
@@ -35,13 +34,15 @@ def _extract_info(video, video_id=None, require_title=True):
 
         uploader_id = try_get(video, lambda x: x['author']['id'])
         category = try_get(video, lambda x: x['category']['name'])
+        description = video.get('description')
+        duration = int_or_none(video.get('duration'))
 
         return {
             'id': video.get('id') or video_id if video_id else video['id'],
             'title': title,
-            'description': video.get('description'),
+            'description': description,
             'thumbnail': video.get('thumbnail_url'),
-            'duration': int_or_none(video.get('duration')),
+            'duration': duration,
             'uploader': try_get(video, lambda x: x['author']['name']),
             'uploader_id': compat_str(uploader_id) if uploader_id else None,
             'timestamp': unified_timestamp(video.get('created_ts')),
@@ -50,6 +51,7 @@ def _extract_info(video, video_id=None, require_title=True):
             'view_count': int_or_none(video.get('hits')),
             'comment_count': int_or_none(video.get('comments_count')),
             'is_live': bool_or_none(video.get('is_livestream')),
+            'chapters': self._extract_chapters_from_description(description, duration),
         }
 
     def _download_and_extract_info(self, video_id, query=None):
@@ -111,8 +113,9 @@ class RutubeIE(RutubeBaseIE):
             'view_count': int,
             'thumbnail': 'http://pic.rutubelist.ru/video/d2/a0/d2a0aec998494a396deafc7ba2c82add.jpg',
             'category': ['Новости и СМИ'],
-
+            'chapters': [],
         },
+        'expected_warnings': ['Unable to download f4m'],
     }, {
         'url': 'http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661',
         'only_matching': True,
@@ -142,7 +145,28 @@ class RutubeIE(RutubeBaseIE):
             'view_count': int,
             'thumbnail': 'http://pic.rutubelist.ru/video/f2/d4/f2d42b54be0a6e69c1c22539e3152156.jpg',
             'category': ['Видеоигры'],
+            'chapters': [],
         },
+        'expected_warnings': ['Unable to download f4m'],
+    }, {
+        'url': 'https://rutube.ru/video/c65b465ad0c98c89f3b25cb03dcc87c6/',
+        'info_dict': {
+            'id': 'c65b465ad0c98c89f3b25cb03dcc87c6',
+            'ext': 'mp4',
+            'chapters': 'count:4',
+            'category': ['Бизнес и предпринимательство'],
+            'description': 'md5:252feac1305257d8c1bab215cedde75d',
+            'thumbnail': 'http://pic.rutubelist.ru/video/71/8f/718f27425ea9706073eb80883dd3787b.png',
+            'duration': 782,
+            'age_limit': 0,
+            'uploader_id': '23491359',
+            'timestamp': 1677153329,
+            'view_count': int,
+            'upload_date': '20230223',
+            'title': 'Бизнес с нуля: найм сотрудников. Интервью с директором строительной компании',
+            'uploader': 'Стас Быков',
+        },
+        'expected_warnings': ['Unable to download f4m'],
     }]
 
     @classmethod
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index b02e0153a..b8bb980f3 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3205,11 +3205,11 @@ def _extract_chapters_from_json(self, data, duration):
                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
             ), expected_type=list)
 
-        return self._extract_chapters(
+        return self._extract_chapters_helper(
             chapter_list,
-            chapter_time=lambda chapter: float_or_none(
+            start_function=lambda chapter: float_or_none(
                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
-            chapter_title=lambda chapter: traverse_obj(
+            title_function=lambda chapter: traverse_obj(
                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
             duration=duration)
 
@@ -3222,42 +3222,10 @@ def _extract_chapters_from_engagement_panel(self, data, duration):
         chapter_title = lambda chapter: self._get_text(chapter, 'title')
 
         return next(filter(None, (
-            self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
-                                   chapter_time, chapter_title, duration)
+            self._extract_chapters_helper(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
+                                          chapter_time, chapter_title, duration)
             for contents in content_list)), [])
 
-    def _extract_chapters_from_description(self, description, duration):
-        duration_re = r'(?:\d+:)?\d{1,2}:\d{2}'
-        sep_re = r'(?m)^\s*(%s)\b\W*\s(%s)\s*$'
-        return self._extract_chapters(
-            re.findall(sep_re % (duration_re, r'.+?'), description or ''),
-            chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
-            duration=duration, strict=False) or self._extract_chapters(
-            re.findall(sep_re % (r'.+?', duration_re), description or ''),
-            chapter_time=lambda x: parse_duration(x[1]), chapter_title=lambda x: x[0],
-            duration=duration, strict=False)
-
-    def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
-        if not duration:
-            return
-        chapter_list = [{
-            'start_time': chapter_time(chapter),
-            'title': chapter_title(chapter),
-        } for chapter in chapter_list or []]
-        if not strict:
-            chapter_list.sort(key=lambda c: c['start_time'] or 0)
-
-        chapters = [{'start_time': 0}]
-        for idx, chapter in enumerate(chapter_list):
-            if chapter['start_time'] is None:
-                self.report_warning(f'Incomplete chapter {idx}')
-            elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
-                chapters.append(chapter)
-            elif chapter not in chapters:
-                self.report_warning(
-                    f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
-        return chapters[1:]
-
     def _extract_comment(self, comment_renderer, parent=None):
         comment_id = comment_renderer.get('commentId')
         if not comment_id:

From 283a0b5bc511f3b350eead4488158f50c20ec526 Mon Sep 17 00:00:00 2001
From: Yakabuff <yekeb00f@gmail.com>
Date: Sat, 4 Mar 2023 08:34:27 -0500
Subject: [PATCH 125/871] [xvideos:quickies] Add extractor (#6414)

Authored by: Yakabuff
Closes #6356
---
 yt_dlp/extractor/_extractors.py |  5 ++++-
 yt_dlp/extractor/xvideos.py     | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index a7bcafb4c..0731845ba 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2315,7 +2315,10 @@
 from .xstream import XstreamIE
 from .xtube import XTubeUserIE, XTubeIE
 from .xuite import XuiteIE
-from .xvideos import XVideosIE
+from .xvideos import (
+    XVideosIE,
+    XVideosQuickiesIE
+)
 from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,
diff --git a/yt_dlp/extractor/xvideos.py b/yt_dlp/extractor/xvideos.py
index 5c505c850..5df071503 100644
--- a/yt_dlp/extractor/xvideos.py
+++ b/yt_dlp/extractor/xvideos.py
@@ -157,3 +157,24 @@ def _real_extract(self, url):
             'thumbnails': thumbnails,
             'age_limit': 18,
         }
+
+
+class XVideosQuickiesIE(InfoExtractor):
+    IE_NAME = 'xvideos:quickies'
+    _VALID_URL = r'https?://(?P<domain>(?:[^/]+\.)?xvideos2?\.com)/amateur-channels/[^#]+#quickies/a/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.xvideos.com/amateur-channels/wifeluna#quickies/a/47258683',
+        'md5': '16e322a93282667f1963915568f782c1',
+        'info_dict': {
+            'id': '47258683',
+            'ext': 'mp4',
+            'title': 'Verification video',
+            'age_limit': 18,
+            'duration': 16,
+            'thumbnail': r're:^https://cdn.*-pic.xvideos-cdn.com/.+\.jpg',
+        }
+    }]
+
+    def _real_extract(self, url):
+        domain, id_ = self._match_valid_url(url).group('domain', 'id')
+        return self.url_result(f'https://{domain}/video{id_}/_', XVideosIE, id_)

From 7accdd9845fe7ce9d0aa5a9d16faaa489c1294eb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 4 Mar 2023 16:39:23 +0530
Subject: [PATCH 126/871] [devscripts] `make_changelog`: Stop at `Release ...`
 commit

Closes #6415
---
 .github/workflows/release-nightly.yml |  2 +-
 devscripts/changelog_override.json    |  4 +--
 devscripts/make_changelog.py          | 36 ++++++---------------------
 3 files changed, 10 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index b0634fa5a..d4f01ab64 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -4,7 +4,7 @@ on:
     branches:
       - master
     paths:
-      - "**.py"
+      - "yt_dlp/**.py"
       - "!yt_dlp/version.py"
 concurrency:
   group: release-nightly
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index a5872d4b4..e5c9d1aa2 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -1,12 +1,12 @@
 [
     {
         "action": "add",
-        "when": "2023.02.17",
+        "when": "776d1c3f0c9b00399896dd2e40e78e9a43218109",
         "short": "[priority] **A new release type has been added!**\n    * [`nightly`](https://github.com/yt-dlp/yt-dlp/releases/tag/nightly) builds will be made after each push, containing the latest fixes (but also possibly bugs).\n    * When using `--update`/`-U`, a release binary will only update to its current channel (either `stable` or `nightly`).\n    * The `--update-to` option has been added allowing the user more control over program upgrades (or downgrades).\n    * `--update-to` can change the release channel (`stable`, `nightly`) and also upgrade or downgrade to specific tags.\n    * **Usage**: `--update-to CHANNEL`, `--update-to TAG`, `--update-to CHANNEL@TAG`"
     },
     {
         "action": "add",
-        "when": "2023.02.17",
+        "when": "776d1c3f0c9b00399896dd2e40e78e9a43218109",
         "short": "[priority] **YouTube throttling fixes!**"
     }
 ]
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index 07aa3285b..722315333 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -248,30 +248,6 @@ def __init__(self, start, end, default_author=None) -> None:
         self._commits, self._fixes = self._get_commits_and_fixes(default_author)
         self._commits_added = []
 
-    @classmethod
-    def from_single(cls, commitish='HEAD', default_author=None):
-        start_commitish = cls.get_prev_tag(commitish)
-        end_commitish = cls.get_next_tag(commitish)
-        if start_commitish == end_commitish:
-            start_commitish = cls.get_prev_tag(f'{commitish}~')
-        logger.info(f'Determined range from {commitish!r}: {start_commitish}..{end_commitish}')
-        return cls(start_commitish, end_commitish, default_author)
-
-    @classmethod
-    def get_prev_tag(cls, commitish):
-        command = [cls.COMMAND, 'describe', '--tags', '--abbrev=0', '--exclude=*[^0-9.]*', commitish]
-        return subprocess.check_output(command, text=True).strip()
-
-    @classmethod
-    def get_next_tag(cls, commitish):
-        result = subprocess.run(
-            [cls.COMMAND, 'describe', '--contains', '--abbrev=0', commitish],
-            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
-        if result.returncode:
-            return 'HEAD'
-
-        return result.stdout.partition('~')[0].strip()
-
     def __iter__(self):
         return iter(itertools.chain(self._commits.values(), self._commits_added))
 
@@ -293,13 +269,12 @@ def _is_ancestor(self, commitish):
     def _get_commits_and_fixes(self, default_author):
         result = subprocess.check_output([
             self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}',
-            f'{self._start}..{self._end}'], text=True)
+            f'{self._start}..{self._end}' if self._start else self._end], text=True)
 
         commits = {}
         fixes = defaultdict(list)
         lines = iter(result.splitlines(False))
-        for line in lines:
-            commit_hash = line
+        for i, commit_hash in enumerate(lines):
             short = next(lines)
             skip = short.startswith('Release ') or short == '[version] update'
 
@@ -310,9 +285,12 @@ def _get_commits_and_fixes(self, default_author):
                     authors = sorted(map(str.strip, line[match.end():].split(',')), key=str.casefold)
 
             commit = Commit(commit_hash, short, authors)
-            if skip:
+            if skip and (self._start or not i):
                 logger.debug(f'Skipped commit: {commit}')
                 continue
+            elif skip:
+                logger.debug(f'Reached Release commit, breaking: {commit}')
+                break
 
             fix_match = self.FIXES_RE.search(commit.short)
             if fix_match:
@@ -471,7 +449,7 @@ def get_new_contributors(contributors_path, commits):
         datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}',
         level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr)
 
-    commits = CommitRange.from_single(args.commitish, args.default_author)
+    commits = CommitRange(None, args.commitish, args.default_author)
 
     if not args.no_override:
         if args.override_path.exists():

From 8a83baaf218ab89e6e7faa76b7c7be3a2ec19e3a Mon Sep 17 00:00:00 2001
From: LXYan2333 <z00823823@126.com>
Date: Sat, 4 Mar 2023 22:44:48 +0800
Subject: [PATCH 127/871] [extractor/bilibili] Fix for downloading wrong
 subtitles (#6358)

Closes #6357
Authored by: LXYan2333
---
 yt_dlp/extractor/bilibili.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 2252840b3..c34439779 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -81,7 +81,7 @@ def json2srt(self, json_data):
                          f'{line["content"]}\n\n')
         return srt_data
 
-    def _get_subtitles(self, video_id, initial_state, cid):
+    def _get_subtitles(self, video_id, aid, cid):
         subtitles = {
             'danmaku': [{
                 'ext': 'xml',
@@ -89,7 +89,8 @@ def _get_subtitles(self, video_id, initial_state, cid):
             }]
         }
 
-        for s in traverse_obj(initial_state, ('videoData', 'subtitle', 'list')) or []:
+        video_info_json = self._download_json(f'https://api.bilibili.com/x/player/v2?aid={aid}&cid={cid}', video_id)
+        for s in traverse_obj(video_info_json, ('data', 'subtitle', 'subtitles', ...)):
             subtitles.setdefault(s['lan'], []).append({
                 'ext': 'srt',
                 'data': self.json2srt(self._download_json(s['subtitle_url'], video_id))
@@ -331,7 +332,7 @@ def _real_extract(self, url):
             'timestamp': traverse_obj(initial_state, ('videoData', 'pubdate')),
             'duration': float_or_none(play_info.get('timelength'), scale=1000),
             'chapters': self._get_chapters(aid, cid),
-            'subtitles': self.extract_subtitles(video_id, initial_state, cid),
+            'subtitles': self.extract_subtitles(video_id, aid, cid),
             '__post_extractor': self.extract_comments(aid),
             'http_headers': {'Referer': url},
         }

From 45db357289b4e1eec09093c8bc5446520378f426 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 4 Mar 2023 21:45:57 +0530
Subject: [PATCH 128/871] [extractor/SportDeutschland] Rewrite extractor

Note: `multi_video` live streams are untested

Closes #6417, closes #6418, closes #6420
---
 yt_dlp/extractor/sportdeutschland.py | 144 ++++++++++++++++-----------
 1 file changed, 88 insertions(+), 56 deletions(-)

diff --git a/yt_dlp/extractor/sportdeutschland.py b/yt_dlp/extractor/sportdeutschland.py
index 6fc3ce9eb..6490f4203 100644
--- a/yt_dlp/extractor/sportdeutschland.py
+++ b/yt_dlp/extractor/sportdeutschland.py
@@ -1,10 +1,9 @@
 from .common import InfoExtractor
-
 from ..utils import (
-    format_field,
+    join_nonempty,
+    strip_or_none,
     traverse_obj,
     unified_timestamp,
-    strip_or_none
 )
 
 
@@ -13,98 +12,131 @@ class SportDeutschlandIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://sportdeutschland.tv/blauweissbuchholztanzsport/buchholzer-formationswochenende-2023-samstag-1-bundesliga-landesliga',
         'info_dict': {
-            'id': '983758e9-5829-454d-a3cf-eb27bccc3c94',
+            'id': '9839a5c7-0dbb-48a8-ab63-3b408adc7b54',
             'ext': 'mp4',
             'title': 'Buchholzer Formationswochenende 2023 - Samstag - 1. Bundesliga / Landesliga',
+            'display_id': 'blauweissbuchholztanzsport/buchholzer-formationswochenende-2023-samstag-1-bundesliga-landesliga',
             'description': 'md5:a288c794a5ee69e200d8f12982f81a87',
             'live_status': 'was_live',
             'channel': 'Blau-Weiss Buchholz Tanzsport',
             'channel_url': 'https://sportdeutschland.tv/blauweissbuchholztanzsport',
             'channel_id': '93ec33c9-48be-43b6-b404-e016b64fdfa3',
-            'display_id': '9839a5c7-0dbb-48a8-ab63-3b408adc7b54',
             'duration': 32447,
             'upload_date': '20230114',
-            'timestamp': 1673730018.0,
+            'timestamp': 1673733618,
         }
     }, {
         'url': 'https://sportdeutschland.tv/deutscherbadmintonverband/bwf-tour-1-runde-feld-1-yonex-gainward-german-open-2022-0',
         'info_dict': {
-            'id': '95b97d9a-04f6-4880-9039-182985c33943',
+            'id': '95c80c52-6b9a-4ae9-9197-984145adfced',
             'ext': 'mp4',
             'title': 'BWF Tour: 1. Runde Feld 1 - YONEX GAINWARD German Open 2022',
+            'display_id': 'deutscherbadmintonverband/bwf-tour-1-runde-feld-1-yonex-gainward-german-open-2022-0',
             'description': 'md5:2afb5996ceb9ac0b2ac81f563d3a883e',
             'live_status': 'was_live',
             'channel': 'Deutscher Badminton Verband',
             'channel_url': 'https://sportdeutschland.tv/deutscherbadmintonverband',
             'channel_id': '93ca5866-2551-49fc-8424-6db35af58920',
-            'display_id': '95c80c52-6b9a-4ae9-9197-984145adfced',
             'duration': 41097,
             'upload_date': '20220309',
             'timestamp': 1646860727.0,
         }
+    }, {
+        'url': 'https://sportdeutschland.tv/ggcbremen/formationswochenende-latein-2023',
+        'info_dict': {
+            'id': '9889785e-55b0-4d97-a72a-ce9a9f157cce',
+            'title': 'Formationswochenende Latein 2023 - Samstag',
+            'display_id': 'ggcbremen/formationswochenende-latein-2023',
+            'description': 'md5:6e4060d40ff6a8f8eeb471b51a8f08b2',
+            'live_status': 'was_live',
+            'channel': 'Grün-Gold-Club Bremen e.V.',
+            'channel_id': '9888f04e-bb46-4c7f-be47-df960a4167bb',
+            'channel_url': 'https://sportdeutschland.tv/ggcbremen',
+        },
+        'playlist_count': 3,
+        'playlist': [{
+            'info_dict': {
+                'id': '988e1fea-9d44-4fab-8c72-3085fb667547',
+                'ext': 'mp4',
+                'channel_url': 'https://sportdeutschland.tv/ggcbremen',
+                'channel_id': '9888f04e-bb46-4c7f-be47-df960a4167bb',
+                'channel': 'Grün-Gold-Club Bremen e.V.',
+                'duration': 86,
+                'title': 'Formationswochenende Latein 2023 - Samstag Part 1',
+                'upload_date': '20230225',
+                'timestamp': 1677349909,
+                'live_status': 'was_live',
+            }
+        }]
+    }, {
+        'url': 'https://sportdeutschland.tv/dtb/gymnastik-international-tag-1',
+        'info_dict': {
+            'id': '95d71b8a-370a-4b87-ad16-94680da18528',
+            'ext': 'mp4',
+            'title': r're:Gymnastik International - Tag 1 .+',
+            'display_id': 'dtb/gymnastik-international-tag-1',
+            'channel_id': '936ecef1-2f4a-4e08-be2f-68073cb7ecab',
+            'channel': 'Deutscher Turner-Bund',
+            'channel_url': 'https://sportdeutschland.tv/dtb',
+            'description': 'md5:07a885dde5838a6f0796ee21dc3b0c52',
+            'live_status': 'is_live',
+        },
+        'skip': 'live',
     }]
 
+    def _process_video(self, asset_id, video):
+        is_live = video['type'] == 'mux_live'
+        token = self._download_json(
+            f'https://api.sportdeutschland.tv/api/frontend/asset-token/{asset_id}',
+            video['id'], query={'type': video['type'], 'playback_id': video['src']})['token']
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            f'https://stream.mux.com/{video["src"]}.m3u8?token={token}', video['id'], live=is_live)
+
+        return {
+            'is_live': is_live,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(video, {
+                'id': 'id',
+                'duration': ('duration', {lambda x: float(x) > 0 and float(x)}),
+                'timestamp': ('created_at', {unified_timestamp})
+            }),
+        }
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
         meta = self._download_json(
-            'https://api.sportdeutschland.tv/api/stateless/frontend/assets/' + display_id,
+            f'https://api.sportdeutschland.tv/api/stateless/frontend/assets/{display_id}',
             display_id, query={'access_token': 'true'})
 
-        asset_id = traverse_obj(meta, 'id', 'uuid')
-
         info = {
-            'id': asset_id,
-            'channel_url': format_field(meta, ('profile', 'slug'), 'https://sportdeutschland.tv/%s'),
+            'display_id': display_id,
             **traverse_obj(meta, {
+                'id': (('id', 'uuid'), ),
                 'title': (('title', 'name'), {strip_or_none}),
                 'description': 'description',
                 'channel': ('profile', 'name'),
                 'channel_id': ('profile', 'id'),
                 'is_live': 'currently_live',
-                'was_live': 'was_live'
+                'was_live': 'was_live',
+                'channel_url': ('profile', 'slug', {lambda x: f'https://sportdeutschland.tv/{x}'}),
             }, get_all=False)
         }
 
-        videos = meta.get('videos') or []
-
-        if len(videos) > 1:
-            info.update({
-                '_type': 'multi_video',
-                'entries': self.processVideoOrStream(asset_id, video)
-            } for video in enumerate(videos) if video.get('formats'))
-
-        elif len(videos) == 1:
-            info.update(
-                self.processVideoOrStream(asset_id, videos[0])
-            )
-
-        livestream = meta.get('livestream')
-
-        if livestream is not None:
-            info.update(
-                self.processVideoOrStream(asset_id, livestream)
-            )
-
-        return info
-
-    def process_video_or_stream(self, asset_id, video):
-        video_id = video['id']
-        video_src = video['src']
-        video_type = video['type']
-
-        token = self._download_json(
-            f'https://api.sportdeutschland.tv/api/frontend/asset-token/{asset_id}',
-            video_id, query={'type': video_type, 'playback_id': video_src})['token']
-        formats = self._extract_m3u8_formats(f'https://stream.mux.com/{video_src}.m3u8?token={token}', video_id)
-
-        video_data = {
-            'display_id': video_id,
-            'formats': formats,
+        parts = traverse_obj(meta, (('livestream', ('videos', ...)), ))
+        entries = [{
+            'title': join_nonempty(info.get('title'), f'Part {i}', delim=' '),
+            **traverse_obj(info, {'channel': 'channel', 'channel_id': 'channel_id',
+                                  'channel_url': 'channel_url', 'was_live': 'was_live'}),
+            **self._process_video(info['id'], video),
+        } for i, video in enumerate(parts, 1)]
+        print(entries[0]['duration'])
+        return {
+            '_type': 'multi_video',
+            **info,
+            'entries': entries,
+        } if len(entries) > 1 else {
+            **info,
+            **entries[0],
+            'title': info.get('title'),
         }
-        if video_type == 'mux_vod':
-            video_data.update({
-                'duration': video.get('duration'),
-                'timestamp': unified_timestamp(video.get('created_at'))
-            })
-
-        return video_data

From cf605226521e99c89fc8dff26a319025810e63a0 Mon Sep 17 00:00:00 2001
From: "lauren n. liberda" <lauren@selfisekai.rocks>
Date: Sat, 4 Mar 2023 18:51:33 +0100
Subject: [PATCH 129/871] [extractor/twitter] Fix retweet extraction (#6422)

Authored by: selfisekai
---
 yt_dlp/extractor/twitter.py | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index d9d446832..3f1899e96 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -838,6 +838,28 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1670306984.0,
         },
         'params': {'extractor_args': {'twitter': {'force_graphql': ['']}}},
+    }, {
+        # url to retweet id
+        'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
+        'info_dict': {
+            'id': '1623274794488659969',
+            'display_id': '1623739803874349067',
+            'ext': 'mp4',
+            'title': 'Johnny Bullets - Me after going viral to over 30million people:    Whoopsie-daisy',
+            'description': 'md5:e873616a4a8fe0f93e71872678a672f3',
+            'uploader': 'Johnny Bullets',
+            'uploader_id': 'Johnnybull3ts',
+            'uploader_url': 'https://twitter.com/Johnnybull3ts',
+            'age_limit': 0,
+            'tags': [],
+            'duration': 8.033,
+            'timestamp': 1675853859.0,
+            'upload_date': '20230208',
+            'thumbnail': r're:https://pbs\.twimg\.com/ext_tw_video_thumb/.+',
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+        },
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -949,13 +971,13 @@ def _real_extract(self, url):
             status = self._graphql_to_legacy(result, twid)
 
         else:
-            status = self._call_api(f'statuses/show/{twid}.json', twid, {
+            status = traverse_obj(self._call_api(f'statuses/show/{twid}.json', twid, {
                 'cards_platform': 'Web-12',
                 'include_cards': 1,
                 'include_reply_count': 1,
                 'include_user_entities': 0,
                 'tweet_mode': 'extended',
-            })
+            }), 'retweeted_status', None)
 
         title = description = status['full_text'].replace('\n', ' ')
         # strip  'https -_t.co_BJYgOjSeGA' junk from filenames

From ed4cc4ea793314c50ae3f82e98248c1de1c25694 Mon Sep 17 00:00:00 2001
From: columndeeply <106948293+columndeeply@users.noreply.github.com>
Date: Sat, 4 Mar 2023 17:52:15 +0000
Subject: [PATCH 130/871] [extractor/Prankcast] Fix tags (#6316)

Authored by: columndeeply
---
 yt_dlp/extractor/prankcast.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/prankcast.py b/yt_dlp/extractor/prankcast.py
index 0eb5f98d1..b2ec5bbb8 100644
--- a/yt_dlp/extractor/prankcast.py
+++ b/yt_dlp/extractor/prankcast.py
@@ -18,7 +18,7 @@ class PrankCastIE(InfoExtractor):
             'cast': ['Devonanustart', 'Phonelosers'],
             'description': '',
             'categories': ['prank'],
-            'tags': ['prank call', 'prank'],
+            'tags': ['prank call', 'prank', 'live show'],
             'upload_date': '20220825'
         }
     }, {
@@ -35,7 +35,7 @@ class PrankCastIE(InfoExtractor):
             'cast': ['phonelosers'],
             'description': '',
             'categories': ['prank'],
-            'tags': ['prank call', 'prank'],
+            'tags': ['prank call', 'prank', 'live show'],
             'upload_date': '20221006'
         }
     }]
@@ -62,5 +62,5 @@ def _real_extract(self, url):
             'cast': list(filter(None, [uploader] + traverse_obj(guests_json, (..., 'name')))),
             'description': json_info.get('broadcast_description'),
             'categories': [json_info.get('broadcast_category')],
-            'tags': self._parse_json(json_info.get('broadcast_tags') or '{}', video_id)
+            'tags': try_call(lambda: json_info['broadcast_tags'].split(','))
         }

From 1f8489cccbdc6e96027ef527b88717458f0900e8 Mon Sep 17 00:00:00 2001
From: Chris Caruso <carusochrisr@gmail.com>
Date: Sat, 4 Mar 2023 11:22:11 -0800
Subject: [PATCH 131/871] [extractor/lumni] Add extractor (#6302)

Authored by: carusocr
Closes #6202
---
 yt_dlp/extractor/_extractors.py |  3 +++
 yt_dlp/extractor/lumni.py       | 24 ++++++++++++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100644 yt_dlp/extractor/lumni.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 0731845ba..ccac634b3 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -962,6 +962,9 @@
     LRTVODIE,
     LRTStreamIE
 )
+from .lumni import (
+    LumniIE
+)
 from .lynda import (
     LyndaIE,
     LyndaCourseIE
diff --git a/yt_dlp/extractor/lumni.py b/yt_dlp/extractor/lumni.py
new file mode 100644
index 000000000..5810da0c8
--- /dev/null
+++ b/yt_dlp/extractor/lumni.py
@@ -0,0 +1,24 @@
+from .common import InfoExtractor
+from .francetv import FranceTVIE
+
+
+class LumniIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?lumni\.fr/video/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.lumni.fr/video/l-homme-et-son-environnement-dans-la-revolution-industrielle',
+        'md5': '960e8240c4f2c7a20854503a71e52f5e',
+        'info_dict': {
+            'id': 'd2b9a4e5-a526-495b-866c-ab72737e3645',
+            'ext': 'mp4',
+            'title': "L'homme et son environnement dans la révolution industrielle - L'ère de l'homme",
+            'thumbnail': 'https://assets.webservices.francetelevisions.fr/v1/assets/images/a7/17/9f/a7179f5f-63a5-4e11-8d4d-012ab942d905.jpg',
+            'duration': 230,
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._html_search_regex(
+            r'<div[^>]+data-factoryid\s*=\s*["\']([^"\']+)', webpage, 'video id')
+        return self.url_result(f'francetv:{video_id}', FranceTVIE, video_id)

From b40471282286bd2b09c485bf79afd271d229272c Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Sat, 4 Mar 2023 13:41:41 -0600
Subject: [PATCH 132/871] [extractor/telecaribe] Add extractor (#6311)

Authored by: elyse0
Closes #6001
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/telecaribe.py  | 77 +++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 yt_dlp/extractor/telecaribe.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ccac634b3..b7bce6a5e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1854,6 +1854,7 @@
 from .tele5 import Tele5IE
 from .tele13 import Tele13IE
 from .telebruxelles import TeleBruxellesIE
+from .telecaribe import TelecaribePlayIE
 from .telecinco import TelecincoIE
 from .telegraaf import TelegraafIE
 from .telegram import TelegramEmbedIE
diff --git a/yt_dlp/extractor/telecaribe.py b/yt_dlp/extractor/telecaribe.py
new file mode 100644
index 000000000..b6d88a809
--- /dev/null
+++ b/yt_dlp/extractor/telecaribe.py
@@ -0,0 +1,77 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import traverse_obj
+
+
+class TelecaribePlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?play\.telecaribe\.co/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.play.telecaribe.co/breicok',
+        'info_dict': {
+            'id': 'breicok',
+            'title': 'Breicok',
+        },
+        'playlist_count': 7,
+    }, {
+        'url': 'https://www.play.telecaribe.co/si-fue-gol-de-yepes',
+        'info_dict': {
+            'id': 'si-fue-gol-de-yepes',
+            'title': 'Sí Fue Gol de Yepes',
+        },
+        'playlist_count': 6,
+    }, {
+        'url': 'https://www.play.telecaribe.co/ciudad-futura',
+        'info_dict': {
+            'id': 'ciudad-futura',
+            'title': 'Ciudad Futura',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'https://www.play.telecaribe.co/live',
+        'info_dict': {
+            'id': 'live',
+            'title': r're:^Señal en vivo',
+            'live_status': 'is_live',
+            'ext': 'mp4',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        }
+    }]
+
+    def _download_player_webpage(self, webpage, display_id):
+        page_id = self._search_regex(
+            (r'window.firstPageId\s*=\s*["\']([^"\']+)', r'<div[^>]+id\s*=\s*"pageBackground_([^"]+)'),
+            webpage, 'page_id')
+
+        props = self._download_json(self._search_regex(
+            rf'<link[^>]+href\s*=\s*"([^"]+)"[^>]+id\s*=\s*"features_{page_id}"',
+            webpage, 'json_props_url'), display_id)['props']['render']['compProps']
+
+        return self._download_webpage(traverse_obj(props, (..., 'url'))[-1], display_id)
+
+    def _get_clean_title(self, title):
+        return re.sub(r'\s*\|\s*Telecaribe\s*VOD', '', title or '').strip() or None
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        player = self._download_player_webpage(webpage, display_id)
+
+        if display_id != 'live':
+            return self.playlist_from_matches(
+                re.findall(r'<a[^>]+href\s*=\s*"([^"]+\.mp4)', player), display_id,
+                self._get_clean_title(self._og_search_title(webpage)))
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            self._search_regex(r'(?:let|const|var)\s+source\s*=\s*["\']([^"\']+)', player, 'm3u8 url'),
+            display_id, 'mp4')
+
+        return {
+            'id': display_id,
+            'title': self._get_clean_title(self._og_search_title(webpage)),
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': True,
+        }

From 46580ced56c90b559885aded6aa8f46f20a9cdce Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Sat, 4 Mar 2023 14:05:19 -0600
Subject: [PATCH 133/871] [extractor/tunein] Fix extractors (#6310)

Authored by: elyse0
Closes #2973
---
 yt_dlp/extractor/_extractors.py |   5 +-
 yt_dlp/extractor/tunein.py      | 282 +++++++++++++++++++-------------
 2 files changed, 173 insertions(+), 114 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b7bce6a5e..917cc6ab9 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1967,10 +1967,9 @@
 )
 from .tumblr import TumblrIE
 from .tunein import (
-    TuneInClipIE,
     TuneInStationIE,
-    TuneInProgramIE,
-    TuneInTopicIE,
+    TuneInPodcastIE,
+    TuneInPodcastEpisodeIE,
     TuneInShortenerIE,
 )
 from .tunepk import TunePkIE
diff --git a/yt_dlp/extractor/tunein.py b/yt_dlp/extractor/tunein.py
index 43b4f673c..e02121bd8 100644
--- a/yt_dlp/extractor/tunein.py
+++ b/yt_dlp/extractor/tunein.py
@@ -1,149 +1,201 @@
-import re
+import urllib.parse
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
-from ..compat import compat_urlparse
+from ..utils import (
+    OnDemandPagedList,
+    determine_ext,
+    parse_iso8601,
+    traverse_obj,
+)
 
 
 class TuneInBaseIE(InfoExtractor):
-    _API_BASE_URL = 'http://tunein.com/tuner/tune/'
+    _VALID_URL_BASE = r'https?://(?:www\.)?tunein\.com'
 
-    def _real_extract(self, url):
-        content_id = self._match_id(url)
-
-        content_info = self._download_json(
-            self._API_BASE_URL + self._API_URL_QUERY % content_id,
-            content_id, note='Downloading JSON metadata')
-
-        title = content_info['Title']
-        thumbnail = content_info.get('Logo')
-        location = content_info.get('Location')
-        streams_url = content_info.get('StreamUrl')
-        if not streams_url:
-            raise ExtractorError('No downloadable streams found', expected=True)
-        if not streams_url.startswith('http://'):
-            streams_url = compat_urlparse.urljoin(url, streams_url)
+    def _extract_metadata(self, webpage, content_id):
+        return self._search_json(r'window.INITIAL_STATE=', webpage, 'hydration', content_id, fatal=False)
 
+    def _extract_formats_and_subtitles(self, content_id):
         streams = self._download_json(
-            streams_url, content_id, note='Downloading stream data',
-            transform_source=lambda s: re.sub(r'^\s*\((.*)\);\s*$', r'\1', s))['Streams']
+            f'https://opml.radiotime.com/Tune.ashx?render=json&formats=mp3,aac,ogg,flash,hls&id={content_id}',
+            content_id)['body']
 
-        is_live = None
-        formats = []
+        formats, subtitles = [], {}
         for stream in streams:
-            if stream.get('Type') == 'Live':
-                is_live = True
-            reliability = stream.get('Reliability')
-            format_note = (
-                'Reliability: %d%%' % reliability
-                if reliability is not None else None)
-            formats.append({
-                'preference': (
-                    0 if reliability is None or reliability > 90
-                    else 1),
-                'abr': stream.get('Bandwidth'),
-                'ext': stream.get('MediaType').lower(),
-                'acodec': stream.get('MediaType'),
-                'vcodec': 'none',
-                'url': stream.get('Url'),
-                'source_preference': reliability,
-                'format_note': format_note,
-            })
+            if stream.get('media_type') == 'hls':
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(stream['url'], content_id, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            elif determine_ext(stream['url']) == 'pls':
+                playlist_content = self._download_webpage(stream['url'], content_id)
+                formats.append({
+                    'url': self._search_regex(r'File1=(.*)', playlist_content, 'url', fatal=False),
+                    'abr': stream.get('bitrate'),
+                    'ext': stream.get('media_type'),
+                })
+            else:
+                formats.append({
+                    'url': stream['url'],
+                    'abr': stream.get('bitrate'),
+                    'ext': stream.get('media_type'),
+                })
 
-        return {
-            'id': content_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-            'location': location,
-            'is_live': is_live,
-        }
-
-
-class TuneInClipIE(TuneInBaseIE):
-    IE_NAME = 'tunein:clip'
-    _VALID_URL = r'https?://(?:www\.)?tunein\.com/station/.*?audioClipId\=(?P<id>\d+)'
-    _API_URL_QUERY = '?tuneType=AudioClip&audioclipId=%s'
-
-    _TESTS = [{
-        'url': 'http://tunein.com/station/?stationId=246119&audioClipId=816',
-        'md5': '99f00d772db70efc804385c6b47f4e77',
-        'info_dict': {
-            'id': '816',
-            'title': '32m',
-            'ext': 'mp3',
-        },
-    }]
+        return formats, subtitles
 
 
 class TuneInStationIE(TuneInBaseIE):
-    IE_NAME = 'tunein:station'
-    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-s|station/.*?StationId=|embed/player/s)(?P<id>\d+)'
-    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/[pst]\d+)']
-    _API_URL_QUERY = '?tuneType=Station&stationId=%s'
-
-    @classmethod
-    def suitable(cls, url):
-        return False if TuneInClipIE.suitable(url) else super(TuneInStationIE, cls).suitable(url)
+    _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'(?:/radio/[^?#]+-|/embed/player/)(?P<id>s\d+)'
+    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/s\d+)']
 
     _TESTS = [{
-        'url': 'http://tunein.com/radio/Jazz24-885-s34682/',
+        'url': 'https://tunein.com/radio/Jazz24-885-s34682/',
         'info_dict': {
-            'id': '34682',
-            'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
+            'id': 's34682',
+            'title': 're:^Jazz24',
+            'description': 'md5:d6d0b89063fd68d529fa7058ee98619b',
+            'thumbnail': 're:^https?://[^?&]+/s34682',
+            'location': 'Seattle-Tacoma, US',
             'ext': 'mp3',
-            'location': 'Tacoma, WA',
+            'live_status': 'is_live',
         },
         'params': {
-            'skip_download': True,  # live stream
+            'skip_download': True,
         },
     }, {
-        'url': 'http://tunein.com/embed/player/s6404/',
+        'url': 'https://tunein.com/embed/player/s6404/',
         'only_matching': True,
-    }]
-
-
-class TuneInProgramIE(TuneInBaseIE):
-    IE_NAME = 'tunein:program'
-    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-p|program/.*?ProgramId=|embed/player/p)(?P<id>\d+)'
-    _API_URL_QUERY = '?tuneType=Program&programId=%s'
-
-    _TESTS = [{
-        'url': 'http://tunein.com/radio/Jazz-24-p2506/',
+    }, {
+        'url': 'https://tunein.com/radio/BBC-Radio-1-988-s24939/',
         'info_dict': {
-            'id': '2506',
-            'title': 'Jazz 24 on 91.3 WUKY-HD3',
+            'id': 's24939',
+            'title': 're:^BBC Radio 1',
+            'description': 'md5:f3f75f7423398d87119043c26e7bfb84',
+            'thumbnail': 're:^https?://[^?&]+/s24939',
+            'location': 'London, UK',
             'ext': 'mp3',
-            'location': 'Lexington, KY',
+            'live_status': 'is_live',
         },
         'params': {
-            'skip_download': True,  # live stream
+            'skip_download': True,
         },
-    }, {
-        'url': 'http://tunein.com/embed/player/p191660/',
-        'only_matching': True,
     }]
 
+    def _real_extract(self, url):
+        station_id = self._match_id(url)
 
-class TuneInTopicIE(TuneInBaseIE):
-    IE_NAME = 'tunein:topic'
-    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:topic/.*?TopicId=|embed/player/t)(?P<id>\d+)'
-    _API_URL_QUERY = '?tuneType=Topic&topicId=%s'
+        webpage = self._download_webpage(url, station_id)
+        metadata = self._extract_metadata(webpage, station_id)
+
+        formats, subtitles = self._extract_formats_and_subtitles(station_id)
+        return {
+            'id': station_id,
+            'title': traverse_obj(metadata, ('profiles', station_id, 'title')),
+            'description': traverse_obj(metadata, ('profiles', station_id, 'description')),
+            'thumbnail': traverse_obj(metadata, ('profiles', station_id, 'image')),
+            'timestamp': parse_iso8601(
+                traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'publishTime'))),
+            'location': traverse_obj(
+                metadata, ('profiles', station_id, 'metadata', 'properties', 'location', 'displayName'),
+                ('profiles', station_id, 'properties', 'location', 'displayName')),
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'isLive')),
+        }
+
+
+class TuneInPodcastIE(TuneInBaseIE):
+    _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/(?:podcasts/[^?#]+-|embed/player/)(?P<id>p\d+)/?(?:#|$)'
+    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/p\d+)']
 
     _TESTS = [{
-        'url': 'http://tunein.com/topic/?TopicId=101830576',
-        'md5': 'c31a39e6f988d188252eae7af0ef09c9',
+        'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019',
         'info_dict': {
-            'id': '101830576',
-            'title': 'Votez pour moi du 29 octobre 2015 (29/10/15)',
-            'ext': 'mp3',
-            'location': 'Belgium',
+            'id': 'p1153019',
+            'title': 'Lex Fridman Podcast',
+            'description': 'md5:bedc4e5f1c94f7dec6e4317b5654b00d',
         },
+        'playlist_mincount': 200,
     }, {
-        'url': 'http://tunein.com/embed/player/t101830576/',
-        'only_matching': True,
+        'url': 'https://tunein.com/embed/player/p191660/',
+        'only_matching': True
+    }, {
+        'url': 'https://tunein.com/podcasts/World-News/BBC-News-p14/',
+        'info_dict': {
+            'id': 'p14',
+            'title': 'BBC News',
+            'description': 'md5:1218e575eeaff75f48ed978261fa2068',
+        },
+        'playlist_mincount': 200,
     }]
 
+    _PAGE_SIZE = 30
+
+    def _real_extract(self, url):
+        podcast_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, podcast_id, fatal=False)
+        metadata = self._extract_metadata(webpage, podcast_id)
+
+        def page_func(page_num):
+            api_response = self._download_json(
+                f'https://api.tunein.com/profiles/{podcast_id}/contents', podcast_id,
+                note=f'Downloading page {page_num + 1}', query={
+                    'filter': 't:free',
+                    'offset': page_num * self._PAGE_SIZE,
+                    'limit': self._PAGE_SIZE,
+                })
+
+            return [
+                self.url_result(
+                    f'https://tunein.com/podcasts/{podcast_id}?topicId={episode["GuideId"][1:]}',
+                    TuneInPodcastEpisodeIE, title=episode.get('Title'))
+                for episode in api_response['Items']]
+
+        entries = OnDemandPagedList(page_func, self._PAGE_SIZE)
+        return self.playlist_result(
+            entries, playlist_id=podcast_id, title=traverse_obj(metadata, ('profiles', podcast_id, 'title')),
+            description=traverse_obj(metadata, ('profiles', podcast_id, 'description')))
+
+
+class TuneInPodcastEpisodeIE(TuneInBaseIE):
+    _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/podcasts/(?:[^?&]+-)?(?P<podcast_id>p\d+)/?\?topicId=(?P<id>\w\d+)'
+
+    _TESTS = [{
+        'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019/?topicId=236404354',
+        'info_dict': {
+            'id': 't236404354',
+            'title': '#351 \u2013 MrBeast: Future of YouTube, Twitter, TikTok, and Instagram',
+            'description': 'md5:e1734db6f525e472c0c290d124a2ad77',
+            'thumbnail': 're:^https?://[^?&]+/p1153019',
+            'timestamp': 1673458571,
+            'upload_date': '20230111',
+            'series_id': 'p1153019',
+            'series': 'Lex Fridman Podcast',
+            'ext': 'mp3',
+        },
+    }]
+
+    def _real_extract(self, url):
+        podcast_id, episode_id = self._match_valid_url(url).group('podcast_id', 'id')
+        episode_id = f't{episode_id}'
+
+        webpage = self._download_webpage(url, episode_id)
+        metadata = self._extract_metadata(webpage, episode_id)
+
+        formats, subtitles = self._extract_formats_and_subtitles(episode_id)
+        return {
+            'id': episode_id,
+            'title': traverse_obj(metadata, ('profiles', episode_id, 'title')),
+            'description': traverse_obj(metadata, ('profiles', episode_id, 'description')),
+            'thumbnail': traverse_obj(metadata, ('profiles', episode_id, 'image')),
+            'timestamp': parse_iso8601(
+                traverse_obj(metadata, ('profiles', episode_id, 'actions', 'play', 'publishTime'))),
+            'series_id': podcast_id,
+            'series': traverse_obj(metadata, ('profiles', podcast_id, 'title')),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
 
 class TuneInShortenerIE(InfoExtractor):
     IE_NAME = 'tunein:shortener'
@@ -154,10 +206,13 @@ class TuneInShortenerIE(InfoExtractor):
         # test redirection
         'url': 'http://tun.in/ser7s',
         'info_dict': {
-            'id': '34682',
-            'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
+            'id': 's34682',
+            'title': 're:^Jazz24',
+            'description': 'md5:d6d0b89063fd68d529fa7058ee98619b',
+            'thumbnail': 're:^https?://[^?&]+/s34682',
+            'location': 'Seattle-Tacoma, US',
             'ext': 'mp3',
-            'location': 'Tacoma, WA',
+            'live_status': 'is_live',
         },
         'params': {
             'skip_download': True,  # live stream
@@ -169,6 +224,11 @@ def _real_extract(self, url):
         # The server doesn't support HEAD requests
         urlh = self._request_webpage(
             url, redirect_id, note='Downloading redirect page')
+
         url = urlh.geturl()
+        url_parsed = urllib.parse.urlparse(url)
+        if url_parsed.port == 443:
+            url = url_parsed._replace(netloc=url_parsed.hostname).geturl()
+
         self.to_screen('Following redirect: %s' % url)
         return self.url_result(url)

From f44cb4e77bb9be8be291d02ab6f79dc0b4c0d4a1 Mon Sep 17 00:00:00 2001
From: Ferdinand Bachmann <theferdi265@gmail.com>
Date: Sat, 4 Mar 2023 22:58:16 +0100
Subject: [PATCH 134/871] [extractor/tubetugraz] Support `--twofactor` (#6424)
 (#6427)

Authored by: Ferdi265
Closes #6424
---
 yt_dlp/extractor/tubetugraz.py | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/tubetugraz.py b/yt_dlp/extractor/tubetugraz.py
index ebabedc9c..2199fea19 100644
--- a/yt_dlp/extractor/tubetugraz.py
+++ b/yt_dlp/extractor/tubetugraz.py
@@ -21,17 +21,36 @@ def _perform_login(self, username, password):
         if not urlh:
             return
 
-        urlh = self._request_webpage(
+        content, urlh = self._download_webpage_handle(
             urlh.geturl(), None, fatal=False, headers={'referer': urlh.geturl()},
-            note='logging in', errnote='unable to log in', data=urlencode_postdata({
+            note='logging in', errnote='unable to log in',
+            data=urlencode_postdata({
                 'lang': 'de',
                 '_eventId_proceed': '',
                 'j_username': username,
                 'j_password': password
             }))
+        if not urlh or urlh.geturl() == 'https://tube.tugraz.at/paella/ui/index.html':
+            return
 
-        if urlh and urlh.geturl() != 'https://tube.tugraz.at/paella/ui/index.html':
+        if not self._html_search_regex(
+                r'<p\b[^>]*>(Bitte geben Sie einen OTP-Wert ein:)</p>',
+                content, 'TFA prompt', default=None):
             self.report_warning('unable to login: incorrect password')
+            return
+
+        content, urlh = self._download_webpage_handle(
+            urlh.geturl(), None, fatal=False, headers={'referer': urlh.geturl()},
+            note='logging in with TFA', errnote='unable to log in with TFA',
+            data=urlencode_postdata({
+                'lang': 'de',
+                '_eventId_proceed': '',
+                'j_tokenNumber': self._get_tfa_info(),
+            }))
+        if not urlh or urlh.geturl() == 'https://tube.tugraz.at/paella/ui/index.html':
+            return
+
+        self.report_warning('unable to login: incorrect TFA code')
 
     def _extract_episode(self, episode_info):
         id = episode_info.get('id')

From eb8fd6d044e8926532772b72be0645c6b8ecb3aa Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Sat, 4 Mar 2023 16:00:45 -0600
Subject: [PATCH 135/871] [extractor/lefigaro] Add extractors (#6309)

Authored by: elyse0
Closes #6197
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/lefigaro.py    | 135 ++++++++++++++++++++++++++++++++
 2 files changed, 139 insertions(+)
 create mode 100644 yt_dlp/extractor/lefigaro.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 917cc6ab9..cc7490590 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -914,6 +914,10 @@
     LePlaylistIE,
     LetvCloudIE,
 )
+from .lefigaro import (
+    LeFigaroVideoEmbedIE,
+    LeFigaroVideoSectionIE,
+)
 from .lego import LEGOIE
 from .lemonde import LemondeIE
 from .lenta import LentaIE
diff --git a/yt_dlp/extractor/lefigaro.py b/yt_dlp/extractor/lefigaro.py
new file mode 100644
index 000000000..9465095db
--- /dev/null
+++ b/yt_dlp/extractor/lefigaro.py
@@ -0,0 +1,135 @@
+import json
+import math
+
+from .common import InfoExtractor
+from ..utils import (
+    InAdvancePagedList,
+    traverse_obj,
+)
+
+
+class LeFigaroVideoEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.lefigaro\.fr/embed/[^?#]+/(?P<id>[\w-]+)'
+
+    _TESTS = [{
+        'url': 'https://video.lefigaro.fr/embed/figaro/video/les-francais-ne-veulent-ils-plus-travailler-suivez-en-direct-le-club-le-figaro-idees/',
+        'md5': 'e94de44cd80818084352fcf8de1ce82c',
+        'info_dict': {
+            'id': 'g9j7Eovo',
+            'title': 'Les Français ne veulent-ils plus travailler ? Retrouvez Le Club Le Figaro Idées',
+            'description': 'md5:862b8813148ba4bf10763a65a69dfe41',
+            'upload_date': '20230216',
+            'timestamp': 1676581615,
+            'duration': 3076,
+            'thumbnail': r're:^https?://[^?#]+\.(?:jpeg|jpg)',
+            'ext': 'mp4',
+        },
+    }, {
+        'url': 'https://video.lefigaro.fr/embed/figaro/video/intelligence-artificielle-faut-il-sen-mefier/',
+        'md5': '0b3f10332b812034b3a3eda1ef877c5f',
+        'info_dict': {
+            'id': 'LeAgybyc',
+            'title': 'Intelligence artificielle : faut-il s’en méfier ?',
+            'description': 'md5:249d136e3e5934a67c8cb704f8abf4d2',
+            'upload_date': '20230124',
+            'timestamp': 1674584477,
+            'duration': 860,
+            'thumbnail': r're:^https?://[^?#]+\.(?:jpeg|jpg)',
+            'ext': 'mp4',
+        },
+    }]
+
+    _WEBPAGE_TESTS = [{
+        'url': 'https://video.lefigaro.fr/figaro/video/suivez-en-direct-le-club-le-figaro-international-avec-philippe-gelie-9/',
+        'md5': '3972ddf2d5f8b98699f191687258e2f9',
+        'info_dict': {
+            'id': 'QChnbPYA',
+            'title': 'Où en est le couple franco-allemand ? Retrouvez Le Club Le Figaro International',
+            'description': 'md5:6f47235b7e7c93b366fd8ebfa10572ac',
+            'upload_date': '20230123',
+            'timestamp': 1674503575,
+            'duration': 3153,
+            'thumbnail': r're:^https?://[^?#]+\.(?:jpeg|jpg)',
+            'age_limit': 0,
+            'ext': 'mp4',
+        },
+    }, {
+        'url': 'https://video.lefigaro.fr/figaro/video/la-philosophe-nathalie-sarthou-lajus-est-linvitee-du-figaro-live/',
+        'md5': '3ac0a0769546ee6be41ab52caea5d9a9',
+        'info_dict': {
+            'id': 'QJzqoNbf',
+            'title': 'La philosophe Nathalie Sarthou-Lajus est l’invitée du Figaro Live',
+            'description': 'md5:c586793bb72e726c83aa257f99a8c8c4',
+            'upload_date': '20230217',
+            'timestamp': 1676661986,
+            'duration': 1558,
+            'thumbnail': r're:^https?://[^?#]+\.(?:jpeg|jpg)',
+            'age_limit': 0,
+            'ext': 'mp4',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        player_data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['pageData']['playerData']
+
+        return self.url_result(
+            f'jwplatform:{player_data["videoId"]}', title=player_data.get('title'),
+            description=player_data.get('description'), thumbnail=player_data.get('poster'))
+
+
+class LeFigaroVideoSectionIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.lefigaro\.fr/figaro/(?P<id>[\w-]+)/?(?:[#?]|$)'
+
+    _TESTS = [{
+        'url': 'https://video.lefigaro.fr/figaro/le-club-le-figaro-idees/',
+        'info_dict': {
+            'id': 'le-club-le-figaro-idees',
+            'title': 'Le Club Le Figaro Idées',
+        },
+        'playlist_mincount': 14,
+    }, {
+        'url': 'https://video.lefigaro.fr/figaro/factu/',
+        'info_dict': {
+            'id': 'factu',
+            'title': 'Factu',
+        },
+        'playlist_mincount': 519,
+    }]
+
+    _PAGE_SIZE = 20
+
+    def _get_api_response(self, display_id, page_num, note=None):
+        return self._download_json(
+            'https://api-graphql.lefigaro.fr/graphql', display_id, note=note,
+            query={
+                'id': 'flive-website_UpdateListPage_1fb260f996bca2d78960805ac382544186b3225f5bedb43ad08b9b8abef79af6',
+                'variables': json.dumps({
+                    'slug': display_id,
+                    'videosLimit': self._PAGE_SIZE,
+                    'sort': 'DESC',
+                    'order': 'PUBLISHED_AT',
+                    'page': page_num,
+                }).encode(),
+            })
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        initial_response = self._get_api_response(display_id, page_num=1)['data']['playlist']
+
+        def page_func(page_num):
+            api_response = self._get_api_response(display_id, page_num + 1, note=f'Downloading page {page_num + 1}')
+
+            return [self.url_result(
+                video['embedUrl'], LeFigaroVideoEmbedIE, **traverse_obj(video, {
+                    'title': 'name',
+                    'description': 'description',
+                    'thumbnail': 'thumbnailUrl',
+                })) for video in api_response['data']['playlist']['jsonLd'][0]['itemListElement']]
+
+        entries = InAdvancePagedList(
+            page_func, math.ceil(initial_response['videoCount'] / self._PAGE_SIZE), self._PAGE_SIZE)
+
+        return self.playlist_result(entries, playlist_id=display_id, playlist_title=initial_response.get('title'))

From 392389b7df7b818f794b231f14dc396d4875fbad Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 4 Mar 2023 22:40:08 +0530
Subject: [PATCH 136/871] [cleanup] Misc

---
 README.md                            |  5 ++-
 devscripts/make_changelog.py         | 49 ++++++++++++++--------------
 devscripts/update-version.py         |  5 ++-
 devscripts/utils.py                  | 13 +++++++-
 yt_dlp/YoutubeDL.py                  |  2 +-
 yt_dlp/extractor/sportdeutschland.py |  2 +-
 yt_dlp/extractor/youtube.py          |  6 ++--
 yt_dlp/update.py                     |  2 +-
 8 files changed, 46 insertions(+), 38 deletions(-)

diff --git a/README.md b/README.md
index 5788e9c0e..f28bf8e12 100644
--- a/README.md
+++ b/README.md
@@ -192,9 +192,8 @@ ## UPDATE
 <a id="update-channels"/>
 
 There are currently two release channels for binaries, `stable` and `nightly`.
-`stable` releases are what the program will update to by default, and have had many of their changes tested by users of the master branch.
-`nightly` releases are built after each push to the master branch, and will have the most recent fixes and additions, but also have the potential for bugs.
-The latest `nightly` is available as a [pre-release from this repository](https://github.com/yt-dlp/yt-dlp/releases/tag/nightly), and all `nightly` releases are [archived in their own repo](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases).
+`stable` is the default channel, and many of its changes have been tested by users of the nightly channel.
+The `nightly` channel has releases built after each push to the master branch, and will have the most recent fixes and additions, but also have more risk of regressions. They are available in [their own repo](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases).
 
 When using `--update`/`-U`, a release binary will only update to its current channel.
 This release channel can be changed by using the `--update-to` option. `--update-to` can also be used to upgrade or downgrade to specific tags from a channel.
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index 722315333..b159bc1b9 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -1,19 +1,26 @@
 from __future__ import annotations
 
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import enum
 import itertools
 import json
 import logging
 import re
-import subprocess
-import sys
 from collections import defaultdict
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
 
+from devscripts.utils import read_file, run_process, write_file
+
 BASE_URL = 'https://github.com'
 LOCATION_PATH = Path(__file__).parent
+HASH_LENGTH = 7
 
 logger = logging.getLogger(__name__)
 
@@ -82,7 +89,7 @@ def __str__(self):
         result = f'{self.short!r}'
 
         if self.hash:
-            result += f' ({self.hash[:7]})'
+            result += f' ({self.hash[:HASH_LENGTH]})'
 
         if self.authors:
             authors = ', '.join(self.authors)
@@ -208,7 +215,7 @@ def format_single_change(self, info):
 
     def _format_message_link(self, message, hash):
         assert message or hash, 'Improperly defined commit message or override'
-        message = message if message else hash[:7]
+        message = message if message else hash[:HASH_LENGTH]
         return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message
 
     def _format_issues(self, issues):
@@ -242,9 +249,8 @@ class CommitRange:
     FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})')
     UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)')
 
-    def __init__(self, start, end, default_author=None) -> None:
-        self._start = start
-        self._end = end
+    def __init__(self, start, end, default_author=None):
+        self._start, self._end = start, end
         self._commits, self._fixes = self._get_commits_and_fixes(default_author)
         self._commits_added = []
 
@@ -262,14 +268,10 @@ def __contains__(self, commit):
 
         return commit in self._commits
 
-    def _is_ancestor(self, commitish):
-        return bool(subprocess.call(
-            [self.COMMAND, 'merge-base', '--is-ancestor', commitish, self._start]))
-
     def _get_commits_and_fixes(self, default_author):
-        result = subprocess.check_output([
+        result = run_process(
             self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}',
-            f'{self._start}..{self._end}' if self._start else self._end], text=True)
+            f'{self._start}..{self._end}' if self._start else self._end).stdout
 
         commits = {}
         fixes = defaultdict(list)
@@ -301,12 +303,12 @@ def _get_commits_and_fixes(self, default_author):
 
         for commitish, fix_commits in fixes.items():
             if commitish in commits:
-                hashes = ', '.join(commit.hash[:7] for commit in fix_commits)
-                logger.info(f'Found fix(es) for {commitish[:7]}: {hashes}')
+                hashes = ', '.join(commit.hash[:HASH_LENGTH] for commit in fix_commits)
+                logger.info(f'Found fix(es) for {commitish[:HASH_LENGTH]}: {hashes}')
                 for fix_commit in fix_commits:
                     del commits[fix_commit.hash]
             else:
-                logger.debug(f'Commit with fixes not in changes: {commitish[:7]}')
+                logger.debug(f'Commit with fixes not in changes: {commitish[:HASH_LENGTH]}')
 
         return commits, fixes
 
@@ -397,11 +399,10 @@ def groups(self):
 def get_new_contributors(contributors_path, commits):
     contributors = set()
     if contributors_path.exists():
-        with contributors_path.open() as file:
-            for line in filter(None, map(str.strip, file)):
-                author, _, _ = line.partition(' (')
-                authors = author.split('/')
-                contributors.update(map(str.casefold, authors))
+        for line in read_file(contributors_path).splitlines():
+            author, _, _ = line.strip().partition(' (')
+            authors = author.split('/')
+            contributors.update(map(str.casefold, authors))
 
     new_contributors = set()
     for commit in commits:
@@ -453,8 +454,7 @@ def get_new_contributors(contributors_path, commits):
 
     if not args.no_override:
         if args.override_path.exists():
-            with args.override_path.open() as file:
-                overrides = json.load(file)
+            overrides = json.loads(read_file(args.override_path))
             commits.apply_overrides(overrides)
         else:
             logger.warning(f'File {args.override_path.as_posix()} does not exist')
@@ -464,8 +464,7 @@ def get_new_contributors(contributors_path, commits):
     new_contributors = get_new_contributors(args.contributors_path, commits)
     if new_contributors:
         if args.contributors:
-            with args.contributors_path.open('a') as file:
-                file.writelines(f'{contributor}\n' for contributor in new_contributors)
+            write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a')
         logger.info(f'New contributors: {", ".join(new_contributors)}')
 
     print(Changelog(commits.groups(), args.repo))
diff --git a/devscripts/update-version.py b/devscripts/update-version.py
index 60ebcff62..d888be881 100644
--- a/devscripts/update-version.py
+++ b/devscripts/update-version.py
@@ -9,11 +9,10 @@
 
 import argparse
 import contextlib
-import subprocess
 import sys
 from datetime import datetime
 
-from devscripts.utils import read_version, write_file
+from devscripts.utils import read_version, run_process, write_file
 
 
 def get_new_version(version, revision):
@@ -32,7 +31,7 @@ def get_new_version(version, revision):
 
 def get_git_head():
     with contextlib.suppress(Exception):
-        return subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() or None
+        return run_process('git', 'rev-parse', 'HEAD').stdout.strip()
 
 
 VERSION_TEMPLATE = '''\
diff --git a/devscripts/utils.py b/devscripts/utils.py
index b91b8e65a..f75a84da9 100644
--- a/devscripts/utils.py
+++ b/devscripts/utils.py
@@ -1,5 +1,6 @@
 import argparse
 import functools
+import subprocess
 
 
 def read_file(fname):
@@ -12,8 +13,8 @@ def write_file(fname, content, mode='w'):
         return f.write(content)
 
 
-# Get the version without importing the package
 def read_version(fname='yt_dlp/version.py'):
+    """Get the version without importing the package"""
     exec(compile(read_file(fname), fname, 'exec'))
     return locals()['__version__']
 
@@ -33,3 +34,13 @@ def get_filename_args(has_infile=False, default_outfile=None):
 
 def compose_functions(*functions):
     return lambda x: functools.reduce(lambda y, f: f(y), functions, x)
+
+
+def run_process(*args, **kwargs):
+    kwargs.setdefault('text', True)
+    kwargs.setdefault('check', True)
+    kwargs.setdefault('capture_output', True)
+    if kwargs['text']:
+        kwargs.setdefault('encoding', 'utf-8')
+        kwargs.setdefault('errors', 'replace')
+    return subprocess.run(args, **kwargs)
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 5d21b43cf..f701738c9 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3784,7 +3784,7 @@ def get_encoding(stream):
         klass = type(self)
         write_debug(join_nonempty(
             f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
-            __version__ + {'stable': '', 'nightly': '*'}.get(CHANNEL, f' <{CHANNEL}>'),
+            f'{CHANNEL}@{__version__}',
             f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '',
             '' if source == 'unknown' else f'({source})',
             '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}',
diff --git a/yt_dlp/extractor/sportdeutschland.py b/yt_dlp/extractor/sportdeutschland.py
index 6490f4203..30dbcf370 100644
--- a/yt_dlp/extractor/sportdeutschland.py
+++ b/yt_dlp/extractor/sportdeutschland.py
@@ -130,7 +130,7 @@ def _real_extract(self, url):
                                   'channel_url': 'channel_url', 'was_live': 'was_live'}),
             **self._process_video(info['id'], video),
         } for i, video in enumerate(parts, 1)]
-        print(entries[0]['duration'])
+
         return {
             '_type': 'multi_video',
             **info,
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index b8bb980f3..acd4077f4 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3717,10 +3717,10 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
                 'filesize': int_or_none(fmt.get('contentLength')),
                 'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
                 'format_note': join_nonempty(
-                    '%s%s' % (audio_track.get('displayName') or '',
-                              ' (default)' if language_preference > 0 else ''),
+                    join_nonempty(audio_track.get('displayName'),
+                                  language_preference > 0 and ' (default)', delim=''),
                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
-                    'DRC' if fmt.get('isDrc') else None,
+                    fmt.get('isDrc') and 'DRC',
                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index 297539bb6..5a752d716 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -29,13 +29,13 @@
     'stable': 'yt-dlp/yt-dlp',
     'nightly': 'yt-dlp/yt-dlp-nightly-builds',
 }
+REPOSITORY = UPDATE_SOURCES['stable']
 
 _VERSION_RE = re.compile(r'(\d+\.)*\d+')
 
 API_BASE_URL = 'https://api.github.com/repos'
 
 # Backwards compatibility variables for the current channel
-REPOSITORY = UPDATE_SOURCES[CHANNEL]
 API_URL = f'{API_BASE_URL}/{REPOSITORY}/releases'
 
 

From 8729e7b57c0d6e6350a76f82436e05d7b9891188 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@example.com>
Date: Sat, 4 Mar 2023 22:24:51 +0000
Subject: [PATCH 137/871] Release 2023.03.04

Created by: pukkandan

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |  8 ++--
 .../ISSUE_TEMPLATE/2_site_support_request.yml |  8 ++--
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |  8 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |  8 ++--
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |  8 ++--
 .github/ISSUE_TEMPLATE/6_question.yml         |  8 ++--
 CONTRIBUTORS                                  |  3 ++
 Changelog.md                                  | 39 +++++++++++++++++++
 supportedsites.md                             | 12 ++++--
 yt_dlp/version.py                             |  4 +-
 10 files changed, 76 insertions(+), 30 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index c273c952e..cdbb86760 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that a **supported** site is broken
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -64,7 +64,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -72,8 +72,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.03, Current version: 2023.03.03
-        yt-dlp is up to date (2023.03.03)
+        Latest version: 2023.03.04, Current version: 2023.03.04
+        yt-dlp is up to date (2023.03.04)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index bfb362e56..890df48fa 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -76,7 +76,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -84,8 +84,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.03, Current version: 2023.03.03
-        yt-dlp is up to date (2023.03.03)
+        Latest version: 2023.03.04, Current version: 2023.03.04
+        yt-dlp is up to date (2023.03.04)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index 164f7fa9f..ef9bda36a 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -72,7 +72,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -80,8 +80,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.03, Current version: 2023.03.03
-        yt-dlp is up to date (2023.03.03)
+        Latest version: 2023.03.04, Current version: 2023.03.04
+        yt-dlp is up to date (2023.03.04)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index 3cfcb1c03..bf1d97bba 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -57,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -65,8 +65,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.03, Current version: 2023.03.03
-        yt-dlp is up to date (2023.03.03)
+        Latest version: 2023.03.04, Current version: 2023.03.04
+        yt-dlp is up to date (2023.03.04)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index 35204ca57..b17c65658 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -53,7 +53,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -61,7 +61,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.03, Current version: 2023.03.03
-        yt-dlp is up to date (2023.03.03)
+        Latest version: 2023.03.04, Current version: 2023.03.04
+        yt-dlp is up to date (2023.03.04)
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index dc2ccf1d4..c694e5a5a 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.03** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -59,7 +59,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.03 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -67,7 +67,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.03, Current version: 2023.03.03
-        yt-dlp is up to date (2023.03.03)
+        Latest version: 2023.03.04, Current version: 2023.03.04
+        yt-dlp is up to date (2023.03.04)
         <more lines>
       render: shell
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index bd5c6ca97..d6ba617b7 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -406,3 +406,6 @@ rohieb
 sdht0
 seproDev
 Hill-98
+LXYan2333
+mushbite
+venkata-krishnas
diff --git a/Changelog.md b/Changelog.md
index 580ff958b..186998ede 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,45 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.03.04
+
+#### Extractor changes
+- bilibili
+    - [Fix for downloading wrong subtitles](https://github.com/yt-dlp/yt-dlp/commit/8a83baaf218ab89e6e7faa76b7c7be3a2ec19e3a) ([#6358](https://github.com/yt-dlp/yt-dlp/issues/6358)) by [LXYan2333](https://github.com/LXYan2333)
+- ESPNcricinfo
+    - [Handle new URL pattern](https://github.com/yt-dlp/yt-dlp/commit/640c934823fc2d1ec77ec932566078014058635f) ([#6321](https://github.com/yt-dlp/yt-dlp/issues/6321)) by [venkata-krishnas](https://github.com/venkata-krishnas)
+- lefigaro
+    - [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/eb8fd6d044e8926532772b72be0645c6b8ecb3aa) ([#6309](https://github.com/yt-dlp/yt-dlp/issues/6309)) by [elyse0](https://github.com/elyse0)
+- lumni
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/1f8489cccbdc6e96027ef527b88717458f0900e8) ([#6302](https://github.com/yt-dlp/yt-dlp/issues/6302)) by [carusocr](https://github.com/carusocr)
+- Prankcast
+    - [Fix tags](https://github.com/yt-dlp/yt-dlp/commit/ed4cc4ea793314c50ae3f82e98248c1de1c25694) ([#6316](https://github.com/yt-dlp/yt-dlp/issues/6316)) by [columndeeply](https://github.com/columndeeply)
+- rutube
+    - [Extract chapters from description](https://github.com/yt-dlp/yt-dlp/commit/22ccd5420b3eb0782776071f12cccd1fedaa1fd0) ([#6345](https://github.com/yt-dlp/yt-dlp/issues/6345)) by [mushbite](https://github.com/mushbite)
+- SportDeutschland
+    - [Rewrite extractor](https://github.com/yt-dlp/yt-dlp/commit/45db357289b4e1eec09093c8bc5446520378f426) by [pukkandan](https://github.com/pukkandan)
+- telecaribe
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/b40471282286bd2b09c485bf79afd271d229272c) ([#6311](https://github.com/yt-dlp/yt-dlp/issues/6311)) by [elyse0](https://github.com/elyse0)
+- tubetugraz
+    - [Support `--twofactor` (#6424)](https://github.com/yt-dlp/yt-dlp/commit/f44cb4e77bb9be8be291d02ab6f79dc0b4c0d4a1) ([#6427](https://github.com/yt-dlp/yt-dlp/issues/6427)) by [Ferdi265](https://github.com/Ferdi265)
+- tunein
+    - [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/46580ced56c90b559885aded6aa8f46f20a9cdce) ([#6310](https://github.com/yt-dlp/yt-dlp/issues/6310)) by [elyse0](https://github.com/elyse0)
+- twitch
+    - [Update for GraphQL API changes](https://github.com/yt-dlp/yt-dlp/commit/4a6272c6d1bff89969b67cd22b26ebe6d7e72279) ([#6318](https://github.com/yt-dlp/yt-dlp/issues/6318)) by [elyse0](https://github.com/elyse0)
+- twitter
+    - [Fix retweet extraction](https://github.com/yt-dlp/yt-dlp/commit/cf605226521e99c89fc8dff26a319025810e63a0) ([#6422](https://github.com/yt-dlp/yt-dlp/issues/6422)) by [selfisekai](https://github.com/selfisekai)
+- xvideos
+    - quickies: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/283a0b5bc511f3b350eead4488158f50c20ec526) ([#6414](https://github.com/yt-dlp/yt-dlp/issues/6414)) by [Yakabuff](https://github.com/Yakabuff)
+
+#### Misc. changes
+- build
+    - [Fix publishing to PyPI and homebrew](https://github.com/yt-dlp/yt-dlp/commit/55676fe498345a389a2539d8baaba958d6d61c3e) by [bashonly](https://github.com/bashonly)
+    - [Only archive if `vars.ARCHIVE_REPO` is set](https://github.com/yt-dlp/yt-dlp/commit/08ff6d59f97b5f5f0128f6bf6fbef56fd836cc52) by [Grub4K](https://github.com/Grub4K)
+- cleanup
+    - Miscellaneous: [392389b](https://github.com/yt-dlp/yt-dlp/commit/392389b7df7b818f794b231f14dc396d4875fbad) by [pukkandan](https://github.com/pukkandan)
+- devscripts
+    - `make_changelog`: [Stop at `Release ...` commit](https://github.com/yt-dlp/yt-dlp/commit/7accdd9845fe7ce9d0aa5a9d16faaa489c1294eb) by [pukkandan](https://github.com/pukkandan)
+
 ### 2023.03.03
 
 #### Important changes
diff --git a/supportedsites.md b/supportedsites.md
index d7ac6dce5..f5c8c3829 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -663,6 +663,8 @@ # Supported sites
  - **Lecturio**: [*lecturio*](## "netrc machine")
  - **LecturioCourse**: [*lecturio*](## "netrc machine")
  - **LecturioDeCourse**: [*lecturio*](## "netrc machine")
+ - **LeFigaroVideoEmbed**
+ - **LeFigaroVideoSection**
  - **LEGO**
  - **Lemonde**
  - **Lenta**
@@ -696,6 +698,7 @@ # Supported sites
  - **LoveHomePorn**
  - **LRTStream**
  - **LRTVOD**
+ - **Lumni**
  - **lynda**: [*lynda*](## "netrc machine") lynda.com videos
  - **lynda:course**: [*lynda*](## "netrc machine") lynda.com online courses
  - **m6**
@@ -1365,6 +1368,7 @@ # Supported sites
  - **Tele13**
  - **Tele5**
  - **TeleBruxelles**
+ - **TelecaribePlay**
  - **Telecinco**: telecinco.es, cuatro.com and mediaset.es
  - **Telegraaf**
  - **telegram:embed**
@@ -1440,10 +1444,9 @@ # Supported sites
  - **TubiTv**: [*tubitv*](## "netrc machine")
  - **TubiTvShow**
  - **Tumblr**: [*tumblr*](## "netrc machine")
- - **tunein:clip**
- - **tunein:program**
- - **tunein:station**
- - **tunein:topic**
+ - **TuneInPodcast**
+ - **TuneInPodcastEpisode**
+ - **TuneInStation**
  - **TunePk**
  - **Turbo**
  - **tv.dfb.de**
@@ -1695,6 +1698,7 @@ # Supported sites
  - **XTubeUser**: XTube user profile
  - **Xuite**: 隨意窩Xuite影音
  - **XVideos**
+ - **xvideos:quickies**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
  - **yahoo:gyao**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index 7b27fcabb..04bece038 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.03.03'
+__version__ = '2023.03.04'
 
-RELEASE_GIT_HEAD = '93449642815a6973a4b09b289982ca7e1f961b5f'
+RELEASE_GIT_HEAD = '392389b7df7b818f794b231f14dc396d4875fbad'
 
 VARIANT = None
 

From c459d45dd4d417fb80a52e1a04e607776a44baa4 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 5 Mar 2023 12:36:48 -0600
Subject: [PATCH 138/871] [extractor/teamcoco] Fix extractor (#6437)

Closes #6339
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   5 +-
 yt_dlp/extractor/teamcoco.py    | 337 ++++++++++++++++++++------------
 2 files changed, 212 insertions(+), 130 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index cc7490590..545cbe204 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1846,7 +1846,10 @@
     TeacherTubeUserIE,
 )
 from .teachingchannel import TeachingChannelIE
-from .teamcoco import TeamcocoIE
+from .teamcoco import (
+    TeamcocoIE,
+    ConanClassicIE,
+)
 from .teamtreehouse import TeamTreeHouseIE
 from .techtalks import TechTalksIE
 from .ted import (
diff --git a/yt_dlp/extractor/teamcoco.py b/yt_dlp/extractor/teamcoco.py
index a822b676f..d32f81262 100644
--- a/yt_dlp/extractor/teamcoco.py
+++ b/yt_dlp/extractor/teamcoco.py
@@ -1,57 +1,109 @@
 import json
+import re
 
 from .turner import TurnerBaseIE
 from ..utils import (
-    determine_ext,
     ExtractorError,
-    int_or_none,
+    clean_html,
+    determine_ext,
+    make_archive_id,
+    merge_dicts,
     mimetype2ext,
     parse_duration,
-    parse_iso8601,
-    qualities,
+    parse_qs,
+    traverse_obj,
+    unified_timestamp,
+    urljoin,
+    url_or_none,
 )
 
 
-class TeamcocoIE(TurnerBaseIE):
-    _VALID_URL = r'https?://(?:\w+\.)?teamcoco\.com/(?P<id>([^/]+/)*[^/?#]+)'
+class TeamcocoBaseIE(TurnerBaseIE):
+    _QUALITIES = {
+        'low': (480, 272),
+        'sd': (640, 360),
+        'hd': (1280, 720),
+        'uhd': (1920, 1080),
+    }
+
+    def _get_formats_and_subtitles(self, info, video_id):
+        formats, subtitles = [], {}
+
+        for src in traverse_obj(info, ('src', ..., {dict})):
+            format_id = src.get('label')
+            src_url = src.get('src')
+            if re.match(r'https?:/[^/]', src_url):
+                src_url = src_url.replace(':/', '://', 1)
+            ext = determine_ext(src_url, mimetype2ext(src.get('type')))
+
+            if not format_id or not src_url:
+                continue
+            elif format_id == 'hls' or ext == 'm3u8':
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    src_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+
+            elif format_id in self._QUALITIES:
+                if src_url.startswith('/mp4:protected/'):
+                    # TODO: Correct extraction for these files
+                    continue
+                formats.append({
+                    'url': src_url,
+                    'ext': ext,
+                    'format_id': format_id,
+                    'width': self._QUALITIES[format_id][0],
+                    'height': self._QUALITIES[format_id][1],
+                })
+
+        return formats, subtitles
+
+
+class TeamcocoIE(TeamcocoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?teamcoco\.com/(?P<id>([^/]+/)*[^/?#]+)'
     _TESTS = [
         {
             'url': 'http://teamcoco.com/video/mary-kay-remote',
-            'md5': '55d532f81992f5c92046ad02fec34d7d',
             'info_dict': {
                 'id': '80187',
+                'display_id': 'video_mary-kay-remote',
                 'ext': 'mp4',
                 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
-                'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.',
-                'duration': 495.0,
+                'description': 'md5:9fb64e45b5aef6b2af1b67612b36c162',
+                'thumbnail': 'https://teamcoco.com/image/thumb?id=80187',
                 'upload_date': '20140402',
-                'timestamp': 1396407600,
-            }
+                'timestamp': 1396440000,
+            },
+            'params': {
+                'skip_download': 'm3u8',
+            },
         }, {
             'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
-            'md5': 'cde9ba0fa3506f5f017ce11ead928f9a',
             'info_dict': {
                 'id': '19705',
+                'display_id': 'video_louis-ck-interview-george-w-bush',
                 'ext': 'mp4',
-                'description': 'Louis C.K. got starstruck by George W. Bush, so what? Part one.',
                 'title': 'Louis C.K. Interview Pt. 1 11/3/11',
-                'duration': 288,
+                'description': 'Louis C.K. got starstruck by George W. Bush, so what? Part one.',
+                'thumbnail': 'https://teamcoco.com/image/thumb?id=19705',
                 'upload_date': '20111104',
-                'timestamp': 1320405840,
-            }
+                'timestamp': 1320408000,
+            },
+            'params': {
+                'skip_download': 'm3u8',
+            },
         }, {
             'url': 'http://teamcoco.com/video/timothy-olyphant-drinking-whiskey',
             'info_dict': {
                 'id': '88748',
+                'display_id': 'video_timothy-olyphant-drinking-whiskey',
                 'ext': 'mp4',
                 'title': 'Timothy Olyphant Raises A Toast To “Justified”',
                 'description': 'md5:15501f23f020e793aeca761205e42c24',
                 'upload_date': '20150415',
-                'timestamp': 1429088400,
+                'timestamp': 1429099200,
+                'thumbnail': 'https://teamcoco.com/image/thumb?id=88748',
             },
-            'params': {
-                'skip_download': True,  # m3u8 downloads
-            }
         }, {
             'url': 'http://teamcoco.com/video/full-episode-mon-6-1-joel-mchale-jake-tapper-and-musical-guest-courtney-barnett?playlist=x;eyJ0eXBlIjoidGFnIiwiaWQiOjl9',
             'info_dict': {
@@ -60,9 +112,6 @@ class TeamcocoIE(TurnerBaseIE):
                 'title': 'Full Episode - Mon. 6/1 - Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
                 'description': 'Guests: Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
             },
-            'params': {
-                'skip_download': True,  # m3u8 downloads
-            },
             'skip': 'This video is no longer available.',
         }, {
             'url': 'http://teamcoco.com/video/the-conan-audiencey-awards-for-04/25/18',
@@ -76,126 +125,156 @@ class TeamcocoIE(TurnerBaseIE):
         }, {
             'url': 'http://teamcoco.com/israel/conan-hits-the-streets-beaches-of-tel-aviv',
             'only_matching': True,
-        }, {
-            'url': 'https://conan25.teamcoco.com/video/ice-cube-kevin-hart-conan-share-lyft',
-            'only_matching': True,
-        }
+        },
     ]
-    _RECORD_TEMPL = '''id
-        title
-        teaser
-        publishOn
-        thumb {
-          preview
-        }
-        tags {
-          name
-        }
-        duration
-        turnerMediaId
-        turnerMediaAuthToken'''
 
-    def _graphql_call(self, query_template, object_type, object_id):
-        find_object = 'find' + object_type
-        return self._download_json(
-            'https://teamcoco.com/graphql', object_id, data=json.dumps({
-                'query': query_template % (find_object, object_id)
-            }).encode(), headers={
-                'Content-Type': 'application/json',
-            })['data'][find_object]
+    def _real_extract(self, url):
+        display_id = self._match_id(url).replace('/', '_')
+        webpage = self._download_webpage(url, display_id)
+        data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['pageData']
+        info = merge_dicts(*traverse_obj(data, (
+            'blocks', lambda _, v: v['name'] in ('meta-tags', 'video-player', 'video-info'), 'props', {dict})))
+
+        thumbnail = traverse_obj(
+            info, (('image', 'poster'), {lambda x: urljoin('https://teamcoco.com/', x)}), get_all=False)
+        video_id = traverse_obj(parse_qs(thumbnail), ('id', 0)) or display_id
+
+        formats, subtitles = self._get_formats_and_subtitles(info, video_id)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnail': thumbnail,
+            **traverse_obj(info, {
+                'title': 'title',
+                'description': (('descriptionHtml', 'description'), {clean_html}),
+                'timestamp': ('publishedOn', {lambda x: f'{x} 12:00AM'}, {unified_timestamp}),
+            }, get_all=False),
+        }
+
+
+class ConanClassicIE(TeamcocoBaseIE):
+    _VALID_URL = r'https?://(?:(?:www\.)?conanclassic|conan25\.teamcoco)\.com/(?P<id>([^/]+/)*[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://conanclassic.com/video/ice-cube-kevin-hart-conan-share-lyft',
+        'info_dict': {
+            'id': '74709',
+            'ext': 'mp4',
+            'title': 'Ice Cube, Kevin Hart, & Conan Share A Lyft Car',
+            'display_id': 'video/ice-cube-kevin-hart-conan-share-lyft',
+            'description': 'The stars of "Ride Along" teach Conan how to roll around Hollywood.',
+            'thumbnail': 'http://cdn.teamcococdn.com/image/640x360/lyft-5bd75f82b616c.png',
+            'duration': 570.0,
+            'upload_date': '20131211',
+            'timestamp': 1386721620,
+            '_old_archive_ids': ['teamcoco 74709'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://conan25.teamcoco.com/video/ice-cube-kevin-hart-conan-share-lyft',
+        'only_matching': True,
+    }]
+
+    _GRAPHQL_QUERY = '''query find($id: ID!) {
+  findRecord(id: $id) {
+
+... on MetaInterface {
+  id
+  title
+  teaser
+  publishOn
+  slug
+  thumb {
+
+... on FileInterface {
+  id
+  path
+  preview
+  mime
+}
+
+  }
+}
+
+... on Video {
+  videoType
+  duration
+  isLive
+  youtubeId
+  turnerMediaId
+  turnerMediaAuthToken
+  airDate
+}
+
+... on Episode {
+  airDate
+  seasonNumber
+  episodeNumber
+  guestNames
+}
+
+  }
+  findRecordVideoMetadata(id: $id) {
+    turnerMediaId
+    turnerMediaAuthToken
+    duration
+    src
+  }
+}'''
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['pageData']
+        video_id = traverse_obj(
+            data, ('blocks', ..., 'props', 'fieldDefs', lambda _, v: v['name'] == 'incomingVideoId', 'value'),
+            ('blocks', ..., 'props', 'fields', 'incomingVideoRecord', 'id'), get_all=False)
+        if not video_id:
+            self.raise_no_formats('Unable to extract video ID from webpage', expected=True)
 
-        response = self._graphql_call('''{
-  %%s(slug: "%%s") {
-    ... on RecordSlug {
-      record {
-        %s
-      }
-    }
-    ... on PageSlug {
-      child {
-        id
-      }
-    }
-    ... on NotFoundSlug {
-      status
-    }
-  }
-}''' % self._RECORD_TEMPL, 'Slug', display_id)
-        if response.get('status'):
-            raise ExtractorError('This video is no longer available.', expected=True)
+        response = self._download_json(
+            'https://conanclassic.com/api/legacy/graphql', video_id, data=json.dumps({
+                'query': self._GRAPHQL_QUERY,
+                'variables': {'id': video_id},
+            }, separators=(',', ':')).encode(), headers={
+                'Content-Type': 'application/json',
+            })
 
-        child = response.get('child')
-        if child:
-            record = self._graphql_call('''{
-  %%s(id: "%%s") {
-    ... on Video {
-      %s
-    }
-  }
-}''' % self._RECORD_TEMPL, 'Record', child['id'])
-        else:
-            record = response['record']
-        video_id = record['id']
+        info = traverse_obj(response, ('data', 'findRecord', {
+            'title': 'title',
+            'description': 'teaser',
+            'thumbnail': ('thumb', 'preview', {url_or_none}),
+            'duration': ('duration', {parse_duration}),
+            'timestamp': ('publishOn', {unified_timestamp}),
+        }))
 
-        info = {
-            'id': video_id,
-            'display_id': display_id,
-            'title': record['title'],
-            'thumbnail': record.get('thumb', {}).get('preview'),
-            'description': record.get('teaser'),
-            'duration': parse_duration(record.get('duration')),
-            'timestamp': parse_iso8601(record.get('publishOn')),
-        }
-
-        media_id = record.get('turnerMediaId')
+        media_id = traverse_obj(
+            response, ('data', ('findRecord', 'findRecordVideoMetadata'), 'turnerMediaId'), get_all=False)
         if media_id:
+            token = traverse_obj(
+                response, ('data', ('findRecord', 'findRecordVideoMetadata'), 'turnerMediaAuthToken'), get_all=False)
+            if not token:
+                raise ExtractorError('No Turner Media auth token found in API response')
             self._initialize_geo_bypass({
                 'countries': ['US'],
             })
             info.update(self._extract_ngtv_info(media_id, {
-                'accessToken': record['turnerMediaAuthToken'],
+                'accessToken': token,
                 'accessTokenType': 'jws',
             }))
         else:
-            video_sources = self._download_json(
-                'https://teamcoco.com/_truman/d/' + video_id,
-                video_id)['meta']['src']
-            if isinstance(video_sources, dict):
-                video_sources = video_sources.values()
+            formats, subtitles = self._get_formats_and_subtitles(
+                traverse_obj(response, ('data', 'findRecordVideoMetadata')), video_id)
+            info.update({
+                'formats': formats,
+                'subtitles': subtitles,
+            })
 
-            formats = []
-            get_quality = qualities(['low', 'sd', 'hd', 'uhd'])
-            for src in video_sources:
-                if not isinstance(src, dict):
-                    continue
-                src_url = src.get('src')
-                if not src_url:
-                    continue
-                format_id = src.get('label')
-                ext = determine_ext(src_url, mimetype2ext(src.get('type')))
-                if format_id == 'hls' or ext == 'm3u8':
-                    # compat_urllib_parse.urljoin does not work here
-                    if src_url.startswith('/'):
-                        src_url = 'http://ht.cdn.turner.com/tbs/big/teamcoco' + src_url
-                    formats.extend(self._extract_m3u8_formats(
-                        src_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
-                else:
-                    if src_url.startswith('/mp4:protected/'):
-                        # TODO Correct extraction for these files
-                        continue
-                    tbr = int_or_none(self._search_regex(
-                        r'(\d+)k\.mp4', src_url, 'tbr', default=None))
-
-                    formats.append({
-                        'url': src_url,
-                        'ext': ext,
-                        'tbr': tbr,
-                        'format_id': format_id,
-                        'quality': get_quality(format_id),
-                    })
-            info['formats'] = formats
-
-        return info
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            '_old_archive_ids': [make_archive_id('Teamcoco', video_id)],
+            **info,
+        }

From d4e6ef40772e0560a8ed33b844ef7549e86837be Mon Sep 17 00:00:00 2001
From: permunkle <125150380+permunkle@users.noreply.github.com>
Date: Mon, 6 Mar 2023 19:02:03 +0000
Subject: [PATCH 139/871] [extractor/nubilesporn] Add extractor (#6231)

Authored by: permunkle
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/nubilesporn.py | 99 +++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 yt_dlp/extractor/nubilesporn.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 545cbe204..01281b5a1 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1295,6 +1295,7 @@
 from .ntvcojp import NTVCoJpCUIE
 from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
+from .nubilesporn import NubilesPornIE
 from .nytimes import (
     NYTimesIE,
     NYTimesArticleIE,
diff --git a/yt_dlp/extractor/nubilesporn.py b/yt_dlp/extractor/nubilesporn.py
new file mode 100644
index 000000000..d4f1d9d67
--- /dev/null
+++ b/yt_dlp/extractor/nubilesporn.py
@@ -0,0 +1,99 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    float_or_none,
+    format_field,
+    get_element_by_class,
+    get_element_by_id,
+    get_element_html_by_class,
+    get_elements_by_class,
+    int_or_none,
+    try_call,
+    unified_timestamp,
+    urlencode_postdata,
+)
+
+
+class NubilesPornIE(InfoExtractor):
+    _NETRC_MACHINE = 'nubiles-porn'
+    _VALID_URL = r'''(?x)
+        https://members.nubiles-porn.com/video/watch/(?P<id>\d+)
+        (?:/(?P<display_id>[\w\-]+-s(?P<season>\d+)e(?P<episode>\d+)))?
+    '''
+
+    _TESTS = [{
+        'url': 'https://members.nubiles-porn.com/video/watch/165320/trying-to-focus-my-one-track-mind-s3e1',
+        'md5': 'fa7f09da8027c35e4bdf0f94f55eac82',
+        'info_dict': {
+            'id': '165320',
+            'title': 'Trying To Focus My One Track Mind - S3:E1',
+            'ext': 'mp4',
+            'display_id': 'trying-to-focus-my-one-track-mind-s3e1',
+            'thumbnail': 'https://images.nubiles-porn.com/videos/trying_to_focus_my_one_track_mind/samples/cover1280.jpg',
+            'description': 'md5:81f3d4372e0e39bff5c801da277a5141',
+            'timestamp': 1676160000,
+            'upload_date': '20230212',
+            'channel': 'Younger Mommy',
+            'channel_id': '64',
+            'channel_url': 'https://members.nubiles-porn.com/video/website/64',
+            'like_count': int,
+            'average_rating': float,
+            'age_limit': 18,
+            'categories': ['Big Boobs', 'Big Naturals', 'Blowjob', 'Brunette', 'Cowgirl', 'Girl Orgasm', 'Girl-Boy',
+                           'Glasses', 'Hardcore', 'Milf', 'Shaved Pussy', 'Tattoos', 'YoungerMommy.com'],
+            'tags': list,
+            'cast': ['Kenzie Love'],
+            'availability': 'needs_auth',
+            'series': 'Younger Mommy',
+            'series_id': '64',
+            'season': 'Season 3',
+            'season_number': 3,
+            'episode': 'Episode 1',
+            'episode_number': 1
+        }
+    }]
+
+    def _perform_login(self, username, password):
+        login_webpage = self._download_webpage('https://nubiles-porn.com/login', video_id=None)
+        inputs = self._hidden_inputs(login_webpage)
+        inputs.update({'username': username, 'password': password})
+        self._request_webpage('https://nubiles-porn.com/authentication/login', None, data=urlencode_postdata(inputs))
+
+    def _real_extract(self, url):
+        url_match = self._match_valid_url(url)
+        video_id = url_match.group('id')
+        page = self._download_webpage(url, video_id)
+
+        media_entries = self._parse_html5_media_entries(
+            url, get_element_by_class('watch-page-video-wrapper', page), video_id)[0]
+
+        channel_id, channel_name = self._search_regex(
+            r'/video/website/(?P<id>\d+).+>(?P<name>\w+).com', get_element_html_by_class('site-link', page),
+            'channel', fatal=False, group=('id', 'name')) or (None, None)
+        channel_name = re.sub(r'([^A-Z]+)([A-Z]+)', r'\1 \2', channel_name)
+
+        return {
+            'id': video_id,
+            'title': self._search_regex('<h2>([^<]+)</h2>', page, 'title', fatal=False),
+            'formats': media_entries.get('formats'),
+            'display_id': url_match.group('display_id'),
+            'thumbnail': media_entries.get('thumbnail'),
+            'description': clean_html(get_element_html_by_class('content-pane-description', page)),
+            'timestamp': unified_timestamp(get_element_by_class('date', page)),
+            'channel': channel_name,
+            'channel_id': channel_id,
+            'channel_url': format_field(channel_id, None, 'https://members.nubiles-porn.com/video/website/%s'),
+            'like_count': int_or_none(get_element_by_id('likecount', page)),
+            'average_rating': float_or_none(get_element_by_class('score', page)),
+            'age_limit': 18,
+            'categories': try_call(lambda: list(map(clean_html, get_elements_by_class('btn', get_element_by_class('categories', page))))),
+            'tags': try_call(lambda: list(map(clean_html, get_elements_by_class('btn', get_elements_by_class('tags', page)[1])))),
+            'cast': get_elements_by_class('content-pane-performer', page),
+            'availability': 'needs_auth',
+            'series': channel_name,
+            'series_id': channel_id,
+            'season_number': int_or_none(url_match.group('season')),
+            'episode_number': int_or_none(url_match.group('episode'))
+        }

From 3b479100df02e20dd949e046003ae96ddbfced57 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 7 Mar 2023 22:34:07 +0100
Subject: [PATCH 140/871] [utils] `write_string`: Fix noconsole behavior

Ref: https://github.com/pyinstaller/pyinstaller/pull/7217

Authored by: Grub4K
---
 yt_dlp/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index e9b889447..8c2c5593c 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -2057,6 +2057,9 @@ def get_windows_version():
 def write_string(s, out=None, encoding=None):
     assert isinstance(s, str)
     out = out or sys.stderr
+    # `sys.stderr` might be `None` (Ref: https://github.com/pyinstaller/pyinstaller/pull/7217)
+    if not out:
+        return
 
     if compat_os_name == 'nt' and supports_terminal_sequences(out):
         s = re.sub(r'([\r\n]+)', r' \1', s)

From 6f4fc5660f40f3458882a8f51601eae4af7be609 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Wed, 8 Mar 2023 06:37:34 -0600
Subject: [PATCH 141/871] [extractor/chilloutzone] Fix extractor (#6445)

Closes #6029
Authored by: bashonly
---
 yt_dlp/extractor/chilloutzone.py | 128 +++++++++++++++++++------------
 1 file changed, 79 insertions(+), 49 deletions(-)

diff --git a/yt_dlp/extractor/chilloutzone.py b/yt_dlp/extractor/chilloutzone.py
index 1a2f77c4e..ac4252f1b 100644
--- a/yt_dlp/extractor/chilloutzone.py
+++ b/yt_dlp/extractor/chilloutzone.py
@@ -1,93 +1,123 @@
-import json
+import base64
 
 from .common import InfoExtractor
-from .youtube import YoutubeIE
-from ..compat import compat_b64decode
 from ..utils import (
     clean_html,
-    ExtractorError
+    int_or_none,
+    traverse_obj,
 )
 
 
 class ChilloutzoneIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chilloutzone\.net/video/(?P<id>[\w|-]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?chilloutzone\.net/video/(?P<id>[\w-]+)\.html'
     _TESTS = [{
-        'url': 'http://www.chilloutzone.net/video/enemene-meck-alle-katzen-weg.html',
+        'url': 'https://www.chilloutzone.net/video/enemene-meck-alle-katzen-weg.html',
         'md5': 'a76f3457e813ea0037e5244f509e66d1',
         'info_dict': {
             'id': 'enemene-meck-alle-katzen-weg',
             'ext': 'mp4',
             'title': 'Enemene Meck - Alle Katzen weg',
             'description': 'Ist das der Umkehrschluss des Niesenden Panda-Babys?',
+            'duration': 24,
         },
     }, {
         'note': 'Video hosted at YouTube',
-        'url': 'http://www.chilloutzone.net/video/eine-sekunde-bevor.html',
+        'url': 'https://www.chilloutzone.net/video/eine-sekunde-bevor.html',
         'info_dict': {
             'id': '1YVQaAgHyRU',
             'ext': 'mp4',
             'title': '16 Photos Taken 1 Second Before Disaster',
             'description': 'md5:58a8fcf6a459fe0a08f54140f0ad1814',
             'uploader': 'BuzzFeedVideo',
-            'uploader_id': 'BuzzFeedVideo',
+            'uploader_id': '@BuzzFeedVideo',
             'upload_date': '20131105',
+            'availability': 'public',
+            'thumbnail': 'https://i.ytimg.com/vi/1YVQaAgHyRU/maxresdefault.jpg',
+            'tags': 'count:41',
+            'like_count': int,
+            'playable_in_embed': True,
+            'channel_url': 'https://www.youtube.com/channel/UCpko_-a4wgz2u_DgDgd9fqA',
+            'chapters': 'count:6',
+            'live_status': 'not_live',
+            'view_count': int,
+            'categories': ['Entertainment'],
+            'age_limit': 0,
+            'channel_id': 'UCpko_-a4wgz2u_DgDgd9fqA',
+            'duration': 100,
+            'uploader_url': 'http://www.youtube.com/@BuzzFeedVideo',
+            'channel_follower_count': int,
+            'channel': 'BuzzFeedVideo',
         },
     }, {
-        'note': 'Video hosted at Vimeo',
-        'url': 'http://www.chilloutzone.net/video/icon-blending.html',
-        'md5': '2645c678b8dc4fefcc0e1b60db18dac1',
+        'url': 'https://www.chilloutzone.net/video/icon-blending.html',
+        'md5': '2f9d6850ec567b24f0f4fa143b9aa2f9',
         'info_dict': {
-            'id': '85523671',
+            'id': 'LLNkHpSjBfc',
             'ext': 'mp4',
-            'title': 'The Sunday Times - Icons',
-            'description': 're:(?s)^Watch the making of - makingoficons.com.{300,}',
-            'uploader': 'Us',
-            'uploader_id': 'usfilms',
-            'upload_date': '20140131'
+            'title': 'The Sunday Times   Making of Icons',
+            'description': 'md5:b9259fcf63a1669e42001e5db677f02a',
+            'uploader': 'MadFoxUA',
+            'uploader_id': '@MadFoxUA',
+            'upload_date': '20140204',
+            'channel_id': 'UCSZa9Y6-Vl7c11kWMcbAfCw',
+            'channel_url': 'https://www.youtube.com/channel/UCSZa9Y6-Vl7c11kWMcbAfCw',
+            'comment_count': int,
+            'uploader_url': 'http://www.youtube.com/@MadFoxUA',
+            'duration': 66,
+            'live_status': 'not_live',
+            'channel_follower_count': int,
+            'playable_in_embed': True,
+            'view_count': int,
+            'like_count': int,
+            'thumbnail': 'https://i.ytimg.com/vi/LLNkHpSjBfc/maxresdefault.jpg',
+            'categories': ['Comedy'],
+            'availability': 'public',
+            'tags': [],
+            'channel': 'MadFoxUA',
+            'age_limit': 0,
+        },
+    }, {
+        'url': 'https://www.chilloutzone.net/video/ordentlich-abgeschuettelt.html',
+        'info_dict': {
+            'id': 'ordentlich-abgeschuettelt',
+            'ext': 'mp4',
+            'title': 'Ordentlich abgeschüttelt',
+            'description': 'md5:d41541966b75d3d1e8ea77a94ea0d329',
+            'duration': 18,
         },
     }]
 
     def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        b64_data = self._html_search_regex(
+            r'var cozVidData\s*=\s*"([^"]+)"', webpage, 'video data')
+        info = self._parse_json(base64.b64decode(b64_data).decode(), video_id)
 
-        base64_video_info = self._html_search_regex(
-            r'var cozVidData = "(.+?)";', webpage, 'video data')
-        decoded_video_info = compat_b64decode(base64_video_info).decode('utf-8')
-        video_info_dict = json.loads(decoded_video_info)
+        video_url = info.get('mediaUrl')
+        native_platform = info.get('nativePlatform')
 
-        # get video information from dict
-        video_url = video_info_dict['mediaUrl']
-        description = clean_html(video_info_dict.get('description'))
-        title = video_info_dict['title']
-        native_platform = video_info_dict['nativePlatform']
-        native_video_id = video_info_dict['nativeVideoId']
-        source_priority = video_info_dict['sourcePriority']
-
-        # If nativePlatform is None a fallback mechanism is used (i.e. youtube embed)
-        if native_platform is None:
-            youtube_url = YoutubeIE._extract_url(webpage)
-            if youtube_url:
-                return self.url_result(youtube_url, ie=YoutubeIE.ie_key())
-
-        # Non Fallback: Decide to use native source (e.g. youtube or vimeo) or
-        # the own CDN
-        if source_priority == 'native':
+        if native_platform and info.get('sourcePriority') == 'native':
+            native_video_id = info['nativeVideoId']
             if native_platform == 'youtube':
-                return self.url_result(native_video_id, ie='Youtube')
-            if native_platform == 'vimeo':
-                return self.url_result(
-                    'http://vimeo.com/' + native_video_id, ie='Vimeo')
+                return self.url_result(native_video_id, 'Youtube')
+            elif native_platform == 'vimeo':
+                return self.url_result(f'https://vimeo.com/{native_video_id}', 'Vimeo')
 
-        if not video_url:
-            raise ExtractorError('No video found')
+        elif not video_url:
+            # Possibly a standard youtube embed?
+            # TODO: Investigate if site still does this (there are no tests for it)
+            return self.url_result(url, 'Generic')
 
         return {
             'id': video_id,
             'url': video_url,
             'ext': 'mp4',
-            'title': title,
-            'description': description,
+            **traverse_obj(info, {
+                'title': 'title',
+                'description': ('description', {clean_html}),
+                'duration': ('videoLength', {int_or_none}),
+                'width': ('videoWidth', {int_or_none}),
+                'height': ('videoHeight', {int_or_none}),
+            }),
         }

From 01ddec7e661bf90dc4c34e6924eb9d7629886cef Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Wed, 8 Mar 2023 07:10:19 -0600
Subject: [PATCH 142/871] [postprocessor] Fix chapters if duration is not
 extracted (#6037)

Authored by: bashonly
---
 yt_dlp/postprocessor/ffmpeg.py          | 7 +++++++
 yt_dlp/postprocessor/modify_chapters.py | 1 +
 2 files changed, 8 insertions(+)

diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py
index 123a95a3a..0e8f4c70b 100644
--- a/yt_dlp/postprocessor/ffmpeg.py
+++ b/yt_dlp/postprocessor/ffmpeg.py
@@ -302,6 +302,11 @@ def get_stream_number(self, path, keys, value):
             None)
         return num, len(streams)
 
+    def _fixup_chapters(self, info):
+        last_chapter = traverse_obj(info, ('chapters', -1))
+        if last_chapter and not last_chapter.get('end_time'):
+            last_chapter['end_time'] = self._get_real_video_duration(info['filepath'])
+
     def _get_real_video_duration(self, filepath, fatal=True):
         try:
             duration = float_or_none(
@@ -678,6 +683,7 @@ def _options(target_ext):
 
     @PostProcessor._restrict_to(images=False)
     def run(self, info):
+        self._fixup_chapters(info)
         filename, metadata_filename = info['filepath'], None
         files_to_delete, options = [], []
         if self._add_chapters and info.get('chapters'):
@@ -1040,6 +1046,7 @@ def _ffmpeg_args_for_chapter(self, number, chapter, info):
 
     @PostProcessor._restrict_to(images=False)
     def run(self, info):
+        self._fixup_chapters(info)
         chapters = info.get('chapters') or []
         if not chapters:
             self.to_screen('Chapter information is unavailable')
diff --git a/yt_dlp/postprocessor/modify_chapters.py b/yt_dlp/postprocessor/modify_chapters.py
index a745b4524..f5219868c 100644
--- a/yt_dlp/postprocessor/modify_chapters.py
+++ b/yt_dlp/postprocessor/modify_chapters.py
@@ -23,6 +23,7 @@ def __init__(self, downloader, remove_chapters_patterns=None, remove_sponsor_seg
 
     @PostProcessor._restrict_to(images=False)
     def run(self, info):
+        self._fixup_chapters(info)
         # Chapters must be preserved intact when downloading multiple formats of the same video.
         chapters, sponsor_chapters = self._mark_chapters_to_remove(
             copy.deepcopy(info.get('chapters')) or [],

From 9b7a48abd1b187eae1e3f6c9839c47d43ccec00b Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 8 Mar 2023 21:49:24 +0100
Subject: [PATCH 143/871] [cookies] Defer extraction of v11 key from keyring

Closes #6082

Authored by: Grub4K
---
 yt_dlp/cookies.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index 8ca7cea2c..4cafb522e 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -20,6 +20,7 @@
     aes_gcm_decrypt_and_verify_bytes,
     unpad_pkcs7,
 )
+from .compat import functools
 from .dependencies import (
     _SECRETSTORAGE_UNAVAILABLE_REASON,
     secretstorage,
@@ -383,9 +384,14 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
     def __init__(self, browser_keyring_name, logger, *, keyring=None):
         self._logger = logger
         self._v10_key = self.derive_key(b'peanuts')
-        password = _get_linux_keyring_password(browser_keyring_name, keyring, logger)
-        self._v11_key = None if password is None else self.derive_key(password)
         self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0}
+        self._browser_keyring_name = browser_keyring_name
+        self._keyring = keyring
+
+    @functools.cached_property
+    def _v11_key(self):
+        password = _get_linux_keyring_password(self._browser_keyring_name, self._keyring, self._logger)
+        return None if password is None else self.derive_key(password)
 
     @staticmethod
     def derive_key(password):

From 2d5cae9636714ff922d28c548c349d5f2b48f317 Mon Sep 17 00:00:00 2001
From: D0LLYNH0 <67797325+D0LLYNH0@users.noreply.github.com>
Date: Thu, 9 Mar 2023 04:18:14 -0300
Subject: [PATCH 144/871] [extractor/iq] Set more language codes (#6476)

Authored by: D0LLYNH0
---
 yt_dlp/extractor/iqiyi.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/iqiyi.py b/yt_dlp/extractor/iqiyi.py
index 4443b1991..ebf49e835 100644
--- a/yt_dlp/extractor/iqiyi.py
+++ b/yt_dlp/extractor/iqiyi.py
@@ -440,12 +440,14 @@ class IqIE(InfoExtractor):
         '1': 'zh_CN',
         '2': 'zh_TW',
         '3': 'en',
-        '4': 'kor',
+        '4': 'ko',
+        '5': 'ja',
         '18': 'th',
         '21': 'my',
         '23': 'vi',
         '24': 'id',
         '26': 'es',
+        '27': 'pt',
         '28': 'ar',
     }
 

From 3588be59cee429a0ab5c4ceb2f162298bb44147d Mon Sep 17 00:00:00 2001
From: Daniel Vogt <daniel-vogt@mail.de>
Date: Thu, 9 Mar 2023 17:21:39 +0100
Subject: [PATCH 145/871] [extractor/opencast] Add ltitools to `_VALID_URL`
 (#6371)

Authored by: C0D3D3V
---
 yt_dlp/extractor/opencast.py | 37 +++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/yt_dlp/extractor/opencast.py b/yt_dlp/extractor/opencast.py
index fa46757f7..235ca341c 100644
--- a/yt_dlp/extractor/opencast.py
+++ b/yt_dlp/extractor/opencast.py
@@ -105,10 +105,9 @@ def _parse_mediapackage(self, video):
 
 
 class OpencastIE(OpencastBaseIE):
-    _VALID_URL = r'''(?x)
-                    https?://(?P<host>%s)/paella/ui/watch.html\?.*?
-                    id=(?P<id>%s)
-                    ''' % (OpencastBaseIE._INSTANCES_RE, OpencastBaseIE._UUID_RE)
+    _VALID_URL = rf'''(?x)
+        https?://(?P<host>{OpencastBaseIE._INSTANCES_RE})/paella/ui/watch\.html\?
+        (?:[^#]+&)?id=(?P<id>{OpencastBaseIE._UUID_RE})'''
 
     _API_BASE = 'https://%s/search/episode.json?id=%s'
 
@@ -123,6 +122,9 @@ class OpencastIE(OpencastBaseIE):
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'timestamp': 1606208400,
                 'upload_date': '20201124',
+                'season_id': 'cf68a4a1-36b1-4a53-a6ba-61af5705a0d0',
+                'series': 'Kryptographie - WiSe 15/16',
+                'creator': 'Alexander May',
             },
         }
     ]
@@ -134,10 +136,11 @@ def _real_extract(self, url):
 
 
 class OpencastPlaylistIE(OpencastBaseIE):
-    _VALID_URL = r'''(?x)
-                            https?://(?P<host>%s)/engage/ui/index.html\?.*?
-                            epFrom=(?P<id>%s)
-                    ''' % (OpencastBaseIE._INSTANCES_RE, OpencastBaseIE._UUID_RE)
+    _VALID_URL = rf'''(?x)
+        https?://(?P<host>{OpencastBaseIE._INSTANCES_RE})(?:
+            /engage/ui/index\.html\?(?:[^#]+&)?epFrom=|
+            /ltitools/index\.html\?(?:[^#]+&)?series=
+        )(?P<id>{OpencastBaseIE._UUID_RE})'''
 
     _API_BASE = 'https://%s/search/episode.json?sid=%s'
 
@@ -148,15 +151,23 @@ class OpencastPlaylistIE(OpencastBaseIE):
                 'id': 'cf68a4a1-36b1-4a53-a6ba-61af5705a0d0',
                 'title': 'Kryptographie - WiSe 15/16',
             },
-            'playlist_mincount': 28,
+            'playlist_mincount': 29,
         },
         {
-            'url': 'https://oc-video.ruhr-uni-bochum.de/engage/ui/index.html?e=1&p=1&epFrom=b1a54262-3684-403f-9731-8e77c3766f9a',
+            'url': 'https://oc-video1.ruhr-uni-bochum.de/ltitools/index.html?subtool=series&series=cf68a4a1-36b1-4a53-a6ba-61af5705a0d0&lng=de',
             'info_dict': {
-                'id': 'b1a54262-3684-403f-9731-8e77c3766f9a',
-                'title': 'inSTUDIES-Social movements and prefigurative politics in a global perspective',
+                'id': 'cf68a4a1-36b1-4a53-a6ba-61af5705a0d0',
+                'title': 'Kryptographie - WiSe 15/16',
             },
-            'playlist_mincount': 6,
+            'playlist_mincount': 29,
+        },
+        {
+            'url': 'https://electures.uni-muenster.de/engage/ui/index.html?e=1&p=1&epFrom=39391d10-a711-4d23-b21d-afd2ed7d758c',
+            'info_dict': {
+                'id': '39391d10-a711-4d23-b21d-afd2ed7d758c',
+                'title': '021670 Theologische Themen bei Hans Blumenberg WiSe 2017/18',
+            },
+            'playlist_mincount': 13,
         },
     ]
 

From 66aeaac9aa30b5959069ba84e53a5508232deb38 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Mar 2023 21:57:44 +0530
Subject: [PATCH 146/871] [downloader/curl] Fix progress reporting

Bug in 8c53322cda75394a8d551dde20b2529ee5ad6e89
Closes #6490
---
 yt_dlp/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index 5f54017a8..ee130c827 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -176,7 +176,7 @@ def _call_downloader(self, tmpfilename, info_dict):
         return 0
 
     def _call_process(self, cmd, info_dict):
-        return Popen.run(cmd, text=True, stderr=subprocess.PIPE)
+        return Popen.run(cmd, text=True, stderr=subprocess.PIPE if self._CAPTURE_STDERR else None)
 
 
 class CurlFD(ExternalFD):

From c9abebb851e6188cb34b9eb744c1863dd46af919 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Mar 2023 22:09:23 +0530
Subject: [PATCH 147/871] [extractor/youtube] Bypass throttling for `-f17`

and related cleanup

Thanks @AudricV for the finding
---
 yt_dlp/extractor/youtube.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index acd4077f4..6e6abd65b 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3745,13 +3745,11 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             if mime_mobj:
                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
                 dct.update(parse_codecs(mime_mobj.group(2)))
-            no_audio = dct.get('acodec') == 'none'
-            no_video = dct.get('vcodec') == 'none'
-            if no_audio:
-                dct['vbr'] = tbr
-            if no_video:
-                dct['abr'] = tbr
-            if no_audio or no_video:
+
+            single_stream = 'none' in (dct.get('acodec'), dct.get('vcodec'))
+            if single_stream and dct.get('ext'):
+                dct['container'] = dct['ext'] + '_dash'
+            if single_stream or itag == '17':
                 CHUNK_SIZE = 10 << 20
                 dct.update({
                     'protocol': 'http_dash_segments',
@@ -3760,13 +3758,10 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
                             'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
                         })
                     } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
-                } if dct['filesize'] else {
-                    'downloader_options': {'http_chunk_size': CHUNK_SIZE}  # No longer useful?
+                } if itag != '17' and dct['filesize'] else {
+                    'downloader_options': {'http_chunk_size': CHUNK_SIZE}
                 })
 
-                if dct.get('ext'):
-                    dct['container'] = dct['ext'] + '_dash'
-
             if itag:
                 itags[itag].add(('https', dct.get('language')))
                 stream_ids.append(stream_id)

From 0551511b45f7847f40e4314aa9e624e80d086539 Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Fri, 10 Mar 2023 01:12:38 -0600
Subject: [PATCH 148/871] [extractor/twitch] Fix `is_live` (#6500)

Closes #6494
Authored by: elyse0
---
 yt_dlp/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index efc7db2c9..6321297bb 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -456,7 +456,7 @@ def _extract_info_gql(self, info, item_id):
         thumbnail = url_or_none(info.get('previewThumbnailURL'))
         is_live = None
         if thumbnail:
-            if thumbnail.endswith('/404_processing_{width}x{height}.png'):
+            if re.findall(r'/404_processing_[^.?#]+\.png', thumbnail):
                 is_live, thumbnail = True, None
             else:
                 is_live = False

From 871c907454693940cb56906ed9ea49fcb7154829 Mon Sep 17 00:00:00 2001
From: makeworld <25111343+makeworld-the-better-one@users.noreply.github.com>
Date: Fri, 10 Mar 2023 02:53:19 -0500
Subject: [PATCH 149/871] [extractor/cbc:gem] Update `_VALID_URL` (#6499)

Authored by: makeworld-the-better-one
Closes #6395
---
 yt_dlp/extractor/cbc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index 210f5f8ee..eadb3f8c0 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -202,7 +202,7 @@ def _real_extract(self, url):
 
 class CBCGemIE(InfoExtractor):
     IE_NAME = 'gem.cbc.ca'
-    _VALID_URL = r'https?://gem\.cbc\.ca/media/(?P<id>[0-9a-z-]+/s[0-9]+[a-z][0-9]+)'
+    _VALID_URL = r'https?://gem\.cbc\.ca/(?:media/)?(?P<id>[0-9a-z-]+/s[0-9]+[a-z][0-9]+)'
     _TESTS = [{
         # This is a normal, public, TV show video
         'url': 'https://gem.cbc.ca/media/schitts-creek/s06e01',
@@ -245,6 +245,9 @@ class CBCGemIE(InfoExtractor):
         },
         'params': {'format': 'bv'},
         'skip': 'Geo-restricted to Canada',
+    }, {
+        'url': 'https://gem.cbc.ca/nadiyas-family-favourites/s01e01',
+        'only_matching': True,
     }]
 
     _GEO_COUNTRIES = ['CA']

From ab1de9cb1e39cf421c2b7dc6756c6ff1955bb313 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 10 Mar 2023 14:12:08 +0530
Subject: [PATCH 150/871] Support loading info.json with a list at it's root

---
 yt_dlp/YoutubeDL.py | 23 ++++++++++++-----------
 yt_dlp/__init__.py  |  2 ++
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index f701738c9..a7dced8e8 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3376,18 +3376,19 @@ def download_with_info_file(self, info_filename):
                 [info_filename], mode='r',
                 openhook=fileinput.hook_encoded('utf-8'))) as f:
             # FileInput doesn't have a read method, we can't call json.load
-            info = self.sanitize_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True))
-        try:
-            self.__download_wrapper(self.process_ie_result)(info, download=True)
-        except (DownloadError, EntryNotInPlaylist, ReExtractInfo) as e:
-            if not isinstance(e, EntryNotInPlaylist):
-                self.to_stderr('\r')
-            webpage_url = info.get('webpage_url')
-            if webpage_url is not None:
+            infos = [self.sanitize_info(info, self.params.get('clean_infojson', True))
+                     for info in variadic(json.loads('\n'.join(f)))]
+        for info in infos:
+            try:
+                self.__download_wrapper(self.process_ie_result)(info, download=True)
+            except (DownloadError, EntryNotInPlaylist, ReExtractInfo) as e:
+                if not isinstance(e, EntryNotInPlaylist):
+                    self.to_stderr('\r')
+                webpage_url = info.get('webpage_url')
+                if webpage_url is None:
+                    raise
                 self.report_warning(f'The info failed to download: {e}; trying with URL {webpage_url}')
-                return self.download([webpage_url])
-            else:
-                raise
+                self.download([webpage_url])
         return self._download_retcode
 
     @staticmethod
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 9ef31601c..bdac1212c 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -952,6 +952,8 @@ def _real_main(argv=None):
         parser.destroy()
         try:
             if opts.load_info_filename is not None:
+                if all_urls:
+                    ydl.report_warning('URLs are ignored due to --load-info-json')
                 return ydl.download_with_info_file(expand_path(opts.load_info_filename))
             else:
                 return ydl.download(all_urls)

From e6ab678e36c40ded0aae305bbb866cdab554d417 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 10 Mar 2023 17:27:43 +0530
Subject: [PATCH 151/871] [extractor/hidive] Fix login

Fixes https://github.com/yt-dlp/yt-dlp/issues/6493#issuecomment-1462906556
---
 yt_dlp/extractor/hidive.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/hidive.py b/yt_dlp/extractor/hidive.py
index 8a8749859..df6868df6 100644
--- a/yt_dlp/extractor/hidive.py
+++ b/yt_dlp/extractor/hidive.py
@@ -47,15 +47,16 @@ def _perform_login(self, username, password):
         login_webpage = self._download_webpage(
             self._LOGIN_URL, None, 'Logging in', data=urlencode_postdata(data))
         # If the user has multiple profiles on their account, select one. For now pick the first profile.
-        profile_id = self._search_regex(r'<button [^>]+?data-profile-id="(\w+)"', login_webpage, 'profile_id')
+        profile_id = self._search_regex(
+            r'<button [^>]+?data-profile-id="(\w+)"', login_webpage, 'profile id', default=None)
         if profile_id is None:
             return  # If only one profile, Hidive auto-selects it
-        profile_id_hash = self._search_regex(r'\<button [^>]+?data-hash="(\w+)"', login_webpage, 'profile_id_hash')
         self._request_webpage(
             'https://www.hidive.com/ajax/chooseprofile', None,
             data=urlencode_postdata({
                 'profileId': profile_id,
-                'hash': profile_id_hash,
+                'hash': self._search_regex(
+                    r'\<button [^>]+?data-hash="(\w+)"', login_webpage, 'profile id hash'),
                 'returnUrl': '/dashboard'
             }))
 

From 89dbf0848370deaa55af88c3593a2a264124caf5 Mon Sep 17 00:00:00 2001
From: Daniel Vogt <c0d3d3v@mag-keinen-spam.de>
Date: Sat, 11 Mar 2023 16:10:32 +0100
Subject: [PATCH 152/871] [extractor/opencast] Fix format bug (#6512)

Authored by: C0D3D3V
---
 yt_dlp/extractor/opencast.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/opencast.py b/yt_dlp/extractor/opencast.py
index 235ca341c..1fafd9afb 100644
--- a/yt_dlp/extractor/opencast.py
+++ b/yt_dlp/extractor/opencast.py
@@ -55,9 +55,9 @@ def _parse_mediapackage(self, video):
             transport = track.get('transport')
 
             if transport == 'DASH' or ext == 'mpd':
-                formats.extend(self._extract_mpd_formats_and_subtitles(href, video_id, mpd_id='dash', fatal=False))
+                formats.extend(self._extract_mpd_formats(href, video_id, mpd_id='dash', fatal=False))
             elif transport == 'HLS' or ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats_and_subtitles(
+                formats.extend(self._extract_m3u8_formats(
                     href, video_id, m3u8_id='hls', entry_protocol='m3u8_native', fatal=False))
             elif transport == 'HDS' or ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(href, video_id, f4m_id='hds', fatal=False))

From 7a6c8a0807941dd24fbf0d6172e811884f98e027 Mon Sep 17 00:00:00 2001
From: vampirefrog <vampirefrog@users.noreply.github.com>
Date: Sat, 11 Mar 2023 18:52:36 +0200
Subject: [PATCH 153/871] [extractor/rokfin] Re-construct manifest url (#6507)

Authored by: vampirefrog
---
 yt_dlp/extractor/rokfin.py | 50 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/rokfin.py b/yt_dlp/extractor/rokfin.py
index ade3cd0a4..0e40eb32a 100644
--- a/yt_dlp/extractor/rokfin.py
+++ b/yt_dlp/extractor/rokfin.py
@@ -45,6 +45,7 @@ class RokfinIE(InfoExtractor):
             'live_status': 'not_live',
             'dislike_count': int,
             'like_count': int,
+            'duration': 213,
         }
     }, {
         'url': 'https://rokfin.com/post/223/Julian-Assange-Arrested-Streaming-In-Real-Time',
@@ -72,7 +73,7 @@ class RokfinIE(InfoExtractor):
             'title': '"It\'s A Crazy Mess" Regional Director Blows Whistle On Pfizer\'s Vaccine Trial Data',
             'thumbnail': r're:https://img\.production\.rokfin\.com/.+',
             'description': 'md5:324ce2d3e3b62e659506409e458b9d8e',
-            'channel': 'Ryan Cristián',
+            'channel': 'TLAVagabond',
             'channel_id': 53856,
             'channel_url': 'https://rokfin.com/TLAVagabond',
             'availability': 'public',
@@ -86,6 +87,47 @@ class RokfinIE(InfoExtractor):
             'dislike_count': int,
             'like_count': int,
             'tags': ['FreeThinkingMedia^'],
+            'duration': None,
+        }
+    }, {
+        'url': 'https://rokfin.com/post/126703/Brave-New-World--Aldous-Huxley-DEEPDIVE--Chpts-13--Quite-Frankly--Jay-Dyer',
+        'info_dict': {
+            'id': 'post/126703',
+            'ext': 'mp4',
+            'title': 'Brave New World - Aldous Huxley DEEPDIVE!  (Chpts 1-3) - Quite Frankly & Jay Dyer',
+            'thumbnail': r're:https://img\.production\.rokfin\.com/.+',
+            'channel': 'Jay Dyer',
+            'channel_id': 186881,
+            'channel_url': 'https://rokfin.com/jaydyer',
+            'availability': 'premium_only',
+            'live_status': 'not_live',
+            'dislike_count': int,
+            'like_count': int,
+            'timestamp': 1678213357,
+            'upload_date': '20230307',
+            'tags': ['FreeThinkingMedia^', 'OpenMind^'],
+            'description': 'md5:cb04e32e68326c9b2b251b297bacff35',
+            'duration': 3100,
+        }
+    }, {
+        'url': 'https://rokfin.com/stream/31332/The-Grayzone-live-on-Nordstream-blame-game',
+        'info_dict': {
+            'id': 'stream/31332',
+            'ext': 'mp4',
+            'title': 'The Grayzone live on Nordstream blame game',
+            'thumbnail': r're:https://image\.v\.rokfin\.com/.+',
+            'channel': 'Max Blumenthal',
+            'channel_id': 248902,
+            'channel_url': 'https://rokfin.com/MaxBlumenthal',
+            'availability': 'premium_only',
+            'live_status': 'was_live',
+            'dislike_count': int,
+            'like_count': int,
+            'timestamp': 1678475166,
+            'release_timestamp': 1678475166.0,
+            'release_date': '20230310',
+            'upload_date': '20230310',
+            'tags': ['FreeThinkingMedia^'],
         }
     }]
 
@@ -100,6 +142,12 @@ def _real_extract(self, url):
                        else 'not_live')
 
         video_url = traverse_obj(metadata, 'url', ('content', 'contentUrl'), expected_type=url_or_none)
+        if video_url in (None, 'fake.m3u8'):
+            video_url = format_field(self._search_regex(
+                r'https?://[^/]+/([^/]+)/storyboard.vtt',
+                traverse_obj(metadata, 'timelineUrl', ('content', 'timelineUrl'), expected_type=url_or_none),
+                video_id, default=None), None, 'https://stream.v.rokfin.com/%s.m3u8')
+
         formats, subtitles = [{'url': video_url}] if video_url else [], {}
         if determine_ext(video_url) == 'm3u8':
             formats, subtitles = self._extract_m3u8_formats_and_subtitles(

From c795c39f27244cbce846067891827e4847036441 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Sun, 12 Mar 2023 02:03:23 +0900
Subject: [PATCH 154/871] [extractor/youtube] Add client name to `format_note`
 when `-v` (#6254)

Authored by: Lesmiscore, pukkandan
---
 yt_dlp/extractor/youtube.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 6e6abd65b..79174b882 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -66,6 +66,8 @@
     variadic,
 )
 
+
+STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client'
 # any clients starting with _ cannot be explicitly requested by the user
 INNERTUBE_CLIENTS = {
     'web': {
@@ -248,6 +250,11 @@ def _split_innertube_client(client_name):
     return client_name, base, variant[0] if variant else None
 
 
+def short_client_name(client_name):
+    main, *parts = _split_innertube_client(client_name)[0].replace('embedscreen', 'e_s').split('_')
+    return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
+
+
 def build_innertube_clients():
     THIRD_PARTY = {
         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
@@ -3594,6 +3601,12 @@ def append_client(*client_names):
                     self.report_warning(
                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
                 else:
+                    # Save client name for introspection later
+                    name = short_client_name(client)
+                    sd = traverse_obj(pr, ('streamingData', {dict})) or {}
+                    sd[STREAMING_DATA_CLIENT_NAME] = name
+                    for f in traverse_obj(sd, (('formats', 'adaptiveFormats'), ..., {dict})):
+                        f[STREAMING_DATA_CLIENT_NAME] = name
                     prs.append(pr)
 
             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
@@ -3712,6 +3725,8 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             if is_damaged:
                 self.report_warning(
                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
+
+            client_name = fmt.get(STREAMING_DATA_CLIENT_NAME)
             dct = {
                 'asr': int_or_none(fmt.get('audioSampleRate')),
                 'filesize': int_or_none(fmt.get('contentLength')),
@@ -3723,7 +3738,9 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
                     fmt.get('isDrc') and 'DRC',
                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
-                    throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
+                    throttled and 'THROTTLED', is_damaged and 'DAMAGED',
+                    self.get_param('verbose') and client_name,
+                    delim=', '),
                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
                 'fps': int_or_none(fmt.get('fps')) or None,
@@ -3784,7 +3801,7 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
             skip_manifests.add('dash')
 
-        def process_manifest_format(f, proto, itag):
+        def process_manifest_format(f, proto, client_name, itag):
             key = (proto, f.get('language'))
             if key in itags[itag]:
                 return False
@@ -3798,17 +3815,21 @@ def process_manifest_format(f, proto, itag):
             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
             if f['quality'] == -1 and f.get('height'):
                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
+            if self.get_param('verbose'):
+                f['format_note'] = join_nonempty(f.get('format_note'), client_name, delim=', ')
             return True
 
         subtitles = {}
         for sd in streaming_data:
+            client_name = sd.get(STREAMING_DATA_CLIENT_NAME)
+
             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
             if hls_manifest_url:
                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
                 subtitles = self._merge_subtitles(subs, subtitles)
                 for f in fmts:
-                    if process_manifest_format(f, 'hls', self._search_regex(
+                    if process_manifest_format(f, 'hls', client_name, self._search_regex(
                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
                         yield f
 
@@ -3817,7 +3838,7 @@ def process_manifest_format(f, proto, itag):
                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
                 for f in formats:
-                    if process_manifest_format(f, 'dash', f['format_id']):
+                    if process_manifest_format(f, 'dash', client_name, f['format_id']):
                         f['filesize'] = int_or_none(self._search_regex(
                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
                         if needs_live_processing:

From 86cb922118b236306310a72657f70426c20e28bb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Mar 2023 23:13:02 +0530
Subject: [PATCH 155/871] [extractor/youtube] Add extractor-arg
 `include_duplicate_formats`

---
 README.md                   |  1 +
 yt_dlp/extractor/youtube.py | 38 +++++++++++++++++++++----------------
 2 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index f28bf8e12..de83e421f 100644
--- a/README.md
+++ b/README.md
@@ -1787,6 +1787,7 @@ #### youtube
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
 * `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all`
     * E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total
+* `include_duplicate_formats`: Extract formats with identical content but different URLs or protocol. This is useful if some of the formats are unavailable or throttled.
 * `include_incomplete_formats`: Extract formats that cannot be downloaded completely (live dash and post-live m3u8)
 * `innertube_host`: Innertube API host to use for all API requests; e.g. `studio.youtube.com`, `youtubei.googleapis.com`. Note that cookies exported from one subdomain will not work on others
 * `innertube_key`: Innertube API key to use for all API requests
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 79174b882..48f822e44 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3640,6 +3640,7 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
         ])
         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
+        all_formats = self._configuration_arg('include_duplicate_formats')
 
         for fmt in streaming_formats:
             if fmt.get('targetDurationSec'):
@@ -3648,8 +3649,9 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             itag = str_or_none(fmt.get('itag'))
             audio_track = fmt.get('audioTrack') or {}
             stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
-            if stream_id in stream_ids:
-                continue
+            if not all_formats:
+                if stream_id in stream_ids:
+                    continue
 
             quality = fmt.get('quality')
             height = int_or_none(fmt.get('height'))
@@ -3739,7 +3741,7 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
                     throttled and 'THROTTLED', is_damaged and 'DAMAGED',
-                    self.get_param('verbose') and client_name,
+                    (self.get_param('verbose') or all_formats) and client_name,
                     delim=', '),
                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
@@ -3762,26 +3764,28 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             if mime_mobj:
                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
                 dct.update(parse_codecs(mime_mobj.group(2)))
-
+            if itag:
+                itags[itag].add(('https', dct.get('language')))
+                stream_ids.append(stream_id)
             single_stream = 'none' in (dct.get('acodec'), dct.get('vcodec'))
             if single_stream and dct.get('ext'):
                 dct['container'] = dct['ext'] + '_dash'
-            if single_stream or itag == '17':
-                CHUNK_SIZE = 10 << 20
-                dct.update({
+
+            CHUNK_SIZE = 10 << 20
+            if dct['filesize']:
+                yield {
+                    **dct,
+                    'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
                     'protocol': 'http_dash_segments',
                     'fragments': [{
                         'url': update_url_query(dct['url'], {
                             'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
                         })
                     } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
-                } if itag != '17' and dct['filesize'] else {
-                    'downloader_options': {'http_chunk_size': CHUNK_SIZE}
-                })
-
-            if itag:
-                itags[itag].add(('https', dct.get('language')))
-                stream_ids.append(stream_id)
+                }
+                if not all_formats:
+                    continue
+            dct['downloader_options'] = {'http_chunk_size': CHUNK_SIZE}
             yield dct
 
         needs_live_processing = self._needs_live_processing(live_status, duration)
@@ -3803,11 +3807,13 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
 
         def process_manifest_format(f, proto, client_name, itag):
             key = (proto, f.get('language'))
-            if key in itags[itag]:
+            if not all_formats and key in itags[itag]:
                 return False
             itags[itag].add(key)
 
-            if any(p != proto for p, _ in itags[itag]):
+            if itag and all_formats:
+                f['format_id'] = f'{itag}-{proto}'
+            elif any(p != proto for p, _ in itags[itag]):
                 f['format_id'] = f'{itag}-{proto}'
             elif itag:
                 f['format_id'] = itag

From 2a23d92d9ec44a0168079e38bcf3d383e5c4c7bb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 11 Mar 2023 22:46:47 +0530
Subject: [PATCH 156/871] [extractor/youtube] Construct fragment list lazily

Building fragment list for all formats take significant time for large videos
---
 yt_dlp/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 48f822e44..4165d795c 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3777,11 +3777,11 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
                     **dct,
                     'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
                     'protocol': 'http_dash_segments',
-                    'fragments': [{
+                    'fragments': LazyList({
                         'url': update_url_query(dct['url'], {
                             'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
                         })
-                    } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
+                    } for range_start in range(0, dct['filesize'], CHUNK_SIZE))
                 }
                 if not all_formats:
                     continue

From e389d172b6f42e4f332ae679dc48543fb7b9b61d Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 12 Mar 2023 14:46:09 +0530
Subject: [PATCH 157/871] Fix 2a23d92d9ec44a0168079e38bcf3d383e5c4c7bb

Closes #6517
---
 yt_dlp/extractor/youtube.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 4165d795c..d7cd0dc62 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3630,6 +3630,7 @@ def _needs_live_processing(self, live_status, duration):
             return live_status
 
     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
+        CHUNK_SIZE = 10 << 20
         itags, stream_ids = collections.defaultdict(set), []
         itag_qualities, res_qualities = {}, {0: None}
         q = qualities([
@@ -3642,6 +3643,13 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
         all_formats = self._configuration_arg('include_duplicate_formats')
 
+        def build_fragments(f):
+            return LazyList({
+                'url': update_url_query(f['url'], {
+                    'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, f["filesize"])}'
+                })
+            } for range_start in range(0, f['filesize'], CHUNK_SIZE))
+
         for fmt in streaming_formats:
             if fmt.get('targetDurationSec'):
                 continue
@@ -3771,17 +3779,12 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             if single_stream and dct.get('ext'):
                 dct['container'] = dct['ext'] + '_dash'
 
-            CHUNK_SIZE = 10 << 20
             if dct['filesize']:
                 yield {
                     **dct,
                     'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
                     'protocol': 'http_dash_segments',
-                    'fragments': LazyList({
-                        'url': update_url_query(dct['url'], {
-                            'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
-                        })
-                    } for range_start in range(0, dct['filesize'], CHUNK_SIZE))
+                    'fragments': build_fragments(dct),
                 }
                 if not all_formats:
                     continue

From 0181b9a1b31db3fde943f7cd3fe9662f23bff292 Mon Sep 17 00:00:00 2001
From: Ha Tien Loi <loiht.b17vt220@stu.ptit.edu.vn>
Date: Sun, 12 Mar 2023 23:34:22 +0700
Subject: [PATCH 158/871] [extractor/thesun] Update `_VALID_URL` (#6522)

Authored by: hatienl0i261299
Closes #6479
---
 yt_dlp/extractor/thesun.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/thesun.py b/yt_dlp/extractor/thesun.py
index ba5848283..5edcf1cc1 100644
--- a/yt_dlp/extractor/thesun.py
+++ b/yt_dlp/extractor/thesun.py
@@ -5,15 +5,22 @@
 
 
 class TheSunIE(InfoExtractor):
-    _VALID_URL = r'https://(?:www\.)?thesun\.co\.uk/[^/]+/(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?the-?sun(\.co\.uk|\.com)/[^/]+/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'https://www.thesun.co.uk/tvandshowbiz/2261604/orlando-bloom-and-katy-perry-post-adorable-instagram-video-together-celebrating-thanksgiving-after-split-rumours/',
         'info_dict': {
             'id': '2261604',
             'title': 'md5:cba22f48bad9218b64d5bbe0e16afddf',
         },
         'playlist_count': 2,
-    }
+    }, {
+        'url': 'https://www.the-sun.com/entertainment/7611415/1000lb-sisters-fans-rip-amy-dangerous-health-decision/',
+        'info_dict': {
+            'id': '7611415',
+            'title': 'md5:e0b9b976f79dc770e5c80f22f40bb844',
+        },
+        'playlist_count': 1,
+    }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):

From 026435714cb7c39613a0d7d2acd15d3823b78d94 Mon Sep 17 00:00:00 2001
From: Ha Tien Loi <loiht.b17vt220@stu.ptit.edu.vn>
Date: Mon, 13 Mar 2023 00:20:40 +0700
Subject: [PATCH 159/871] [extractor/LastFM] Rewrite playlist extraction
 (#6379)

Authored by: hatienl0i261299, pukkandan
Closes #5975
---
 yt_dlp/extractor/lastfm.py | 43 ++++++++++++++++++++------------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/yt_dlp/extractor/lastfm.py b/yt_dlp/extractor/lastfm.py
index f14198cfd..67103352e 100644
--- a/yt_dlp/extractor/lastfm.py
+++ b/yt_dlp/extractor/lastfm.py
@@ -1,33 +1,24 @@
+import itertools
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none, format_field
+from ..utils import int_or_none, parse_qs, traverse_obj
 
 
 class LastFMPlaylistBaseIE(InfoExtractor):
     def _entries(self, url, playlist_id):
-        webpage = self._download_webpage(url, playlist_id)
-        start_page_number = int_or_none(self._search_regex(
-            r'\bpage=(\d+)', url, 'page', default=None)) or 1
-        last_page_number = int_or_none(self._search_regex(
-            r'>(\d+)</a>[^<]*</li>[^<]*<li[^>]+class="pagination-next', webpage, 'last_page', default=None))
-
-        for page_number in range(start_page_number, (last_page_number or start_page_number) + 1):
+        single_page = traverse_obj(parse_qs(url), ('page', -1, {int_or_none}))
+        for page in itertools.count(single_page or 1):
             webpage = self._download_webpage(
-                url, playlist_id,
-                note='Downloading page %d%s' % (page_number, format_field(last_page_number, None, ' of %d')),
-                query={'page': page_number})
-            page_entries = [
-                self.url_result(player_url, 'Youtube')
-                for player_url in set(re.findall(r'data-youtube-url="([^"]+)"', webpage))
-            ]
-
-            for e in page_entries:
-                yield e
+                url, playlist_id, f'Downloading page {page}', query={'page': page})
+            videos = re.findall(r'data-youtube-url="([^"]+)"', webpage)
+            yield from videos
+            if single_page or not videos:
+                return
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-        return self.playlist_result(self._entries(url, playlist_id), playlist_id)
+        return self.playlist_from_matches(self._entries(url, playlist_id), playlist_id, ie='Youtube')
 
 
 class LastFMPlaylistIE(LastFMPlaylistBaseIE):
@@ -37,7 +28,7 @@ class LastFMPlaylistIE(LastFMPlaylistBaseIE):
         'info_dict': {
             'id': 'Oasis',
         },
-        'playlist_count': 11,
+        'playlist_mincount': 11,
     }, {
         'url': 'https://www.last.fm/music/Oasis',
         'only_matching': True,
@@ -73,6 +64,18 @@ class LastFMUserIE(LastFMPlaylistBaseIE):
             'id': '12319471',
         },
         'playlist_count': 30,
+    }, {
+        'url': 'https://www.last.fm/user/naamloos1/playlists/12543760',
+        'info_dict': {
+            'id': '12543760',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'https://www.last.fm/user/naamloos1/playlists/12543760?page=3',
+        'info_dict': {
+            'id': '12543760',
+        },
+        'playlist_count': 32,
     }]
 
 

From 1e3c2b6ec28d7ab5e31341fa93c47b65be4fbff4 Mon Sep 17 00:00:00 2001
From: Joshua Lochner <admin@xenova.com>
Date: Sun, 12 Mar 2023 19:38:27 +0200
Subject: [PATCH 160/871] [extractor/medaltv] Fix clips (#6502)

Closes #6489
Authored by: xenova
---
 yt_dlp/extractor/medaltv.py | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/yt_dlp/extractor/medaltv.py b/yt_dlp/extractor/medaltv.py
index 82be823b8..9e57ee21a 100644
--- a/yt_dlp/extractor/medaltv.py
+++ b/yt_dlp/extractor/medaltv.py
@@ -8,12 +8,12 @@
     float_or_none,
     int_or_none,
     str_or_none,
-    traverse_obj,
+    traverse_obj
 )
 
 
 class MedalTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?medal\.tv/(?P<path>games/[^/?#&]+/clips)/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?medal\.tv/games/[^/?#&]+/clips/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://medal.tv/games/valorant/clips/jTBFnLKdLy15K',
         'md5': '6930f8972914b6b9fdc2bb3918098ba0',
@@ -80,25 +80,14 @@ class MedalTVIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        path = self._match_valid_url(url).group('path')
 
         webpage = self._download_webpage(url, video_id)
 
-        next_data = self._search_json(
-            '<script[^>]*__NEXT_DATA__[^>]*>', webpage,
+        hydration_data = self._search_json(
+            r'<script[^>]*>[^<]*\bhydrationData\s*=', webpage,
             'next data', video_id, end_pattern='</script>', fatal=False)
 
-        build_id = next_data.get('buildId')
-        if not build_id:
-            raise ExtractorError(
-                'Could not find build ID.', video_id=video_id)
-
-        locale = next_data.get('locale', 'en')
-
-        api_response = self._download_json(
-            f'https://medal.tv/_next/data/{build_id}/{locale}/{path}/{video_id}.json', video_id)
-
-        clip = traverse_obj(api_response, ('pageProps', 'clip')) or {}
+        clip = traverse_obj(hydration_data, ('clips', ...), get_all=False)
         if not clip:
             raise ExtractorError(
                 'Could not find video information.', video_id=video_id)
@@ -152,7 +141,7 @@ def add_item(container, item_url, height, id_key='format_id', item_id=None):
 
         # Necessary because the id of the author is not known in advance.
         # Won't raise an issue if no profile can be found as this is optional.
-        author = traverse_obj(api_response, ('pageProps', 'profile')) or {}
+        author = traverse_obj(hydration_data, ('profiles', ...), get_all=False) or {}
         author_id = str_or_none(author.get('userId'))
         author_url = format_field(author_id, None, 'https://medal.tv/users/%s')
 

From 80ea6d3dea8483cddd39fc89b5ee1fc06670c33c Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Sun, 12 Mar 2023 14:02:17 -0400
Subject: [PATCH 161/871] [extractor/Parler] Rewrite extractor (#6446)

Authored by: JChris246
Closes #6068
---
 yt_dlp/extractor/parler.py | 94 +++++++++++++++-----------------------
 1 file changed, 37 insertions(+), 57 deletions(-)

diff --git a/yt_dlp/extractor/parler.py b/yt_dlp/extractor/parler.py
index 68a60bc84..2af805e7f 100644
--- a/yt_dlp/extractor/parler.py
+++ b/yt_dlp/extractor/parler.py
@@ -1,13 +1,14 @@
+import functools
+
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..utils import (
     clean_html,
-    format_field,
     int_or_none,
     strip_or_none,
     traverse_obj,
     unified_timestamp,
-    urlencode_postdata,
+    urljoin,
 )
 
 
@@ -24,7 +25,7 @@ class ParlerIE(InfoExtractor):
                 'thumbnail': 'https://bl-images.parler.com/videos/6ce7cdf3-a27a-4d72-bf9c-d3e17ce39a66/thumbnail.jpeg',
                 'title': 'Parler video #df79fdba-07cc-48fe-b085-3293897520d7',
                 'description': 'md5:6f220bde2df4a97cbb89ac11f1fd8197',
-                'timestamp': 1659744000,
+                'timestamp': 1659785481,
                 'upload_date': '20220806',
                 'uploader': 'Tulsi Gabbard',
                 'uploader_id': 'TulsiGabbard',
@@ -34,78 +35,57 @@ class ParlerIE(InfoExtractor):
                 'repost_count': int,
             },
         },
-        {
-            'url': 'https://parler.com/feed/a7406eb4-91e5-4793-b5e3-ade57a24e287',
-            'md5': '11687e2f5bb353682cee338d181422ed',
-            'info_dict': {
-                'id': 'a7406eb4-91e5-4793-b5e3-ade57a24e287',
-                'ext': 'mp4',
-                'thumbnail': 'https://bl-images.parler.com/videos/317827a8-1e48-4cbc-981f-7dd17d4c1183/thumbnail.jpeg',
-                'title': 'Parler video #a7406eb4-91e5-4793-b5e3-ade57a24e287',
-                'description': 'This man should run for office',
-                'timestamp': 1659657600,
-                'upload_date': '20220805',
-                'uploader': 'Benny Johnson',
-                'uploader_id': 'BennyJohnson',
-                'uploader_url': 'https://parler.com/BennyJohnson',
-                'view_count': int,
-                'comment_count': int,
-                'repost_count': int,
-            },
-        },
         {
             'url': 'https://parler.com/feed/f23b85c1-6558-470f-b9ff-02c145f28da5',
             'md5': 'eaba1ff4a10fe281f5ce74e930ab2cb4',
             'info_dict': {
                 'id': 'r5vkSaz8PxQ',
                 'ext': 'mp4',
-                'thumbnail': 'https://i.ytimg.com/vi_webp/r5vkSaz8PxQ/maxresdefault.webp',
-                'title': 'Tom MacDonald Names Reaction',
-                'description': 'md5:33c21f0d35ae6dc2edf3007d6696baea',
-                'upload_date': '20220716',
-                'duration': 1267,
-                'uploader': 'Mahesh Chookolingo',
-                'uploader_id': 'maheshchookolingo',
-                'uploader_url': 'http://www.youtube.com/user/maheshchookolingo',
-                'channel': 'Mahesh Chookolingo',
-                'channel_id': 'UCox6YeMSY1PQInbCtTaZj_w',
-                'channel_url': 'https://www.youtube.com/channel/UCox6YeMSY1PQInbCtTaZj_w',
-                'categories': ['Entertainment'],
-                'tags': list,
-                'availability': 'public',
                 'live_status': 'not_live',
-                'view_count': int,
                 'comment_count': int,
+                'duration': 1267,
                 'like_count': int,
                 'channel_follower_count': int,
-                'age_limit': 0,
+                'channel_id': 'UCox6YeMSY1PQInbCtTaZj_w',
+                'upload_date': '20220716',
+                'thumbnail': 'https://i.ytimg.com/vi/r5vkSaz8PxQ/maxresdefault.jpg',
+                'tags': 'count:17',
+                'availability': 'public',
+                'categories': ['Entertainment'],
                 'playable_in_embed': True,
+                'channel': 'Who Knows What! With Mahesh & Friends',
+                'title': 'Tom MacDonald Names Reaction',
+                'uploader': 'Who Knows What! With Mahesh & Friends',
+                'uploader_id': '@maheshchookolingo',
+                'age_limit': 0,
+                'description': 'md5:33c21f0d35ae6dc2edf3007d6696baea',
+                'channel_url': 'https://www.youtube.com/channel/UCox6YeMSY1PQInbCtTaZj_w',
+                'view_count': int,
+                'uploader_url': 'http://www.youtube.com/@maheshchookolingo',
             },
         },
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        data = self._download_json(
-            'https://parler.com/open-api/ParleyDetailEndpoint.php', video_id,
-            data=urlencode_postdata({'uuid': video_id}))['data'][0]
-        primary = data['primary']
-
-        embed = self._parse_json(primary.get('V2LINKLONG') or '', video_id, fatal=False)
-        if embed:
-            return self.url_result(embed[0], YoutubeIE)
+        data = self._download_json(f'https://api.parler.com/v0/public/parleys/{video_id}',
+                                   video_id)['data']
+        if data.get('link'):
+            return self.url_result(data['link'], YoutubeIE)
 
         return {
             'id': video_id,
-            'url': traverse_obj(primary, ('video_data', 'videoSrc')),
-            'thumbnail': traverse_obj(primary, ('video_data', 'thumbnailUrl')),
-            'title': '',
-            'description': strip_or_none(clean_html(primary.get('full_body'))) or None,
-            'timestamp': unified_timestamp(primary.get('date_created')),
-            'uploader': strip_or_none(primary.get('name')),
-            'uploader_id': strip_or_none(primary.get('username')),
-            'uploader_url': format_field(strip_or_none(primary.get('username')), None, 'https://parler.com/%s'),
-            'view_count': int_or_none(primary.get('view_count')),
-            'comment_count': int_or_none(traverse_obj(data, ('engagement', 'commentCount'))),
-            'repost_count': int_or_none(traverse_obj(data, ('engagement', 'echoCount'))),
+            'title': strip_or_none(data.get('title')) or '',
+            **traverse_obj(data, {
+                'url': ('video', 'videoSrc'),
+                'thumbnail': ('video', 'thumbnailUrl'),
+                'description': ('body', {clean_html}),
+                'timestamp': ('date_created', {unified_timestamp}),
+                'uploader': ('user', 'name', {strip_or_none}),
+                'uploader_id': ('user', 'username', {str}),
+                'uploader_url': ('user', 'username', {functools.partial(urljoin, 'https://parler.com/')}),
+                'view_count': ('views', {int_or_none}),
+                'comment_count': ('total_comments', {int_or_none}),
+                'repost_count': ('echos', {int_or_none}),
+            })
         }

From cf9fd52fabe71d6e7c30d3ea525029ffa561fc9c Mon Sep 17 00:00:00 2001
From: Chris Caruso <carusochrisr@gmail.com>
Date: Sun, 12 Mar 2023 11:07:34 -0700
Subject: [PATCH 162/871] [extractor/jwplatform] Update `_extract_embed_urls`
 (#6383)

Authored by: carusocr
---
 yt_dlp/extractor/jwplatform.py | 37 ++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/extractor/jwplatform.py b/yt_dlp/extractor/jwplatform.py
index c94968943..bc47aa6d3 100644
--- a/yt_dlp/extractor/jwplatform.py
+++ b/yt_dlp/extractor/jwplatform.py
@@ -8,14 +8,16 @@ class JWPlatformIE(InfoExtractor):
     _VALID_URL = r'(?:https?://(?:content\.jwplatform|cdn\.jwplayer)\.com/(?:(?:feed|player|thumb|preview|manifest)s|jw6|v2/media)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
     _TESTS = [{
         'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
-        'md5': 'fa8899fa601eb7c83a64e9d568bdf325',
+        'md5': '3aa16e4f6860e6e78b7df5829519aed3',
         'info_dict': {
             'id': 'nPripu9l',
-            'ext': 'mov',
+            'ext': 'mp4',
             'title': 'Big Buck Bunny Trailer',
             'description': 'Big Buck Bunny is a short animated film by the Blender Institute. It is made using free and open source software.',
             'upload_date': '20081127',
             'timestamp': 1227796140,
+            'duration': 32.0,
+            'thumbnail': 'https://cdn.jwplayer.com/v2/media/nPripu9l/poster.jpg?width=720',
         }
     }, {
         'url': 'https://cdn.jwplayer.com/players/nPripu9l-ALJ3XQCI.js',
@@ -37,18 +39,31 @@ class JWPlatformIE(InfoExtractor):
         },
     }, {
         # Player url not surrounded by quotes
-        'url': 'https://www.deutsche-kinemathek.de/en/online/streaming/darling-berlin',
+        'url': 'https://www.deutsche-kinemathek.de/en/online/streaming/school-trip',
         'info_dict': {
-            'id': 'R10NQdhY',
-            'title': 'Playgirl',
+            'id': 'jUxh5uin',
+            'title': 'Klassenfahrt',
             'ext': 'mp4',
-            'upload_date': '20220624',
-            'thumbnail': 'https://cdn.jwplayer.com/v2/media/R10NQdhY/poster.jpg?width=720',
-            'timestamp': 1656064800,
-            'description': 'BRD 1966, Will Tremper',
-            'duration': 5146.0,
+            'upload_date': '20230109',
+            'thumbnail': 'https://cdn.jwplayer.com/v2/media/jUxh5uin/poster.jpg?width=720',
+            'timestamp': 1673270298,
+            'description': '',
+            'duration': 5193.0,
         },
         'params': {'allowed_extractors': ['generic', 'jwplatform']},
+    }, {
+        # iframe src attribute includes backslash before URL string
+        'url': 'https://www.elespectador.com/colombia/video-asi-se-evito-la-fuga-de-john-poulos-presunto-feminicida-de-valentina-trespalacios-explicacion',
+        'info_dict': {
+            'id': 'QD3gsexj',
+            'title': 'Así se evitó la fuga de John Poulos, presunto feminicida de Valentina Trespalacios',
+            'ext': 'mp4',
+            'upload_date': '20230127',
+            'thumbnail': 'https://cdn.jwplayer.com/v2/media/QD3gsexj/poster.jpg?width=720',
+            'timestamp': 1674862986,
+            'description': 'md5:128fd74591c4e1fc2da598c5cb6f5ce4',
+            'duration': 263.0,
+        },
     }]
 
     @classmethod
@@ -57,7 +72,7 @@ def _extract_embed_urls(cls, url, webpage):
             # <input value=URL> is used by hyland.com
             # if we find <iframe>, dont look for <input>
             ret = re.findall(
-                r'<%s[^>]+?%s=["\']?((?:https?:)?//(?:content\.jwplatform|cdn\.jwplayer)\.com/players/[a-zA-Z0-9]{8})' % (tag, key),
+                r'<%s[^>]+?%s=\\?["\']?((?:https?:)?//(?:content\.jwplatform|cdn\.jwplayer)\.com/players/[a-zA-Z0-9]{8})' % (tag, key),
                 webpage)
             if ret:
                 return ret

From cbfe2e5cbe0f4649a91e323a82b8f5f774f36662 Mon Sep 17 00:00:00 2001
From: unbeatable-101 <daviswill048@icloud.com>
Date: Sun, 12 Mar 2023 18:25:05 -0500
Subject: [PATCH 163/871] [extractor/nebula] Add `beta.nebula.tv` (#6516)

Authored by: unbeatable-101
---
 yt_dlp/extractor/nebula.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py
index 81e2f56e6..5c1b7c712 100644
--- a/yt_dlp/extractor/nebula.py
+++ b/yt_dlp/extractor/nebula.py
@@ -5,7 +5,7 @@
 from .common import InfoExtractor
 from ..utils import ExtractorError, parse_iso8601
 
-_BASE_URL_RE = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)'
+_BASE_URL_RE = r'https?://(?:www\.|beta\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)'
 
 
 class NebulaBaseIE(InfoExtractor):
@@ -183,6 +183,10 @@ class NebulaIE(NebulaBaseIE):
             'url': 'https://watchnebula.com/videos/money-episode-1-the-draw',
             'only_matching': True,
         },
+        {
+            'url': 'https://beta.nebula.tv/videos/money-episode-1-the-draw',
+            'only_matching': True,
+        },
     ]
 
     def _fetch_video_metadata(self, slug):

From 98ac902c4979e4529b166e873473bef42baa2e3e Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 13 Mar 2023 05:19:13 +0530
Subject: [PATCH 164/871] [dependencies/Cryptodome] Fix `__bool__`

Bug in 65f6e807804d2af5e00f2aecd72bfc43af19324a
---
 yt_dlp/dependencies/Cryptodome.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/dependencies/Cryptodome.py b/yt_dlp/dependencies/Cryptodome.py
index 74ab6575c..2cfa4c952 100644
--- a/yt_dlp/dependencies/Cryptodome.py
+++ b/yt_dlp/dependencies/Cryptodome.py
@@ -1,4 +1,4 @@
-import types
+from ..compat.compat_utils import passthrough_module
 
 try:
     import Cryptodome as _parent
@@ -6,9 +6,11 @@
     try:
         import Crypto as _parent
     except (ImportError, SyntaxError):  # Old Crypto gives SyntaxError in newer Python
-        _parent = types.ModuleType('no_Cryptodome')
+        _parent = passthrough_module(__name__, 'no_Cryptodome')
         __bool__ = lambda: False
 
+del passthrough_module
+
 __version__ = ''
 AES = PKCS1_v1_5 = Blowfish = PKCS1_OAEP = SHA1 = CMAC = RSA = None
 try:

From 607510b9f2f67bfe7d33d74031a5c1fe22a24862 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Mon, 13 Mar 2023 01:43:37 +0000
Subject: [PATCH 165/871] [extractor/youtube] Handle incomplete initial data
 from watch page (#6510)

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index d7cd0dc62..b024d18b7 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4254,12 +4254,15 @@ def process_language(container, base_url, lang_code, sub_name, query):
         initial_data = None
         if webpage:
             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
+            if not traverse_obj(initial_data, 'contents'):
+                self.report_warning('Incomplete data received in embedded initial data; re-fetching using API.')
+                initial_data = None
         if not initial_data:
             query = {'videoId': video_id}
             query.update(self._get_checkok_params())
             initial_data = self._extract_response(
                 item_id=video_id, ep='next', fatal=False,
-                ytcfg=master_ytcfg, query=query,
+                ytcfg=master_ytcfg, query=query, check_get_keys='contents',
                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
                 note='Downloading initial data API JSON')
 

From 427a8fafbb0e18c28d0ed7960be838d7b26b88d3 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 15 Mar 2023 04:49:22 +0530
Subject: [PATCH 166/871] [build] Pin `pyinstaller` version for MacOS

Workaround for #6541
---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 93668a7bf..aa11c6194 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -192,7 +192,7 @@ jobs:
       - name: Install Requirements
         run: |
           brew install coreutils
-          /usr/bin/python3 -m pip install -U --user pip Pyinstaller -r requirements.txt
+          /usr/bin/python3 -m pip install -U --user pip Pyinstaller==5.8 -r requirements.txt
 
       - name: Prepare
         run: |

From 071670cbeaa01ddf2cc20a95ae6da25f8f086431 Mon Sep 17 00:00:00 2001
From: Nicholas Defranco <39540565+nick-cd@users.noreply.github.com>
Date: Tue, 14 Mar 2023 19:21:14 -0400
Subject: [PATCH 167/871] [extractor/youtube] Fix parsing `comment_count`
 (#6523)

Closes #5849
Authored by: nick-cd
---
 yt_dlp/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index b024d18b7..ca56f112b 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4268,11 +4268,11 @@ def process_language(container, base_url, lang_code, sub_name, query):
 
         info['comment_count'] = traverse_obj(initial_data, (
             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
-            'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
+            'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount'
         ), (
             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
-            'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
-        ), expected_type=int_or_none, get_all=False)
+            'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo'
+        ), expected_type=self._get_count, get_all=False)
 
         try:  # This will error if there is no livechat
             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']

From 03025b6e105139d01cd415ddc51fd692957fd2ba Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 16 Mar 2023 14:53:18 -0500
Subject: [PATCH 168/871] [extractor/mediastream] Improve `WinSports` and embed
 extraction (#6426)

Closes #6419, Closes #6527
Authored by: bashonly
---
 yt_dlp/extractor/mediastream.py | 102 +++++++++++++++++++++-----------
 1 file changed, 66 insertions(+), 36 deletions(-)

diff --git a/yt_dlp/extractor/mediastream.py b/yt_dlp/extractor/mediastream.py
index e8d427a31..cef769f29 100644
--- a/yt_dlp/extractor/mediastream.py
+++ b/yt_dlp/extractor/mediastream.py
@@ -2,16 +2,44 @@
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     remove_end,
-    str_or_none,
-    strip_or_none,
     traverse_obj,
     urljoin,
 )
 
 
-class MediaStreamIE(InfoExtractor):
-    _VALID_URL = r'https?://mdstrm.com/(?:embed|live-stream)/(?P<id>\w+)'
+class MediaStreamBaseIE(InfoExtractor):
+    _EMBED_BASE_URL = 'https://mdstrm.com/embed'
+    _BASE_URL_RE = r'https?://mdstrm\.com/(?:embed|live-stream)'
+
+    def _extract_mediastream_urls(self, webpage):
+        yield from traverse_obj(list(self._yield_json_ld(webpage, None)), (
+            lambda _, v: v['@type'] == 'VideoObject', ('embedUrl', 'contentUrl'),
+            {lambda x: x if re.match(rf'{self._BASE_URL_RE}/\w+', x) else None}))
+
+        for mobj in re.finditer(r'<script[^>]+>[^>]*playerMdStream\.mdstreamVideo\(\s*[\'"](?P<video_id>\w+)', webpage):
+            yield f'{self._EMBED_BASE_URL}/{mobj.group("video_id")}'
+
+        yield from re.findall(
+            rf'<iframe[^>]+\bsrc="({self._BASE_URL_RE}/\w+)', webpage)
+
+        for mobj in re.finditer(
+            r'''(?x)
+                <(?:div|ps-mediastream)[^>]+
+                (class="[^"]*MediaStreamVideoPlayer)[^"]*"[^>]+
+                data-video-id="(?P<video_id>\w+)"
+                (?:\s*data-video-type="(?P<video_type>[^"]+))?
+                (?:[^>]*>\s*<div[^>]+\1[^"]*"[^>]+data-mediastream=["\'][^>]+
+                    https://mdstrm\.com/(?P<live>live-stream))?
+                ''', webpage):
+
+            video_type = 'live-stream' if mobj.group('video_type') == 'live' or mobj.group('live') else 'embed'
+            yield f'https://mdstrm.com/{video_type}/{mobj.group("video_id")}'
+
+
+class MediaStreamIE(MediaStreamBaseIE):
+    _VALID_URL = MediaStreamBaseIE._BASE_URL_RE + r'/(?P<id>\w+)'
 
     _TESTS = [{
         'url': 'https://mdstrm.com/embed/6318e3f1d1d316083ae48831',
@@ -23,6 +51,7 @@ class MediaStreamIE(InfoExtractor):
             'thumbnail': r're:^https?://[^?#]+6318e3f1d1d316083ae48831',
             'ext': 'mp4',
         },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     _WEBPAGE_TESTS = [{
@@ -35,9 +64,7 @@ class MediaStreamIE(InfoExtractor):
             'ext': 'mp4',
             'live_status': 'is_live',
         },
-        'params': {
-            'skip_download': 'Livestream'
-        },
+        'params': {'skip_download': 'Livestream'},
     }, {
         'url': 'https://www.multimedios.com/television/clases-de-llaves-y-castigos-quien-sabe-mas',
         'md5': 'de31f0b1ecc321fb35bf22d58734ea40',
@@ -48,6 +75,7 @@ class MediaStreamIE(InfoExtractor):
             'thumbnail': 're:^https?://[^?#]+63731bab8ec9b308a2c9ed28',
             'ext': 'mp4',
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://www.americatv.com.pe/videos/esto-es-guerra/facundo-gonzalez-sufrio-fuerte-golpe-durante-competencia-frente-hugo-garcia-eeg-noticia-139120',
         'info_dict': {
@@ -57,6 +85,7 @@ class MediaStreamIE(InfoExtractor):
             'thumbnail': 're:^https?://[^?#]+63756df1c638b008a5659dec',
             'ext': 'mp4',
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://www.americatv.com.pe/videos/al-fondo-hay-sitio/nuevas-lomas-town-bernardo-mata-se-enfrento-sujeto-luchar-amor-macarena-noticia-139083',
         'info_dict': {
@@ -66,26 +95,12 @@ class MediaStreamIE(InfoExtractor):
             'thumbnail': 're:^https?://[^?#]+637307669609130f74cd3a6e',
             'ext': 'mp4',
         },
+        'params': {'skip_download': 'm3u8'},
     }]
 
-    @classmethod
-    def _extract_embed_urls(cls, url, webpage):
-        for mobj in re.finditer(r'<script[^>]+>[^>]*playerMdStream.mdstreamVideo\(\s*[\'"](?P<video_id>\w+)', webpage):
-            yield f'https://mdstrm.com/embed/{mobj.group("video_id")}'
-
-        yield from re.findall(
-            r'<iframe[^>]src\s*=\s*"(https://mdstrm.com/[\w-]+/\w+)', webpage)
-
-        for mobj in re.finditer(
-            r'''(?x)
-                <(?:div|ps-mediastream)[^>]+
-                class\s*=\s*"[^"]*MediaStreamVideoPlayer[^"]*"[^>]+
-                data-video-id\s*=\s*"(?P<video_id>\w+)\s*"
-                (?:\s*data-video-type\s*=\s*"(?P<video_type>[^"]+))?
-                ''', webpage):
-
-            video_type = 'live-stream' if mobj.group('video_type') == 'live' else 'embed'
-            yield f'https://mdstrm.com/{video_type}/{mobj.group("video_id")}'
+    def _extract_from_webpage(self, url, webpage):
+        for embed_url in self._extract_mediastream_urls(webpage):
+            yield self.url_result(embed_url, MediaStreamIE, None)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -94,7 +109,7 @@ def _real_extract(self, url):
         if 'Debido a tu ubicación no puedes ver el contenido' in webpage:
             self.raise_geo_restricted()
 
-        player_config = self._search_json(r'window.MDSTRM.OPTIONS\s*=', webpage, 'metadata', video_id)
+        player_config = self._search_json(r'window\.MDSTRM\.OPTIONS\s*=', webpage, 'metadata', video_id)
 
         formats, subtitles = [], {}
         for video_format in player_config['src']:
@@ -122,7 +137,7 @@ def _real_extract(self, url):
         }
 
 
-class WinSportsVideoIE(InfoExtractor):
+class WinSportsVideoIE(MediaStreamBaseIE):
     _VALID_URL = r'https?://www\.winsports\.co/videos/(?P<id>[\w-]+)'
 
     _TESTS = [{
@@ -158,21 +173,36 @@ class WinSportsVideoIE(InfoExtractor):
             'ext': 'mp4',
         },
         'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.winsports.co/videos/bucaramanga-se-quedo-con-el-grito-de-gol-en-la-garganta',
+        'info_dict': {
+            'id': '6402adb62bbf3b18d454e1b0',
+            'display_id': 'bucaramanga-se-quedo-con-el-grito-de-gol-en-la-garganta',
+            'title': '⚽Bucaramanga se quedó con el grito de gol en la garganta',
+            'description': 'Gol anulado Bucaramanga',
+            'thumbnail': r're:^https?://[^?#]+6402adb62bbf3b18d454e1b0',
+            'ext': 'mp4',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        json_ld = self._search_json_ld(webpage, display_id, expected_type='VideoObject', default={})
-        media_setting_json = self._search_json(
-            r'<script\s*[^>]+data-drupal-selector="drupal-settings-json">', webpage, 'drupal-setting-json', display_id)
+        data = self._search_json(
+            r'<script\s*[^>]+data-drupal-selector="drupal-settings-json">', webpage, 'data', display_id)
 
-        mediastream_id = traverse_obj(
-            media_setting_json, ('settings', 'mediastream_formatter', ..., 'mediastream_id', {str_or_none}),
-            get_all=False) or json_ld.get('url')
-        if not mediastream_id:
+        mediastream_url = urljoin(f'{self._EMBED_BASE_URL}/', (
+            traverse_obj(data, (
+                (('settings', 'mediastream_formatter', ..., 'mediastream_id'), 'url'), {str}), get_all=False)
+            or next(self._extract_mediastream_urls(webpage), None)))
+
+        if not mediastream_url:
             self.raise_no_formats('No MediaStream embed found in webpage')
 
+        title = clean_html(remove_end(
+            self._search_json_ld(webpage, display_id, expected_type='VideoObject', default={}).get('title')
+            or self._og_search_title(webpage), '| Win Sports'))
+
         return self.url_result(
-            urljoin('https://mdstrm.com/embed/', mediastream_id), MediaStreamIE, display_id, url_transparent=True,
-            display_id=display_id, video_title=strip_or_none(remove_end(json_ld.get('title'), '| Win Sports')))
+            mediastream_url, MediaStreamIE, display_id, url_transparent=True, display_id=display_id, video_title=title)

From 460da07439718d9af1e3661da2a23e05a913a2e6 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 16 Mar 2023 14:54:25 -0500
Subject: [PATCH 169/871] [extractor/genius] Add support for articles (#6474)

Closes #6465
Authored by: bashonly
---
 yt_dlp/extractor/genius.py | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/genius.py b/yt_dlp/extractor/genius.py
index 62f5a28ff..57c25e71e 100644
--- a/yt_dlp/extractor/genius.py
+++ b/yt_dlp/extractor/genius.py
@@ -10,7 +10,7 @@
 
 
 class GeniusIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?genius\.com/videos/(?P<id>[^?/#]+)'
+    _VALID_URL = r'https?://(?:www\.)?genius\.com/(?:videos|(?P<article>a))/(?P<id>[^?/#]+)'
     _TESTS = [{
         'url': 'https://genius.com/videos/Vince-staples-breaks-down-the-meaning-of-when-sparks-fly',
         'md5': '64c2ad98cfafcfda23bfa0ad0c512f4c',
@@ -41,19 +41,37 @@ class GeniusIE(InfoExtractor):
             'timestamp': 1631209167,
             'thumbnail': r're:^https?://.*\.jpg$',
         },
+    }, {
+        'url': 'https://genius.com/a/cordae-anderson-paak-break-down-the-meaning-of-two-tens',
+        'md5': 'f98a4e03b16b0a2821bd6e52fb3cc9d7',
+        'info_dict': {
+            'id': '6321509903112',
+            'ext': 'mp4',
+            'title': 'Cordae & Anderson .Paak Breaks Down The Meaning Of “Two Tens”',
+            'description': 'md5:1255f0e1161d07342ce56a8464ac339d',
+            'tags': ['song id: 5457554'],
+            'uploader_id': '4863540648001',
+            'duration': 361.813,
+            'upload_date': '20230301',
+            'timestamp': 1677703908,
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        display_id, is_article = self._match_valid_url(url).group('id', 'article')
         webpage = self._download_webpage(url, display_id)
 
         metadata = self._search_json(
-            r'<meta content="', webpage, 'metadata', display_id, transform_source=unescapeHTML)
-        video_id = traverse_obj(
-            metadata, ('video', 'provider_id'),
-            ('dfp_kv', lambda _, x: x['name'] == 'brightcove_video_id', 'values', 0), get_all=False)
+            r'<meta content="', webpage, 'metadata', display_id,
+            end_pattern=r'"\s+itemprop="page_data"', transform_source=unescapeHTML)
+        video_id = traverse_obj(metadata, (
+            (('article', 'media', ...), ('video', None)),
+            ('provider_id', ('dfp_kv', lambda _, v: v['name'] == 'brightcove_video_id', 'values', ...))),
+            get_all=False)
         if not video_id:
-            raise ExtractorError('Brightcove video id not found in webpage')
+            # Not all article pages have videos, expect the error
+            raise ExtractorError('Brightcove video ID not found in webpage', expected=bool(is_article))
 
         config = self._search_json(r'var\s*APP_CONFIG\s*=', webpage, 'config', video_id, default={})
         account_id = config.get('brightcove_account_id', '4863540648001')
@@ -68,7 +86,7 @@ def _real_extract(self, url):
 
 
 class GeniusLyricsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?genius\.com/(?P<id>[^?/#]+)-lyrics[?/#]?'
+    _VALID_URL = r'https?://(?:www\.)?genius\.com/(?P<id>[^?/#]+)-lyrics(?:[?/#]|$)'
     _TESTS = [{
         'url': 'https://genius.com/Lil-baby-heyy-lyrics',
         'playlist_mincount': 2,

From 216bcb66d7dce0762767d751dad10650cb57da9d Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 16 Mar 2023 14:54:56 -0500
Subject: [PATCH 170/871] [extractor/tiktok] Improve `TikTokLive` extractor
 (#6520)

Closes #6459
Authored by: bashonly
---
 yt_dlp/extractor/tiktok.py | 179 ++++++++++++++++++++++++++++++++-----
 1 file changed, 158 insertions(+), 21 deletions(-)

diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index 096748bf7..f1696a2fc 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -1,6 +1,7 @@
 import itertools
 import json
 import random
+import re
 import string
 import time
 
@@ -12,15 +13,18 @@
     LazyList,
     UnsupportedError,
     UserNotLive,
+    format_field,
     get_element_by_id,
     get_first,
     int_or_none,
     join_nonempty,
+    merge_dicts,
     qualities,
     remove_start,
     srt_subtitles_timecode,
     str_or_none,
     traverse_obj,
+    try_call,
     try_get,
     url_or_none,
 )
@@ -563,7 +567,7 @@ def _real_extract(self, url):
             self.report_warning(f'{e}; trying with webpage')
 
         url = self._create_url(user_id, video_id)
-        webpage = self._download_webpage(url, video_id, headers={'User-Agent': 'User-Agent:Mozilla/5.0'})
+        webpage = self._download_webpage(url, video_id, headers={'User-Agent': 'Mozilla/5.0'})
         next_data = self._search_nextjs_data(webpage, video_id, default='{}')
         if next_data:
             status = traverse_obj(next_data, ('props', 'pageProps', 'statusCode'), expected_type=int) or 0
@@ -983,40 +987,173 @@ def _real_extract(self, url):
         return self.url_result(new_url)
 
 
-class TikTokLiveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tiktok\.com/@(?P<id>[\w\.-]+)/live'
+class TikTokLiveIE(TikTokBaseIE):
+    _VALID_URL = r'''(?x)https?://(?:
+        (?:www\.)?tiktok\.com/@(?P<uploader>[\w.-]+)/live|
+        m\.tiktok\.com/share/live/(?P<id>\d+)
+    )'''
     IE_NAME = 'tiktok:live'
 
     _TESTS = [{
+        'url': 'https://www.tiktok.com/@weathernewslive/live',
+        'info_dict': {
+            'id': '7210809319192726273',
+            'ext': 'mp4',
+            'title': r're:ウェザーニュースLiVE[\d\s:-]*',
+            'creator': 'ウェザーニュースLiVE',
+            'uploader': 'weathernewslive',
+            'uploader_id': '6621496731283095554',
+            'uploader_url': 'https://www.tiktok.com/@weathernewslive',
+            'live_status': 'is_live',
+            'concurrent_view_count': int,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.tiktok.com/@pilarmagenta/live',
+        'info_dict': {
+            'id': '7209423610325322522',
+            'ext': 'mp4',
+            'title': str,
+            'creator': 'Pilarmagenta',
+            'uploader': 'pilarmagenta',
+            'uploader_id': '6624846890674683909',
+            'uploader_url': 'https://www.tiktok.com/@pilarmagenta',
+            'live_status': 'is_live',
+            'concurrent_view_count': int,
+        },
+        'skip': 'Livestream',
+    }, {
+        'url': 'https://m.tiktok.com/share/live/7209423610325322522/?language=en',
+        'only_matching': True,
+    }, {
         'url': 'https://www.tiktok.com/@iris04201/live',
         'only_matching': True,
     }]
 
+    def _call_api(self, url, param, room_id, uploader, key=None):
+        response = traverse_obj(self._download_json(
+            url, room_id, fatal=False, query={
+                'aid': '1988',
+                param: room_id,
+            }), (key, {dict}), default={})
+
+        # status == 2 if live else 4
+        if int_or_none(response.get('status')) == 2:
+            return response
+        # If room_id is obtained via mobile share URL and cannot be refreshed, do not wait for live
+        elif not uploader:
+            raise ExtractorError('This livestream has ended', expected=True)
+        raise UserNotLive(video_id=uploader)
+
     def _real_extract(self, url):
-        uploader = self._match_id(url)
-        webpage = self._download_webpage(url, uploader, headers={'User-Agent': 'User-Agent:Mozilla/5.0'})
-        room_id = self._html_search_regex(r'snssdk\d*://live\?room_id=(\d+)', webpage, 'room ID', default=None)
+        uploader, room_id = self._match_valid_url(url).group('uploader', 'id')
+        webpage = self._download_webpage(
+            url, uploader or room_id, headers={'User-Agent': 'Mozilla/5.0'}, fatal=not room_id)
+
+        if webpage:
+            data = try_call(lambda: self._get_sigi_state(webpage, uploader or room_id))
+            room_id = (traverse_obj(data, ('UserModule', 'users', ..., 'roomId', {str_or_none}), get_all=False)
+                       or self._search_regex(r'snssdk\d*://live\?room_id=(\d+)', webpage, 'room ID', default=None)
+                       or room_id)
+            uploader = uploader or traverse_obj(
+                data, ('LiveRoom', 'liveRoomUserInfo', 'user', 'uniqueId'),
+                ('UserModule', 'users', ..., 'uniqueId'), get_all=False, expected_type=str)
+
         if not room_id:
             raise UserNotLive(video_id=uploader)
-        live_info = traverse_obj(self._download_json(
-            'https://www.tiktok.com/api/live/detail/', room_id, query={
-                'aid': '1988',
-                'roomID': room_id,
-            }), 'LiveRoomInfo', expected_type=dict, default={})
 
-        if 'status' not in live_info:
-            raise ExtractorError('Unexpected response from TikTok API')
-        # status = 2 if live else 4
-        if not int_or_none(live_info['status']) == 2:
-            raise UserNotLive(video_id=uploader)
+        formats = []
+        live_info = self._call_api(
+            'https://webcast.tiktok.com/webcast/room/info', 'room_id', room_id, uploader, key='data')
+
+        get_quality = qualities(('SD1', 'ld', 'SD2', 'sd', 'HD1', 'hd', 'FULL_HD1', 'uhd', 'ORIGION', 'origin'))
+        parse_inner = lambda x: self._parse_json(x, None)
+
+        for quality, stream in traverse_obj(live_info, (
+                'stream_url', 'live_core_sdk_data', 'pull_data', 'stream_data',
+                {parse_inner}, 'data', {dict}), default={}).items():
+
+            sdk_params = traverse_obj(stream, ('main', 'sdk_params', {parse_inner}, {
+                'vcodec': ('VCodec', {str}),
+                'tbr': ('vbitrate', {lambda x: int_or_none(x, 1000)}),
+                'resolution': ('resolution', {lambda x: re.match(r'(?i)\d+x\d+|\d+p', x).group().lower()}),
+            }))
+
+            flv_url = traverse_obj(stream, ('main', 'flv', {url_or_none}))
+            if flv_url:
+                formats.append({
+                    'url': flv_url,
+                    'ext': 'flv',
+                    'format_id': f'flv-{quality}',
+                    'quality': get_quality(quality),
+                    **sdk_params,
+                })
+
+            hls_url = traverse_obj(stream, ('main', 'hls', {url_or_none}))
+            if hls_url:
+                formats.append({
+                    'url': hls_url,
+                    'ext': 'mp4',
+                    'protocol': 'm3u8_native',
+                    'format_id': f'hls-{quality}',
+                    'quality': get_quality(quality),
+                    **sdk_params,
+                })
+
+        def get_vcodec(*keys):
+            return traverse_obj(live_info, (
+                'stream_url', *keys, {parse_inner}, 'VCodec', {str}))
+
+        for stream in ('hls', 'rtmp'):
+            stream_url = traverse_obj(live_info, ('stream_url', f'{stream}_pull_url', {url_or_none}))
+            if stream_url:
+                formats.append({
+                    'url': stream_url,
+                    'ext': 'mp4' if stream == 'hls' else 'flv',
+                    'protocol': 'm3u8_native' if stream == 'hls' else 'https',
+                    'format_id': f'{stream}-pull',
+                    'vcodec': get_vcodec(f'{stream}_pull_url_params'),
+                    'quality': get_quality('ORIGION'),
+                })
+
+        for f_id, f_url in traverse_obj(live_info, ('stream_url', 'flv_pull_url', {dict}), default={}).items():
+            if not url_or_none(f_url):
+                continue
+            formats.append({
+                'url': f_url,
+                'ext': 'flv',
+                'format_id': f'flv-{f_id}'.lower(),
+                'vcodec': get_vcodec('flv_pull_url_params', f_id),
+                'quality': get_quality(f_id),
+            })
+
+        # If uploader is a guest on another's livestream, primary endpoint will not have m3u8 URLs
+        if not traverse_obj(formats, lambda _, v: v['ext'] == 'mp4'):
+            live_info = merge_dicts(live_info, self._call_api(
+                'https://www.tiktok.com/api/live/detail/', 'roomID', room_id, uploader, key='LiveRoomInfo'))
+            if url_or_none(live_info.get('liveUrl')):
+                formats.append({
+                    'url': live_info['liveUrl'],
+                    'ext': 'mp4',
+                    'protocol': 'm3u8_native',
+                    'format_id': 'hls-fallback',
+                    'vcodec': 'h264',
+                    'quality': get_quality('origin'),
+                })
+
+        uploader = uploader or traverse_obj(live_info, ('ownerInfo', 'uniqueId'), ('owner', 'display_id'))
 
         return {
             'id': room_id,
-            'title': live_info.get('title') or self._html_search_meta(['og:title', 'twitter:title'], webpage, default=''),
             'uploader': uploader,
-            'uploader_id': traverse_obj(live_info, ('ownerInfo', 'id')),
-            'creator': traverse_obj(live_info, ('ownerInfo', 'nickname')),
-            'concurrent_view_count': traverse_obj(live_info, ('liveRoomStats', 'userCount'), expected_type=int),
-            'formats': self._extract_m3u8_formats(live_info['liveUrl'], room_id, 'mp4', live=True),
+            'uploader_url': format_field(uploader, None, self._UPLOADER_URL_FORMAT) or None,
             'is_live': True,
+            'formats': formats,
+            '_format_sort_fields': ('quality', 'ext'),
+            **traverse_obj(live_info, {
+                'title': 'title',
+                'uploader_id': (('ownerInfo', 'owner'), 'id', {str_or_none}),
+                'creator': (('ownerInfo', 'owner'), 'nickname'),
+                'concurrent_view_count': (('user_count', ('liveRoomStats', 'userCount')), {int_or_none}),
+            }, get_all=False),
         }

From 9a06b7b1891b48cebbe275652ae8025a36d97d97 Mon Sep 17 00:00:00 2001
From: viktor-enzell <viktor.enzell@gmail.com>
Date: Sat, 18 Mar 2023 14:06:46 +0100
Subject: [PATCH 171/871] [extractor/drtv] Fix radio page extraction (#6552)

Authored by: viktor-enzell
---
 yt_dlp/extractor/drtv.py | 58 +++++++++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 19 deletions(-)

diff --git a/yt_dlp/extractor/drtv.py b/yt_dlp/extractor/drtv.py
index 470546bbc..6c381aa14 100644
--- a/yt_dlp/extractor/drtv.py
+++ b/yt_dlp/extractor/drtv.py
@@ -12,7 +12,6 @@
     mimetype2ext,
     str_or_none,
     traverse_obj,
-    try_get,
     unified_timestamp,
     update_url_query,
     url_or_none,
@@ -25,7 +24,7 @@ class DRTVIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:www\.)?dr\.dk/(?:tv/se|nyheder|(?:radio|lyd)(?:/ondemand)?)/(?:[^/]+/)*|
+                            (?:www\.)?dr\.dk/(?:tv/se|nyheder|(?P<radio>radio|lyd)(?:/ondemand)?)/(?:[^/]+/)*|
                             (?:www\.)?(?:dr\.dk|dr-massive\.com)/drtv/(?:se|episode|program)/
                         )
                         (?P<id>[\da-z_-]+)
@@ -80,7 +79,7 @@ class DRTVIE(InfoExtractor):
             'description': 'md5:8c66dcbc1669bbc6f873879880f37f2a',
             'timestamp': 1546628400,
             'upload_date': '20190104',
-            'duration': 3504.618,
+            'duration': 3504.619,
             'formats': 'mincount:20',
             'release_year': 2017,
             'season_id': 'urn:dr:mu:bundle:5afc03ad6187a4065ca5fd35',
@@ -101,14 +100,16 @@ class DRTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Bonderøven 2019 (1:8)',
             'description': 'md5:b6dcfe9b6f0bea6703e9a0092739a5bd',
-            'timestamp': 1603188600,
-            'upload_date': '20201020',
+            'timestamp': 1654856100,
+            'upload_date': '20220610',
             'duration': 2576.6,
             'season': 'Bonderøven 2019',
             'season_id': 'urn:dr:mu:bundle:5c201667a11fa01ca4528ce5',
             'release_year': 2019,
             'season_number': 2019,
-            'series': 'Frank & Kastaniegaarden'
+            'series': 'Frank & Kastaniegaarden',
+            'episode_number': 1,
+            'episode': 'Episode 1',
         },
         'params': {
             'skip_download': True,
@@ -140,10 +141,26 @@ class DRTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'this video has been removed',
+    }, {
+        'url': 'https://www.dr.dk/lyd/p4kbh/regionale-nyheder-kh4/regionale-nyheder-2023-03-14-10-30-9',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': '14802310112',
+            'timestamp': 1678786200,
+            'duration': 120.043,
+            'season_id': 'urn:dr:mu:bundle:63a4f7c87140143504b6710f',
+            'series': 'P4 København regionale nyheder',
+            'upload_date': '20230314',
+            'release_year': 0,
+            'description': 'Hør seneste regionale nyheder fra P4 København.',
+            'season': 'Regionale nyheder',
+            'title': 'Regionale nyheder',
+        },
     }]
 
     def _real_extract(self, url):
-        raw_video_id = self._match_id(url)
+        raw_video_id, is_radio_url = self._match_valid_url(url).group('id', 'radio')
 
         webpage = self._download_webpage(url, raw_video_id)
 
@@ -170,15 +187,17 @@ def _real_extract(self, url):
             programcard_url = '%s/%s' % (_PROGRAMCARD_BASE, video_id)
         else:
             programcard_url = _PROGRAMCARD_BASE
-            page = self._parse_json(
-                self._search_regex(
-                    r'data\s*=\s*({.+?})\s*(?:;|</script)', webpage,
-                    'data'), '1')['cache']['page']
-            page = page[list(page.keys())[0]]
-            item = try_get(
-                page, (lambda x: x['item'], lambda x: x['entries'][0]['item']),
-                dict)
-            video_id = item['customId'].split(':')[-1]
+            if is_radio_url:
+                video_id = self._search_nextjs_data(
+                    webpage, raw_video_id)['props']['pageProps']['episode']['productionNumber']
+            else:
+                json_data = self._search_json(
+                    r'window\.__data\s*=', webpage, 'data', raw_video_id)
+                video_id = traverse_obj(json_data, (
+                    'cache', 'page', ..., (None, ('entries', 0)), 'item', 'customId',
+                    {lambda x: x.split(':')[-1]}), get_all=False)
+                if not video_id:
+                    raise ExtractorError('Unable to extract video id')
             query['productionnumber'] = video_id
 
         data = self._download_json(
@@ -269,10 +288,11 @@ def decrypt_uri(e):
                                 f['vcodec'] = 'none'
                         formats.extend(f4m_formats)
                     elif target == 'HLS':
-                        formats.extend(self._extract_m3u8_formats(
+                        fmts, subs = self._extract_m3u8_formats_and_subtitles(
                             uri, video_id, 'mp4', entry_protocol='m3u8_native',
-                            quality=preference, m3u8_id=format_id,
-                            fatal=False))
+                            quality=preference, m3u8_id=format_id, fatal=False)
+                        formats.extend(fmts)
+                        self._merge_subtitles(subs, target=subtitles)
                     else:
                         bitrate = link.get('Bitrate')
                         if bitrate:

From c14af7a741931b364bab3d9546c0f4359f318f8c Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Sat, 18 Mar 2023 23:29:02 +0900
Subject: [PATCH 172/871] [extractor/iwara] Overhaul extractors (#6557)

Authored by: Lesmiscore
---
 yt_dlp/extractor/iwara.py | 333 ++++++++++++++++----------------------
 1 file changed, 139 insertions(+), 194 deletions(-)

diff --git a/yt_dlp/extractor/iwara.py b/yt_dlp/extractor/iwara.py
index ec3e59c6d..62a179700 100644
--- a/yt_dlp/extractor/iwara.py
+++ b/yt_dlp/extractor/iwara.py
@@ -1,239 +1,184 @@
-import itertools
-import re
+import functools
 import urllib.parse
+import hashlib
 
 from .common import InfoExtractor
 from ..utils import (
+    OnDemandPagedList,
     int_or_none,
     mimetype2ext,
-    remove_end,
-    strip_or_none,
-    unified_strdate,
-    url_or_none,
-    urljoin,
+    traverse_obj,
+    unified_timestamp,
 )
 
 
-class IwaraBaseIE(InfoExtractor):
-    _BASE_REGEX = r'(?P<base_url>https?://(?:www\.|ecchi\.)?iwara\.tv)'
-
-    def _extract_playlist(self, base_url, webpage):
-        for path in re.findall(r'class="title">\s*<a[^<]+href="([^"]+)', webpage):
-            yield self.url_result(urljoin(base_url, path))
-
-
-class IwaraIE(IwaraBaseIE):
-    _VALID_URL = fr'{IwaraBaseIE._BASE_REGEX}/videos/(?P<id>[a-zA-Z0-9]+)'
+class IwaraIE(InfoExtractor):
+    IE_NAME = 'iwara'
+    _VALID_URL = r'https?://(?:www\.)?iwara\.tv/video/(?P<id>[a-zA-Z0-9]+)'
     _TESTS = [{
-        'url': 'http://iwara.tv/videos/amVwUl1EHpAD9RD',
-        # md5 is unstable
+        # this video cannot be played because of migration
+        'only_matching': True,
+        'url': 'https://www.iwara.tv/video/k2ayoueezfkx6gvq',
         'info_dict': {
-            'id': 'amVwUl1EHpAD9RD',
+            'id': 'k2ayoueezfkx6gvq',
             'ext': 'mp4',
-            'title': '【MMD R-18】ガールフレンド carry_me_off',
             'age_limit': 18,
-            'thumbnail': 'https://i.iwara.tv/sites/default/files/videos/thumbnails/7951/thumbnail-7951_0001.png',
-            'uploader': 'Reimu丨Action',
-            'upload_date': '20150828',
-            'description': 'md5:1d4905ce48c66c9299c617f08e106e0f',
+            'title': 'Defeat of Irybelda - アイリベルダの敗北',
+            'description': 'md5:70278abebe706647a8b4cb04cf23e0d3',
+            'uploader': 'Inwerwm',
+            'uploader_id': 'inwerwm',
+            'tags': 'count:1',
+            'like_count': 6133,
+            'view_count': 1050343,
+            'comment_count': 1,
+            'timestamp': 1677843869,
+            'modified_timestamp': 1679056362,
         },
     }, {
-        'url': 'http://ecchi.iwara.tv/videos/Vb4yf2yZspkzkBO',
-        'md5': '7e5f1f359cd51a027ba4a7b7710a50f0',
+        'url': 'https://iwara.tv/video/1ywe1sbkqwumpdxz5/',
+        'md5': '20691ce1473ec2766c0788e14c60ce66',
         'info_dict': {
-            'id': '0B1LvuHnL-sRFNXB1WHNqbGw4SXc',
-            'ext': 'mp4',
-            'title': '[3D Hentai] Kyonyu × Genkai × Emaki Shinobi Girls.mp4',
-            'age_limit': 18,
-        },
-        'add_ie': ['GoogleDrive'],
-    }, {
-        'url': 'http://www.iwara.tv/videos/nawkaumd6ilezzgq',
-        # md5 is unstable
-        'info_dict': {
-            'id': '6liAP9s2Ojc',
+            'id': '1ywe1sbkqwumpdxz5',
             'ext': 'mp4',
             'age_limit': 18,
-            'title': '[MMD] Do It Again Ver.2 [1080p 60FPS] (Motion,Camera,Wav+DL)',
-            'description': 'md5:590c12c0df1443d833fbebe05da8c47a',
-            'upload_date': '20160910',
-            'uploader': 'aMMDsork',
-            'uploader_id': 'UCVOFyOSCyFkXTYYHITtqB7A',
+            'title': 'Aponia 阿波尼亚SEX  Party Tonight 手动脱衣 大奶 裸腿',
+            'description': 'md5:0c4c310f2e0592d68b9f771d348329ca',
+            'uploader': '龙也zZZ',
+            'uploader_id': 'user792540',
+            'tags': [
+                'uncategorized'
+            ],
+            'like_count': 1809,
+            'view_count': 25156,
+            'comment_count': 1,
+            'timestamp': 1678732213,
+            'modified_timestamp': 1679110271,
         },
-        'add_ie': ['Youtube'],
     }]
 
+    def _extract_formats(self, video_id, fileurl):
+        up = urllib.parse.urlparse(fileurl)
+        q = urllib.parse.parse_qs(up.query)
+        paths = up.path.rstrip('/').split('/')
+        # https://github.com/yt-dlp/yt-dlp/issues/6549#issuecomment-1473771047
+        x_version = hashlib.sha1('_'.join((paths[-1], q['expires'][0], '5nFp9kmbNnHdAFhaqMvt')).encode()).hexdigest()
+
+        files = self._download_json(fileurl, video_id, headers={'X-Version': x_version})
+        for fmt in files:
+            yield traverse_obj(fmt, {
+                'format_id': 'name',
+                'url': ('src', ('view', 'download'), {self._proto_relative_url}),
+                'ext': ('type', {mimetype2ext}),
+                'quality': ('name', {lambda x: int_or_none(x) or 1e4}),
+                'height': ('name', {int_or_none}),
+            }, get_all=False)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        webpage, urlh = self._download_webpage_handle(url, video_id)
-
-        hostname = urllib.parse.urlparse(urlh.geturl()).hostname
-        # ecchi is 'sexy' in Japanese
-        age_limit = 18 if hostname.split('.')[0] == 'ecchi' else 0
-
-        video_data = self._download_json('http://www.iwara.tv/api/video/%s' % video_id, video_id)
-
-        if not video_data:
-            iframe_url = self._html_search_regex(
-                r'<iframe[^>]+src=([\'"])(?P<url>[^\'"]+)\1',
-                webpage, 'iframe URL', group='url')
-            return {
-                '_type': 'url_transparent',
-                'url': iframe_url,
-                'age_limit': age_limit,
-            }
-
-        title = remove_end(self._html_extract_title(webpage), ' | Iwara')
-
-        thumbnail = self._html_search_regex(
-            r'poster=[\'"]([^\'"]+)', webpage, 'thumbnail', default=None)
-
-        uploader = self._html_search_regex(
-            r'class="username">([^<]+)', webpage, 'uploader', fatal=False)
-
-        upload_date = unified_strdate(self._html_search_regex(
-            r'作成日:([^\s]+)', webpage, 'upload_date', fatal=False))
-
-        description = strip_or_none(self._search_regex(
-            r'<p>(.+?(?=</div))', webpage, 'description', fatal=False,
-            flags=re.DOTALL))
-
-        formats = []
-        for a_format in video_data:
-            format_uri = url_or_none(a_format.get('uri'))
-            if not format_uri:
-                continue
-            format_id = a_format.get('resolution')
-            height = int_or_none(self._search_regex(
-                r'(\d+)p', format_id, 'height', default=None))
-            formats.append({
-                'url': self._proto_relative_url(format_uri, 'https:'),
-                'format_id': format_id,
-                'ext': mimetype2ext(a_format.get('mime')) or 'mp4',
-                'height': height,
-                'width': int_or_none(height / 9.0 * 16.0 if height else None),
-                'quality': 1 if format_id == 'Source' else 0,
-            })
+        video_data = self._download_json(f'http://api.iwara.tv/video/{video_id}', video_id)
 
         return {
             'id': video_id,
-            'title': title,
-            'age_limit': age_limit,
-            'formats': formats,
-            'thumbnail': self._proto_relative_url(thumbnail, 'https:'),
-            'uploader': uploader,
-            'upload_date': upload_date,
-            'description': description,
+            'age_limit': 18 if video_data.get('rating') == 'ecchi' else 0,  # ecchi is 'sexy' in Japanese
+            **traverse_obj(video_data, {
+                'title': 'title',
+                'description': 'body',
+                'uploader': ('user', 'name'),
+                'uploader_id': ('user', 'username'),
+                'tags': ('tags', ..., 'id'),
+                'like_count': 'numLikes',
+                'view_count': 'numViews',
+                'comment_count': 'numComments',
+                'timestamp': ('createdAt', {unified_timestamp}),
+                'modified_timestamp': ('updatedAt', {unified_timestamp}),
+                'thumbnail': ('file', 'id', {str}, {
+                    lambda x: f'https://files.iwara.tv/image/thumbnail/{x}/thumbnail-00.jpg'}),
+            }),
+            'formats': list(self._extract_formats(video_id, video_data.get('fileUrl'))),
         }
 
 
-class IwaraPlaylistIE(IwaraBaseIE):
-    _VALID_URL = fr'{IwaraBaseIE._BASE_REGEX}/playlist/(?P<id>[^/?#&]+)'
-    IE_NAME = 'iwara:playlist'
-
-    _TESTS = [{
-        'url': 'https://ecchi.iwara.tv/playlist/best-enf',
-        'info_dict': {
-            'title': 'Best enf',
-            'uploader': 'Jared98112',
-            'id': 'best-enf',
-        },
-        'playlist_mincount': 1097,
-    }, {
-        # urlencoded
-        'url': 'https://ecchi.iwara.tv/playlist/%E3%83%97%E3%83%AC%E3%82%A4%E3%83%AA%E3%82%B9%E3%83%88-2',
-        'info_dict': {
-            'id': 'プレイリスト-2',
-            'title': 'プレイリスト',
-            'uploader': 'mainyu',
-        },
-        'playlist_mincount': 91,
-    }]
-
-    def _real_extract(self, url):
-        playlist_id, base_url = self._match_valid_url(url).group('id', 'base_url')
-        playlist_id = urllib.parse.unquote(playlist_id)
-        webpage = self._download_webpage(url, playlist_id)
-
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'title': self._html_search_regex(r'class="title"[^>]*>([^<]+)', webpage, 'title', fatal=False),
-            'uploader': self._html_search_regex(r'<h2>([^<]+)', webpage, 'uploader', fatal=False),
-            'entries': self._extract_playlist(base_url, webpage),
-        }
-
-
-class IwaraUserIE(IwaraBaseIE):
-    _VALID_URL = fr'{IwaraBaseIE._BASE_REGEX}/users/(?P<id>[^/?#&]+)'
+class IwaraUserIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?iwara\.tv/profile/(?P<id>[^/?#&]+)'
     IE_NAME = 'iwara:user'
+    _PER_PAGE = 32
 
     _TESTS = [{
-        'note': 'number of all videos page is just 1 page. less than 40 videos',
-        'url': 'https://ecchi.iwara.tv/users/infinityyukarip',
+        'url': 'https://iwara.tv/profile/user792540/videos',
         'info_dict': {
-            'title': 'Uploaded videos from Infinity_YukariP',
-            'id': 'infinityyukarip',
-            'uploader': 'Infinity_YukariP',
-            'uploader_id': 'infinityyukarip',
+            'id': 'user792540',
         },
-        'playlist_mincount': 39,
+        'playlist_mincount': 80,
     }, {
-        'note': 'no even all videos page. probably less than 10 videos',
-        'url': 'https://ecchi.iwara.tv/users/mmd-quintet',
+        'url': 'https://iwara.tv/profile/theblackbirdcalls/videos',
         'info_dict': {
-            'title': 'Uploaded videos from mmd quintet',
-            'id': 'mmd-quintet',
-            'uploader': 'mmd quintet',
-            'uploader_id': 'mmd-quintet',
-        },
-        'playlist_mincount': 6,
-    }, {
-        'note': 'has paging. more than 40 videos',
-        'url': 'https://ecchi.iwara.tv/users/theblackbirdcalls',
-        'info_dict': {
-            'title': 'Uploaded videos from TheBlackbirdCalls',
             'id': 'theblackbirdcalls',
-            'uploader': 'TheBlackbirdCalls',
-            'uploader_id': 'theblackbirdcalls',
         },
-        'playlist_mincount': 420,
+        'playlist_mincount': 723,
     }, {
-        'note': 'foreign chars in URL. there must be foreign characters in URL',
-        'url': 'https://ecchi.iwara.tv/users/ぶた丼',
-        'info_dict': {
-            'title': 'Uploaded videos from ぶた丼',
-            'id': 'ぶた丼',
-            'uploader': 'ぶた丼',
-            'uploader_id': 'ぶた丼',
-        },
-        'playlist_mincount': 170,
+        'url': 'https://iwara.tv/profile/user792540',
+        'only_matching': True,
+    }, {
+        'url': 'https://iwara.tv/profile/theblackbirdcalls',
+        'only_matching': True,
     }]
 
-    def _entries(self, playlist_id, base_url):
-        webpage = self._download_webpage(
-            f'{base_url}/users/{playlist_id}', playlist_id)
-        videos_url = self._search_regex(r'<a href="(/users/[^/]+/videos)(?:\?[^"]+)?">', webpage, 'all videos url', default=None)
-        if not videos_url:
-            yield from self._extract_playlist(base_url, webpage)
-            return
-
-        videos_url = urljoin(base_url, videos_url)
-
-        for n in itertools.count(1):
-            page = self._download_webpage(
-                videos_url, playlist_id, note=f'Downloading playlist page {n}',
-                query={'page': str(n - 1)} if n > 1 else {})
-            yield from self._extract_playlist(
-                base_url, page)
-
-            if f'page={n}' not in page:
-                break
+    def _entries(self, playlist_id, user_id, page):
+        videos = self._download_json(
+            'https://api.iwara.tv/videos', playlist_id,
+            note=f'Downloading page {page}',
+            query={
+                'page': page,
+                'sort': 'date',
+                'user': user_id,
+                'limit': self._PER_PAGE,
+            })
+        for x in traverse_obj(videos, ('results', ..., 'id')):
+            yield self.url_result(f'https://iwara.tv/video/{x}')
 
     def _real_extract(self, url):
-        playlist_id, base_url = self._match_valid_url(url).group('id', 'base_url')
-        playlist_id = urllib.parse.unquote(playlist_id)
+        playlist_id = self._match_id(url)
+        user_info = self._download_json(
+            f'https://api.iwara.tv/profile/{playlist_id}', playlist_id,
+            note='Requesting user info')
+        user_id = traverse_obj(user_info, ('user', 'id'))
 
         return self.playlist_result(
-            self._entries(playlist_id, base_url), playlist_id)
+            OnDemandPagedList(
+                functools.partial(self._entries, playlist_id, user_id),
+                self._PER_PAGE),
+            playlist_id, traverse_obj(user_info, ('user', 'name')))
+
+
+class IwaraPlaylistIE(InfoExtractor):
+    # the ID is an UUID but I don't think it's necessary to write concrete regex
+    _VALID_URL = r'https?://(?:www\.)?iwara\.tv/playlist/(?P<id>[0-9a-f-]+)'
+    IE_NAME = 'iwara:playlist'
+    _PER_PAGE = 32
+
+    _TESTS = [{
+        'url': 'https://iwara.tv/playlist/458e5486-36a4-4ac0-b233-7e9eef01025f',
+        'info_dict': {
+            'id': '458e5486-36a4-4ac0-b233-7e9eef01025f',
+        },
+        'playlist_mincount': 3,
+    }]
+
+    def _entries(self, playlist_id, first_page, page):
+        videos = self._download_json(
+            'https://api.iwara.tv/videos', playlist_id, f'Downloading page {page}',
+            query={'page': page, 'limit': self._PER_PAGE}) if page else first_page
+        for x in traverse_obj(videos, ('results', ..., 'id')):
+            yield self.url_result(f'https://iwara.tv/video/{x}')
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        page_0 = self._download_json(
+            f'https://api.iwara.tv/playlist/{playlist_id}?page=0&limit={self._PER_PAGE}', playlist_id,
+            note='Requesting playlist info')
+
+        return self.playlist_result(
+            OnDemandPagedList(
+                functools.partial(self._entries, playlist_id, page_0),
+                self._PER_PAGE),
+            playlist_id, traverse_obj(page_0, ('title', 'name')))

From e4cf7741f9302b3faa092962f2895b55cb3d89bb Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 21 Mar 2023 17:48:22 -0500
Subject: [PATCH 173/871] [extractor/rozhlas] Extract manifest formats (#6590)

Closes #6584
Authored by: bashonly
---
 yt_dlp/extractor/rozhlas.py | 80 ++++++++++++++++++++++++++++---------
 1 file changed, 61 insertions(+), 19 deletions(-)

diff --git a/yt_dlp/extractor/rozhlas.py b/yt_dlp/extractor/rozhlas.py
index 08ebb93e3..5cc664e00 100644
--- a/yt_dlp/extractor/rozhlas.py
+++ b/yt_dlp/extractor/rozhlas.py
@@ -1,5 +1,12 @@
 from .common import InfoExtractor
-from ..utils import extract_attributes, int_or_none, remove_start, traverse_obj
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+    remove_start,
+    str_or_none,
+    traverse_obj,
+    url_or_none,
+)
 
 
 class RozhlasIE(InfoExtractor):
@@ -50,7 +57,7 @@ class RozhlasVltavaIE(InfoExtractor):
         'url': 'https://wave.rozhlas.cz/papej-masicko-porcujeme-a-bilancujeme-filmy-a-serialy-ktere-letos-zabily-8891337',
         'md5': 'ba2fdbc1242fc16771c7695d271ec355',
         'info_dict': {
-            'id': 8891337,
+            'id': '8891337',
             'title': 'md5:21f99739d04ab49d8c189ec711eef4ec',
         },
         'playlist_count': 1,
@@ -69,7 +76,7 @@ class RozhlasVltavaIE(InfoExtractor):
     }, {
         'url': 'https://wave.rozhlas.cz/poslechnete-si-neklid-podcastovy-thriller-o-vine-strachu-a-vztahu-ktery-zasel-8554744',
         'info_dict': {
-            'id': 8554744,
+            'id': '8554744',
             'title': 'Poslechněte si Neklid. Podcastový thriller o vině, strachu a vztahu, který zašel příliš daleko',
         },
         'playlist_count': 5,
@@ -139,27 +146,62 @@ class RozhlasVltavaIE(InfoExtractor):
                 'chapter_number': 5,
             },
         }]
+    }, {
+        'url': 'https://dvojka.rozhlas.cz/karel-siktanc-cerny-jezdec-bily-kun-napinava-pohadka-o-tajemnem-prizraku-8946969',
+        'info_dict': {
+            'id': '8946969',
+            'title': 'Karel Šiktanc: Černý jezdec, bílý kůň. Napínavá pohádka o tajemném přízraku',
+        },
+        'playlist_count': 1,
+        'playlist': [{
+            'info_dict': {
+                'id': '10631121',
+                'ext': 'm4a',
+                'title': 'Karel Šiktanc: Černý jezdec, bílý kůň. Napínavá pohádka o tajemném přízraku',
+                'description': 'Karel Šiktanc: Černý jezdec, bílý kůň',
+                'duration': 2656,
+                'artist': 'Tvůrčí skupina Drama a literatura',
+                'channel_id': 'dvojka',
+            },
+        }],
+        'params': {'skip_download': 'dash'},
     }]
 
     def _extract_video(self, entry):
-        chapter_number = int_or_none(traverse_obj(entry, ('meta', 'ga', 'contentSerialPart')))
+        formats = []
+        audio_id = entry['meta']['ga']['contentId']
+        for audio in traverse_obj(entry, ('audioLinks', lambda _, v: url_or_none(v['url']))):
+            ext = audio.get('variant')
+            if ext == 'dash':
+                formats.extend(self._extract_mpd_formats(
+                    audio['url'], audio_id, mpd_id=ext, fatal=False))
+            elif ext == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    audio['url'], audio_id, 'm4a', m3u8_id=ext, fatal=False))
+            else:
+                formats.append({
+                    'url': audio['url'],
+                    'ext': ext,
+                    'format_id': ext,
+                    'abr': int_or_none(audio.get('bitrate')),
+                    'acodec': ext,
+                    'vcodec': 'none',
+                })
+
+        chapter_number = traverse_obj(entry, ('meta', 'ga', 'contentSerialPart', {int_or_none}))
+
         return {
-            'id': entry['meta']['ga']['contentId'],
-            'title': traverse_obj(entry, ('meta', 'ga', 'contentName')),
-            'description': entry.get('title'),
-            'duration': entry.get('duration'),
-            'artist': traverse_obj(entry, ('meta', 'ga', 'contentAuthor')),
-            'channel_id': traverse_obj(entry, ('meta', 'ga', 'contentCreator')),
+            'id': audio_id,
             'chapter': traverse_obj(entry, ('meta', 'ga', 'contentNameShort')) if chapter_number else None,
             'chapter_number': chapter_number,
-            'formats': [{
-                'url': audio_link['url'],
-                'ext': audio_link.get('variant'),
-                'format_id': audio_link.get('variant'),
-                'abr': audio_link.get('bitrate'),
-                'acodec': audio_link.get('variant'),
-                'vcodec': 'none',
-            } for audio_link in entry['audioLinks']],
+            'formats': formats,
+            **traverse_obj(entry, {
+                'title': ('meta', 'ga', 'contentName'),
+                'description': 'title',
+                'duration': ('duration', {int_or_none}),
+                'artist': ('meta', 'ga', 'contentAuthor'),
+                'channel_id': ('meta', 'ga', 'contentCreator'),
+            })
         }
 
     def _real_extract(self, url):
@@ -173,7 +215,7 @@ def _real_extract(self, url):
 
         return {
             '_type': 'playlist',
-            'id': data.get('embedId'),
+            'id': str_or_none(data.get('embedId')) or video_id,
             'title': traverse_obj(data, ('series', 'title')),
             'entries': map(self._extract_video, data['playlist']),
         }

From 06966cb8966b9aa4f60ab9c44c182a057d4ca3a3 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 21 Mar 2023 17:57:46 -0500
Subject: [PATCH 174/871] [extractor/bravotv] Fix extractor (#6568)

Closes #6562
Authored by: bashonly
---
 yt_dlp/extractor/bravotv.py | 232 +++++++++++++++++++++++-------------
 1 file changed, 150 insertions(+), 82 deletions(-)

diff --git a/yt_dlp/extractor/bravotv.py b/yt_dlp/extractor/bravotv.py
index d4895848e..d4bf9b53b 100644
--- a/yt_dlp/extractor/bravotv.py
+++ b/yt_dlp/extractor/bravotv.py
@@ -1,117 +1,185 @@
-import re
-
 from .adobepass import AdobePassIE
 from ..utils import (
-    smuggle_url,
-    update_url_query,
-    int_or_none,
+    extract_attributes,
     float_or_none,
-    try_get,
-    dict_get,
+    get_element_html_by_class,
+    int_or_none,
+    merge_dicts,
+    parse_age_limit,
+    remove_end,
+    str_or_none,
+    traverse_obj,
+    unescapeHTML,
+    unified_timestamp,
+    update_url_query,
+    url_or_none,
 )
 
 
 class BravoTVIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?(?P<req_id>bravotv|oxygen)\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>bravotv|oxygen)\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.bravotv.com/top-chef/season-16/episode-15/videos/the-top-chef-season-16-winner-is',
-        'md5': 'e34684cfea2a96cd2ee1ef3a60909de9',
         'info_dict': {
-            'id': 'epL0pmK1kQlT',
+            'id': '3923059',
             'ext': 'mp4',
             'title': 'The Top Chef Season 16 Winner Is...',
             'description': 'Find out who takes the title of Top Chef!',
-            'uploader': 'NBCU-BRAV',
             'upload_date': '20190314',
             'timestamp': 1552591860,
             'season_number': 16,
             'episode_number': 15,
             'series': 'Top Chef',
             'episode': 'The Top Chef Season 16 Winner Is...',
-            'duration': 190.0,
-        }
+            'duration': 190.357,
+            'season': 'Season 16',
+            'thumbnail': r're:^https://.+\.jpg',
+        },
+        'params': {'skip_download': 'm3u8'},
     }, {
-        'url': 'http://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
-        'only_matching': True,
+        'url': 'https://www.bravotv.com/top-chef/season-20/episode-1/london-calling',
+        'info_dict': {
+            'id': '9000234570',
+            'ext': 'mp4',
+            'title': 'London Calling',
+            'description': 'md5:5af95a8cbac1856bd10e7562f86bb759',
+            'upload_date': '20230310',
+            'timestamp': 1678410000,
+            'season_number': 20,
+            'episode_number': 1,
+            'series': 'Top Chef',
+            'episode': 'London Calling',
+            'duration': 3266.03,
+            'season': 'Season 20',
+            'chapters': 'count:7',
+            'thumbnail': r're:^https://.+\.jpg',
+            'age_limit': 14,
+        },
+        'params': {'skip_download': 'm3u8'},
+        'skip': 'This video requires AdobePass MSO credentials',
+    }, {
+        'url': 'https://www.oxygen.com/in-ice-cold-blood/season-1/closing-night',
+        'info_dict': {
+            'id': '3692045',
+            'ext': 'mp4',
+            'title': 'Closing Night',
+            'description': 'md5:3170065c5c2f19548d72a4cbc254af63',
+            'upload_date': '20180401',
+            'timestamp': 1522623600,
+            'season_number': 1,
+            'episode_number': 1,
+            'series': 'In Ice Cold Blood',
+            'episode': 'Closing Night',
+            'duration': 2629.051,
+            'season': 'Season 1',
+            'chapters': 'count:6',
+            'thumbnail': r're:^https://.+\.jpg',
+            'age_limit': 14,
+        },
+        'params': {'skip_download': 'm3u8'},
+        'skip': 'This video requires AdobePass MSO credentials',
     }, {
         'url': 'https://www.oxygen.com/in-ice-cold-blood/season-2/episode-16/videos/handling-the-horwitz-house-after-the-murder-season-2',
+        'info_dict': {
+            'id': '3974019',
+            'ext': 'mp4',
+            'title': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
+            'description': 'md5:f9d638dd6946a1c1c0533a9c6100eae5',
+            'upload_date': '20190617',
+            'timestamp': 1560790800,
+            'season_number': 2,
+            'episode_number': 16,
+            'series': 'In Ice Cold Blood',
+            'episode': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
+            'duration': 68.235,
+            'season': 'Season 2',
+            'thumbnail': r're:^https://.+\.jpg',
+            'age_limit': 14,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        site, display_id = self._match_valid_url(url).groups()
+        site, display_id = self._match_valid_url(url).group('site', 'id')
         webpage = self._download_webpage(url, display_id)
-        settings = self._parse_json(self._search_regex(
-            r'<script[^>]+data-drupal-selector="drupal-settings-json"[^>]*>({.+?})</script>', webpage, 'drupal settings'),
-            display_id)
-        info = {}
+        settings = self._search_json(
+            r'<script[^>]+data-drupal-selector="drupal-settings-json"[^>]*>', webpage, 'settings', display_id)
+        tve = extract_attributes(get_element_html_by_class('tve-video-deck-app', webpage) or '')
         query = {
-            'mbr': 'true',
+            'manifest': 'm3u',
+            'formats': 'm3u,mpeg4',
         }
-        account_pid, release_pid = [None] * 2
-        tve = settings.get('ls_tve')
+
         if tve:
-            query['manifest'] = 'm3u'
-            mobj = re.search(r'<[^>]+id="pdk-player"[^>]+data-url=["\']?(?:https?:)?//player\.theplatform\.com/p/([^/]+)/(?:[^/]+/)*select/([^?#&"\']+)', webpage)
-            if mobj:
-                account_pid, tp_path = mobj.groups()
-                release_pid = tp_path.strip('/').split('/')[-1]
-            else:
-                account_pid = 'HNK2IC'
-                tp_path = release_pid = tve['release_pid']
-            if tve.get('entitlement') == 'auth':
-                adobe_pass = settings.get('tve_adobe_auth', {})
-                if site == 'bravotv':
-                    site = 'bravo'
+            account_pid = tve.get('data-mpx-media-account-pid') or 'HNK2IC'
+            account_id = tve['data-mpx-media-account-id']
+            metadata = self._parse_json(
+                tve.get('data-normalized-video', ''), display_id, fatal=False, transform_source=unescapeHTML)
+            video_id = tve.get('data-guid') or metadata['guid']
+            if tve.get('data-entitlement') == 'auth':
+                auth = traverse_obj(settings, ('tve_adobe_auth', {dict})) or {}
+                site = remove_end(site, 'tv')
+                release_pid = tve['data-release-pid']
                 resource = self._get_mvpd_resource(
-                    adobe_pass.get('adobePassResourceId') or site,
-                    tve['title'], release_pid, tve.get('rating'))
-                query['auth'] = self._extract_mvpd_auth(
-                    url, release_pid,
-                    adobe_pass.get('adobePassRequestorId') or site, resource)
+                    tve.get('data-adobe-pass-resource-id') or auth.get('adobePassResourceId') or site,
+                    tve['data-title'], release_pid, tve.get('data-rating'))
+                query.update({
+                    'switch': 'HLSServiceSecure',
+                    'auth': self._extract_mvpd_auth(
+                        url, release_pid, auth.get('adobePassRequestorId') or site, resource),
+                })
+
         else:
-            shared_playlist = settings['ls_playlist']
-            account_pid = shared_playlist['account_pid']
-            metadata = shared_playlist['video_metadata'][shared_playlist['default_clip']]
-            tp_path = release_pid = metadata.get('release_pid')
-            if not release_pid:
-                release_pid = metadata['guid']
-                tp_path = 'media/guid/2140479951/' + release_pid
-            info.update({
-                'title': metadata['title'],
-                'description': metadata.get('description'),
-                'season_number': int_or_none(metadata.get('season_num')),
-                'episode_number': int_or_none(metadata.get('episode_num')),
-            })
-            query['switch'] = 'progressive'
-
-        tp_url = 'http://link.theplatform.com/s/%s/%s' % (account_pid, tp_path)
+            ls_playlist = traverse_obj(settings, ('ls_playlist', ..., {dict}), get_all=False) or {}
+            account_pid = ls_playlist.get('mpxMediaAccountPid') or 'PHSl-B'
+            account_id = ls_playlist['mpxMediaAccountId']
+            video_id = ls_playlist['defaultGuid']
+            metadata = traverse_obj(
+                ls_playlist, ('videos', lambda _, v: v['guid'] == video_id, {dict}), get_all=False)
 
+        tp_url = f'https://link.theplatform.com/s/{account_pid}/media/guid/{account_id}/{video_id}'
         tp_metadata = self._download_json(
-            update_url_query(tp_url, {'format': 'preview'}),
-            display_id, fatal=False)
-        if tp_metadata:
-            info.update({
-                'title': tp_metadata.get('title'),
-                'description': tp_metadata.get('description'),
-                'duration': float_or_none(tp_metadata.get('duration'), 1000),
-                'season_number': int_or_none(
-                    dict_get(tp_metadata, ('pl1$seasonNumber', 'nbcu$seasonNumber'))),
-                'episode_number': int_or_none(
-                    dict_get(tp_metadata, ('pl1$episodeNumber', 'nbcu$episodeNumber'))),
-                # For some reason the series is sometimes wrapped into a single element array.
-                'series': try_get(
-                    dict_get(tp_metadata, ('pl1$show', 'nbcu$show')),
-                    lambda x: x[0] if isinstance(x, list) else x,
-                    expected_type=str),
-                'episode': dict_get(
-                    tp_metadata, ('pl1$episodeName', 'nbcu$episodeName', 'title')),
-            })
+            update_url_query(tp_url, {'format': 'preview'}), video_id, fatal=False)
 
-        info.update({
-            '_type': 'url_transparent',
-            'id': release_pid,
-            'url': smuggle_url(update_url_query(tp_url, query), {'force_smil_url': True}),
-            'ie_key': 'ThePlatform',
-        })
-        return info
+        seconds_or_none = lambda x: float_or_none(x, 1000)
+        chapters = traverse_obj(tp_metadata, ('chapters', ..., {
+            'start_time': ('startTime', {seconds_or_none}),
+            'end_time': ('endTime', {seconds_or_none}),
+        }))
+        # prune pointless single chapters that span the entire duration from short videos
+        if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')):
+            chapters = None
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            update_url_query(f'{tp_url}/stream.m3u8', query), video_id, 'mp4', m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'chapters': chapters,
+            **merge_dicts(traverse_obj(tp_metadata, {
+                'title': 'title',
+                'description': 'description',
+                'duration': ('duration', {seconds_or_none}),
+                'timestamp': ('pubDate', {seconds_or_none}),
+                'season_number': (('pl1$seasonNumber', 'nbcu$seasonNumber'), {int_or_none}),
+                'episode_number': (('pl1$episodeNumber', 'nbcu$episodeNumber'), {int_or_none}),
+                'series': (('pl1$show', 'nbcu$show'), (None, ...), {str}),
+                'episode': (('title', 'pl1$episodeNumber', 'nbcu$episodeNumber'), {str_or_none}),
+                'age_limit': ('ratings', ..., 'rating', {parse_age_limit}),
+            }, get_all=False), traverse_obj(metadata, {
+                'title': 'title',
+                'description': 'description',
+                'duration': ('durationInSeconds', {int_or_none}),
+                'timestamp': ('airDate', {unified_timestamp}),
+                'thumbnail': ('thumbnailUrl', {url_or_none}),
+                'season_number': ('seasonNumber', {int_or_none}),
+                'episode_number': ('episodeNumber', {int_or_none}),
+                'episode': 'episodeTitle',
+                'series': 'show',
+            }))
+        }

From c2e0fc40a73dd85ab3920f977f579d475e66ef59 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 21 Mar 2023 18:12:17 -0500
Subject: [PATCH 175/871] [extractor/generic] Add extractor-args `hls_key`,
 `variant_query` (#6567)

Authored by: bashonly
---
 README.md                   |  2 ++
 yt_dlp/extractor/generic.py | 32 +++++++++++++++++++++-----------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index de83e421f..9ce85d631 100644
--- a/README.md
+++ b/README.md
@@ -1798,6 +1798,8 @@ #### youtubetab (YouTube playlists, channels, feeds, etc.)
 
 #### generic
 * `fragment_query`: Passthrough any query in mpd/m3u8 manifest URLs to their fragments. Does not apply to ffmpeg
+* `variant_query`: Passthrough the master m3u8 URL query to its variant playlist URLs
+* `hls_key`: An HLS AES-128 key URI *or* key (as hex), and optionally the IV (as hex), in the form of `(URI|KEY)[,IV]`; e.g. `generic:hls_key=ABCDEF1234567980,0xFEDCBA0987654321`. Passing any of these values will force usage of the native HLS downloader and override the corresponding values found in the m3u8 playlist
 
 #### funimation
 * `language`: Audio languages to extract, e.g. `funimation:language=english,japanese`
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 49aa5a1f5..075bb36de 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -24,6 +24,7 @@
     mimetype2ext,
     orderedSet,
     parse_duration,
+    parse_qs,
     parse_resolution,
     smuggle_url,
     str_or_none,
@@ -32,6 +33,7 @@
     unescapeHTML,
     unified_timestamp,
     unsmuggle_url,
+    update_url_query,
     url_or_none,
     urljoin,
     variadic,
@@ -2184,12 +2186,21 @@ def report_detected(self, name, num=1, note=None):
 
         self._downloader.write_debug(f'Identified {num} {name}{format_field(note, None, "; %s")}')
 
-    def _fragment_query(self, url):
+    def _extra_manifest_info(self, info, manifest_url):
         if self._configuration_arg('fragment_query'):
-            query_string = urllib.parse.urlparse(url).query
+            query_string = urllib.parse.urlparse(manifest_url).query
             if query_string:
-                return {'extra_param_to_segment_url': query_string}
-        return {}
+                info['extra_param_to_segment_url'] = query_string
+
+        hex_or_none = lambda x: x if re.fullmatch(r'(0x)?[\da-f]+', x, re.IGNORECASE) else None
+        info['hls_aes'] = traverse_obj(self._configuration_arg('hls_key'), {
+            'uri': (0, {url_or_none}), 'key': (0, {hex_or_none}), 'iv': (1, {hex_or_none}),
+        }) or None
+
+        if self._configuration_arg('variant_query'):
+            query = parse_qs(manifest_url)
+            for fmt in self._downloader._get_formats(info):
+                fmt['url'] = update_url_query(fmt['url'], query)
 
     def _extract_rss(self, url, video_id, doc):
         NS_MAP = {
@@ -2397,10 +2408,8 @@ def _real_extract(self, url):
             subtitles = {}
             if format_id.endswith('mpegurl') or ext == 'm3u8':
                 formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4', headers=headers)
-                info_dict.update(self._fragment_query(url))
             elif format_id.endswith('mpd') or format_id.endswith('dash+xml') or ext == 'mpd':
                 formats, subtitles = self._extract_mpd_formats_and_subtitles(url, video_id, headers=headers)
-                info_dict.update(self._fragment_query(url))
             elif format_id == 'f4m' or ext == 'f4m':
                 formats = self._extract_f4m_formats(url, video_id, headers=headers)
             else:
@@ -2415,6 +2424,7 @@ def _real_extract(self, url):
                 'subtitles': subtitles,
                 'http_headers': headers or None,
             })
+            self._extra_manifest_info(info_dict, url)
             return info_dict
 
         if not self.get_param('test', False) and not is_intentional:
@@ -2427,7 +2437,7 @@ def _real_extract(self, url):
         if first_bytes.startswith(b'#EXTM3U'):
             self.report_detected('M3U playlist')
             info_dict['formats'], info_dict['subtitles'] = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4')
-            info_dict.update(self._fragment_query(url))
+            self._extra_manifest_info(info_dict, url)
             return info_dict
 
         # Maybe it's a direct link to a video?
@@ -2478,7 +2488,7 @@ def _real_extract(self, url):
                     doc,
                     mpd_base_url=full_response.geturl().rpartition('/')[0],
                     mpd_url=url)
-                info_dict.update(self._fragment_query(url))
+                self._extra_manifest_info(info_dict, url)
                 self.report_detected('DASH manifest')
                 return info_dict
             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
@@ -2592,7 +2602,7 @@ def _extract_embeds(self, url, webpage, *, urlh=None, info_dict={}):
                     formats.extend(fmts)
                     self._merge_subtitles(subs, target=subtitles)
                 for fmt in formats:
-                    fmt.update(self._fragment_query(src))
+                    self._extra_manifest_info(fmt, src)
 
                 if not formats:
                     formats.append({
@@ -2795,10 +2805,10 @@ def filter_video(urls):
                 return [self._extract_xspf_playlist(video_url, video_id)]
             elif ext == 'm3u8':
                 entry_info_dict['formats'], entry_info_dict['subtitles'] = self._extract_m3u8_formats_and_subtitles(video_url, video_id, ext='mp4', headers=headers)
-                entry_info_dict.update(self._fragment_query(video_url))
+                self._extra_manifest_info(entry_info_dict, video_url)
             elif ext == 'mpd':
                 entry_info_dict['formats'], entry_info_dict['subtitles'] = self._extract_mpd_formats_and_subtitles(video_url, video_id, headers=headers)
-                entry_info_dict.update(self._fragment_query(video_url))
+                self._extra_manifest_info(entry_info_dict, video_url)
             elif ext == 'f4m':
                 entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id, headers=headers)
             elif re.search(r'(?i)\.(?:ism|smil)/manifest', video_url) and video_url != url:

From 44369c9afa996e14e9f466754481d878811b5b4a Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 23 Mar 2023 11:18:42 -0500
Subject: [PATCH 176/871] [extractor/cbs] Add `ParamountPressExpress` extractor
 (#6604)

Closes #6597
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   5 +-
 yt_dlp/extractor/brightcove.py  |   6 +-
 yt_dlp/extractor/cbs.py         | 113 ++++++++++++++++++++++++++++++++
 3 files changed, 121 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 01281b5a1..6c948e5fc 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -298,7 +298,10 @@
     CBCGemPlaylistIE,
     CBCGemLiveIE,
 )
-from .cbs import CBSIE
+from .cbs import (
+    CBSIE,
+    ParamountPressExpressIE,
+)
 from .cbslocal import (
     CBSLocalIE,
     CBSLocalArticleIE,
diff --git a/yt_dlp/extractor/brightcove.py b/yt_dlp/extractor/brightcove.py
index 2b7ddcae8..cd0e8ff27 100644
--- a/yt_dlp/extractor/brightcove.py
+++ b/yt_dlp/extractor/brightcove.py
@@ -575,6 +575,7 @@ def build_format_id(kind):
                 self.raise_no_formats(
                     error.get('message') or error.get('error_subcode') or error['error_code'], expected=True)
 
+        headers.pop('Authorization', None)  # or else http formats will give error 400
         for f in formats:
             f.setdefault('http_headers', {}).update(headers)
 
@@ -895,8 +896,9 @@ def extract_policy_key():
             store_pk(policy_key)
             return policy_key
 
-        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/%ss/%s' % (account_id, content_type, video_id)
-        headers = {}
+        token = smuggled_data.get('token')
+        api_url = f'https://{"edge-auth" if token else "edge"}.api.brightcove.com/playback/v1/accounts/{account_id}/{content_type}s/{video_id}'
+        headers = {'Authorization': f'Bearer {token}'} if token else {}
         referrer = smuggled_data.get('referrer')  # XXX: notice the spelling/case of the key
         if referrer:
             headers.update({
diff --git a/yt_dlp/extractor/cbs.py b/yt_dlp/extractor/cbs.py
index 9aacd50c4..1c0dbdea9 100644
--- a/yt_dlp/extractor/cbs.py
+++ b/yt_dlp/extractor/cbs.py
@@ -1,8 +1,14 @@
+from .brightcove import BrightcoveNewIE
+from .common import InfoExtractor
 from .theplatform import ThePlatformFeedIE
+from .youtube import YoutubeIE
 from ..utils import (
     ExtractorError,
+    extract_attributes,
+    get_element_html_by_id,
     int_or_none,
     find_xpath_attr,
+    smuggle_url,
     xpath_element,
     xpath_text,
     update_url_query,
@@ -162,3 +168,110 @@ def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517):
             'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000),
             'thumbnail': url_or_none(xpath_text(video_data, 'previewImageURL')),
         })
+
+
+class ParamountPressExpressIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?paramountpressexpress\.com(?:/[\w-]+)+/(?P<yt>yt-)?video/?\?watch=(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.paramountpressexpress.com/cbs-entertainment/shows/survivor/video/?watch=pnzew7e2hx',
+        'md5': '56631dbcadaab980d1fc47cb7b76cba4',
+        'info_dict': {
+            'id': '6322981580112',
+            'ext': 'mp4',
+            'title': 'I’m Felicia',
+            'description': 'md5:88fad93f8eede1c9c8f390239e4c6290',
+            'uploader_id': '6055873637001',
+            'upload_date': '20230320',
+            'timestamp': 1679334960,
+            'duration': 49.557,
+            'thumbnail': r're:^https://.+\.jpg',
+            'tags': [],
+        },
+    }, {
+        'url': 'https://www.paramountpressexpress.com/cbs-entertainment/video/?watch=2s5eh8kppc',
+        'md5': 'edcb03e3210b88a3e56c05aa863e0e5b',
+        'info_dict': {
+            'id': '6323036027112',
+            'ext': 'mp4',
+            'title': '‘Y&R’ Set Visit: Jerry O’Connell Quizzes Cast on Pre-Love Scene Rituals and More',
+            'description': 'md5:b929867a357aac5544b783d834c78383',
+            'uploader_id': '6055873637001',
+            'upload_date': '20230321',
+            'timestamp': 1679430180,
+            'duration': 132.032,
+            'thumbnail': r're:^https://.+\.jpg',
+            'tags': [],
+        },
+    }, {
+        'url': 'https://www.paramountpressexpress.com/paramount-plus/yt-video/?watch=OX9wJWOcqck',
+        'info_dict': {
+            'id': 'OX9wJWOcqck',
+            'ext': 'mp4',
+            'title': 'Rugrats | Season 2 Official Trailer | Paramount+',
+            'description': 'md5:1f7e26f5625a9f0d6564d9ad97a9f7de',
+            'uploader': 'Paramount Plus',
+            'uploader_id': '@paramountplus',
+            'uploader_url': 'http://www.youtube.com/@paramountplus',
+            'channel': 'Paramount Plus',
+            'channel_id': 'UCrRttZIypNTA1Mrfwo745Sg',
+            'channel_url': 'https://www.youtube.com/channel/UCrRttZIypNTA1Mrfwo745Sg',
+            'upload_date': '20230316',
+            'duration': 88,
+            'age_limit': 0,
+            'availability': 'public',
+            'live_status': 'not_live',
+            'playable_in_embed': True,
+            'view_count': int,
+            'like_count': int,
+            'channel_follower_count': int,
+            'thumbnail': 'https://i.ytimg.com/vi/OX9wJWOcqck/maxresdefault.jpg',
+            'categories': ['Entertainment'],
+            'tags': ['Rugrats'],
+        },
+    }, {
+        'url': 'https://www.paramountpressexpress.com/showtime/yt-video/?watch=_ljssSoDLkw',
+        'info_dict': {
+            'id': '_ljssSoDLkw',
+            'ext': 'mp4',
+            'title': 'Lavell Crawford: THEE Lavell Crawford Comedy Special Official Trailer | SHOWTIME',
+            'description': 'md5:39581bcc3fd810209b642609f448af70',
+            'uploader': 'SHOWTIME',
+            'uploader_id': '@Showtime',
+            'uploader_url': 'http://www.youtube.com/@Showtime',
+            'channel': 'SHOWTIME',
+            'channel_id': 'UCtwMWJr2BFPkuJTnSvCESSQ',
+            'channel_url': 'https://www.youtube.com/channel/UCtwMWJr2BFPkuJTnSvCESSQ',
+            'upload_date': '20230209',
+            'duration': 49,
+            'age_limit': 0,
+            'availability': 'public',
+            'live_status': 'not_live',
+            'playable_in_embed': True,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'channel_follower_count': int,
+            'thumbnail': 'https://i.ytimg.com/vi_webp/_ljssSoDLkw/maxresdefault.webp',
+            'categories': ['People & Blogs'],
+            'tags': 'count:27',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id, is_youtube = self._match_valid_url(url).group('id', 'yt')
+        if is_youtube:
+            return self.url_result(display_id, YoutubeIE)
+
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(
+            r'\bvideo_id\s*=\s*["\'](\d+)["\']\s*,', webpage, 'Brightcove ID')
+        token = self._search_regex(r'\btoken\s*=\s*["\']([\w.-]+)["\']', webpage, 'token')
+
+        player = extract_attributes(get_element_html_by_id('vcbrightcoveplayer', webpage) or '')
+        account_id = player.get('data-account') or '6055873637001'
+        player_id = player.get('data-player') or 'OtLKgXlO9F'
+        embed = player.get('data-embed') or 'default'
+
+        return self.url_result(smuggle_url(
+            f'https://players.brightcove.net/{account_id}/{player_id}_{embed}/index.html?videoId={video_id}',
+            {'token': token}), BrightcoveNewIE)

From 69b2f838d3d3e37dc17367ef64d978db1bea45cf Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 23 Mar 2023 11:19:37 -0500
Subject: [PATCH 177/871] [extractor/telecaribe] Expand livestream support
 (#6601)

Closes #6598
Authored by: bashonly
---
 yt_dlp/extractor/telecaribe.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/telecaribe.py b/yt_dlp/extractor/telecaribe.py
index b6d88a809..91118a1a4 100644
--- a/yt_dlp/extractor/telecaribe.py
+++ b/yt_dlp/extractor/telecaribe.py
@@ -38,11 +38,23 @@ class TelecaribePlayIE(InfoExtractor):
         'params': {
             'skip_download': 'Livestream',
         }
+    }, {
+        'url': 'https://www.play.telecaribe.co/liveplus',
+        'info_dict': {
+            'id': 'liveplus',
+            'title': r're:^Señal en vivo Plus',
+            'live_status': 'is_live',
+            'ext': 'mp4',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+        'skip': 'Geo-restricted to Colombia',
     }]
 
     def _download_player_webpage(self, webpage, display_id):
         page_id = self._search_regex(
-            (r'window.firstPageId\s*=\s*["\']([^"\']+)', r'<div[^>]+id\s*=\s*"pageBackground_([^"]+)'),
+            (r'window\.firstPageId\s*=\s*["\']([^"\']+)', r'<div[^>]+id\s*=\s*"pageBackground_([^"]+)'),
             webpage, 'page_id')
 
         props = self._download_json(self._search_regex(
@@ -59,14 +71,16 @@ def _real_extract(self, url):
         webpage = self._download_webpage(url, display_id)
         player = self._download_player_webpage(webpage, display_id)
 
-        if display_id != 'live':
+        livestream_url = self._search_regex(
+            r'(?:let|const|var)\s+source\s*=\s*["\']([^"\']+)', player, 'm3u8 url', default=None)
+
+        if not livestream_url:
             return self.playlist_from_matches(
                 re.findall(r'<a[^>]+href\s*=\s*"([^"]+\.mp4)', player), display_id,
                 self._get_clean_title(self._og_search_title(webpage)))
 
         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
-            self._search_regex(r'(?:let|const|var)\s+source\s*=\s*["\']([^"\']+)', player, 'm3u8 url'),
-            display_id, 'mp4')
+            livestream_url, display_id, 'mp4', live=True)
 
         return {
             'id': display_id,

From 78bc1868ff3352108ab2911033d1ac67a55f151e Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 23 Mar 2023 15:16:02 +0530
Subject: [PATCH 178/871] [extractor/rumble] Detect timeline format

Closes #6607
---
 yt_dlp/extractor/rumble.py | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py
index 97f81446c..834fe704f 100644
--- a/yt_dlp/extractor/rumble.py
+++ b/yt_dlp/extractor/rumble.py
@@ -7,6 +7,7 @@
     ExtractorError,
     UnsupportedError,
     clean_html,
+    determine_ext,
     get_element_by_class,
     int_or_none,
     parse_count,
@@ -175,12 +176,16 @@ def _real_extract(self, url):
                         video_info['url'], video_id,
                         ext='mp4', m3u8_id='hls', fatal=False, live=live_status == 'is_live'))
                     continue
+                timeline = ext == 'timeline'
+                if timeline:
+                    ext = determine_ext(video_info['url'])
                 formats.append({
                     'ext': ext,
+                    'acodec': 'none' if timeline else None,
                     'url': video_info['url'],
                     'format_id': '%s-%sp' % (ext, height),
-                    'height': int_or_none(height),
-                    'fps': video.get('fps'),
+                    'format_note': 'Timeline' if timeline else None,
+                    'fps': None if timeline else video.get('fps'),
                     **traverse_obj(meta, {
                         'tbr': 'bitrate',
                         'filesize': 'size',
@@ -247,6 +252,25 @@ class RumbleIE(InfoExtractor):
     }, {
         'url': 'http://www.rumble.com/vDMUM1?key=value',
         'only_matching': True,
+    }, {
+        'note': 'timeline format',
+        'url': 'https://rumble.com/v2ea9qb-the-u.s.-cannot-hide-this-in-ukraine-anymore-redacted-with-natali-and-clayt.html',
+        'md5': '40d61fec6c0945bca3d0e1dc1aa53d79',
+        'params': {'format': 'wv'},
+        'info_dict': {
+            'id': 'v2bou5f',
+            'ext': 'mp4',
+            'uploader': 'Redacted News',
+            'upload_date': '20230322',
+            'timestamp': 1679445010,
+            'title': 'The U.S. CANNOT hide this in Ukraine anymore | Redacted with Natali and Clayton Morris',
+            'duration': 892,
+            'channel': 'Redacted News',
+            'description': 'md5:aaad0c5c3426d7a361c29bdaaced7c42',
+            'channel_url': 'https://rumble.com/c/Redacted',
+            'live_status': 'not_live',
+            'thumbnail': 'https://sp.rmbl.ws/s8/1/d/x/2/O/dx2Oi.qR4e-small-The-U.S.-CANNOT-hide-this-i.jpg',
+        },
     }]
 
     _WEBPAGE_TESTS = [{

From 6994afc030d2a786d8032075ed71a14d7eac5a4f Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 23 Mar 2023 19:09:29 +0530
Subject: [PATCH 179/871] [extractor/rumble] Fix videos without quality
 selection

Closes #6612
---
 yt_dlp/extractor/rumble.py | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py
index 834fe704f..98f660f8b 100644
--- a/yt_dlp/extractor/rumble.py
+++ b/yt_dlp/extractor/rumble.py
@@ -8,8 +8,10 @@
     UnsupportedError,
     clean_html,
     determine_ext,
+    format_field,
     get_element_by_class,
     int_or_none,
+    join_nonempty,
     parse_count,
     parse_iso8601,
     traverse_obj,
@@ -165,7 +167,13 @@ def _real_extract(self, url):
 
         formats = []
         for ext, ext_info in (video.get('ua') or {}).items():
-            for height, video_info in (ext_info or {}).items():
+            if isinstance(ext_info, dict):
+                for height, video_info in ext_info.items():
+                    if not traverse_obj(video_info, ('meta', 'h', {int_or_none})):
+                        video_info.setdefault('meta', {})['h'] = height
+                ext_info = ext_info.values()
+
+            for video_info in ext_info:
                 meta = video_info.get('meta') or {}
                 if not video_info.get('url'):
                     continue
@@ -183,7 +191,7 @@ def _real_extract(self, url):
                     'ext': ext,
                     'acodec': 'none' if timeline else None,
                     'url': video_info['url'],
-                    'format_id': '%s-%sp' % (ext, height),
+                    'format_id': join_nonempty(ext, format_field(meta, 'h', '%sp')),
                     'format_note': 'Timeline' if timeline else None,
                     'fps': None if timeline else video.get('fps'),
                     **traverse_obj(meta, {
@@ -271,6 +279,24 @@ class RumbleIE(InfoExtractor):
             'live_status': 'not_live',
             'thumbnail': 'https://sp.rmbl.ws/s8/1/d/x/2/O/dx2Oi.qR4e-small-The-U.S.-CANNOT-hide-this-i.jpg',
         },
+    }, {
+        'url': 'https://rumble.com/v2e7fju-the-covid-twitter-files-drop-protecting-fauci-while-censoring-the-truth-wma.html',
+        'info_dict': {
+            'id': 'v2blzyy',
+            'ext': 'mp4',
+            'live_status': 'was_live',
+            'release_timestamp': 1679446804,
+            'description': 'md5:2ac4908ccfecfb921f8ffa4b30c1e636',
+            'release_date': '20230322',
+            'timestamp': 1679445692,
+            'duration': 4435,
+            'upload_date': '20230322',
+            'title': 'The Covid Twitter Files Drop: Protecting Fauci While Censoring The Truth w/Matt Taibbi',
+            'uploader': 'Kim Iversen',
+            'channel_url': 'https://rumble.com/c/KimIversen',
+            'channel': 'Kim Iversen',
+            'thumbnail': 'https://sp.rmbl.ws/s8/1/6/b/w/O/6bwOi.qR4e-small-The-Covid-Twitter-Files-Dro.jpg',
+        },
     }]
 
     _WEBPAGE_TESTS = [{

From 5cc0a8fd2e9fec50026fb92170b57993af939e4a Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 23 Mar 2023 11:28:23 -0500
Subject: [PATCH 180/871] [extractor/generic] Accept values for
 `fragment_query`, `variant_query` (#6600)

Closes #6593
Authored by: bashonly
---
 README.md                   |  4 ++--
 yt_dlp/extractor/generic.py | 20 ++++++++++++--------
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 9ce85d631..3e8484314 100644
--- a/README.md
+++ b/README.md
@@ -1797,8 +1797,8 @@ #### youtubetab (YouTube playlists, channels, feeds, etc.)
 * `approximate_date`: Extract approximate `upload_date` and `timestamp` in flat-playlist. This may cause date-based filters to be slightly off
 
 #### generic
-* `fragment_query`: Passthrough any query in mpd/m3u8 manifest URLs to their fragments. Does not apply to ffmpeg
-* `variant_query`: Passthrough the master m3u8 URL query to its variant playlist URLs
+* `fragment_query`: Passthrough any query in mpd/m3u8 manifest URLs to their fragments if no value is provided, or else apply the query string given as `fragment_query=VALUE`. Does not apply to ffmpeg
+* `variant_query`: Passthrough the master m3u8 URL query to its variant playlist URLs if no value is provided, or else apply the query string given as `variant_query=VALUE`
 * `hls_key`: An HLS AES-128 key URI *or* key (as hex), and optionally the IV (as hex), in the form of `(URI|KEY)[,IV]`; e.g. `generic:hls_key=ABCDEF1234567980,0xFEDCBA0987654321`. Passing any of these values will force usage of the native HLS downloader and override the corresponding values found in the m3u8 playlist
 
 #### funimation
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 075bb36de..f9fa01feb 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -24,7 +24,6 @@
     mimetype2ext,
     orderedSet,
     parse_duration,
-    parse_qs,
     parse_resolution,
     smuggle_url,
     str_or_none,
@@ -2187,18 +2186,23 @@ def report_detected(self, name, num=1, note=None):
         self._downloader.write_debug(f'Identified {num} {name}{format_field(note, None, "; %s")}')
 
     def _extra_manifest_info(self, info, manifest_url):
-        if self._configuration_arg('fragment_query'):
-            query_string = urllib.parse.urlparse(manifest_url).query
-            if query_string:
-                info['extra_param_to_segment_url'] = query_string
+        fragment_query = self._configuration_arg('fragment_query', [None], casesense=True)[0]
+        if fragment_query is not None:
+            fragment_query = self._configuration_arg('fragment_query', casesense=True)[0]
+            info['extra_param_to_segment_url'] = (
+                urllib.parse.urlparse(fragment_query).query or fragment_query
+                or urllib.parse.urlparse(manifest_url).query or None)
 
         hex_or_none = lambda x: x if re.fullmatch(r'(0x)?[\da-f]+', x, re.IGNORECASE) else None
-        info['hls_aes'] = traverse_obj(self._configuration_arg('hls_key'), {
+        info['hls_aes'] = traverse_obj(self._configuration_arg('hls_key', casesense=True), {
             'uri': (0, {url_or_none}), 'key': (0, {hex_or_none}), 'iv': (1, {hex_or_none}),
         }) or None
 
-        if self._configuration_arg('variant_query'):
-            query = parse_qs(manifest_url)
+        variant_query = self._configuration_arg('variant_query', [None], casesense=True)[0]
+        if variant_query is not None:
+            query = urllib.parse.parse_qs(
+                urllib.parse.urlparse(variant_query).query or variant_query
+                or urllib.parse.urlparse(manifest_url).query)
             for fmt in self._downloader._get_formats(info):
                 fmt['url'] = update_url_query(fmt['url'], query)
 

From 3ae182ad89e1427ff7b1684d6a44ff93fa857a0c Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 23 Mar 2023 13:45:27 -0500
Subject: [PATCH 181/871] [extractor/pgatour] Add extractor (#6613)

Closes #6537
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/pgatour.py     | 47 +++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 yt_dlp/extractor/pgatour.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 6c948e5fc..4a4d38caf 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1393,6 +1393,7 @@
     PeriscopeIE,
     PeriscopeUserIE,
 )
+from .pgatour import PGATourIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
diff --git a/yt_dlp/extractor/pgatour.py b/yt_dlp/extractor/pgatour.py
new file mode 100644
index 000000000..36c2c6207
--- /dev/null
+++ b/yt_dlp/extractor/pgatour.py
@@ -0,0 +1,47 @@
+from .brightcove import BrightcoveNewIE
+from .common import InfoExtractor
+
+
+class PGATourIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pgatour\.com/video/[\w-]+/(?P<tc>T)?(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.pgatour.com/video/competition/T6322447785112/adam-hadwin-2023-the-players-round-4-18th-hole-shot-1',
+        'info_dict': {
+            'id': '6322447785112',
+            'ext': 'mp4',
+            'title': 'Adam Hadwin | 2023 THE PLAYERS | Round 4 | 18th hole | Shot 1',
+            'uploader_id': '6116716431001',
+            'upload_date': '20230312',
+            'timestamp': 1678653136,
+            'duration': 20.011,
+            'thumbnail': r're:^https://.+\.jpg',
+            'tags': 'count:7',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.pgatour.com/video/features/6322506425112/follow-the-players-trophy-on-championship-sunday',
+        'info_dict': {
+            'id': '6322506425112',
+            'ext': 'mp4',
+            'title': 'Follow THE PLAYERS trophy on Championship Sunday',
+            'description': 'md5:4d29e4bdfa03694a0ebfd08950398568',
+            'uploader_id': '6082840763001',
+            'upload_date': '20230313',
+            'timestamp': 1678739835,
+            'duration': 123.435,
+            'thumbnail': r're:^https://.+\.jpg',
+            'tags': 'count:8',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        video_id, is_tourcast = self._match_valid_url(url).group('id', 'tc')
+
+        # From https://www.pgatour.com/_next/static/chunks/pages/_app-8bcf849560daf38d.js
+        account_id = '6116716431001' if is_tourcast else '6082840763001'
+        player_id = 'Vsd5Umu8r' if is_tourcast else 'FWIBYMBPj'
+
+        return self.url_result(
+            f'https://players.brightcove.net/{account_id}/{player_id}_default/index.html?videoId={video_id}',
+            BrightcoveNewIE)

From 6bdb64e2a2a6d504d8ce1dc830fbfb8a7f199c63 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 23 Mar 2023 13:45:56 -0500
Subject: [PATCH 182/871] [extractor/hollywoodreporter] Add extractors (#6614)

Closes #6525
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py       |  4 ++
 yt_dlp/extractor/hollywoodreporter.py | 72 +++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 yt_dlp/extractor/hollywoodreporter.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 4a4d38caf..69464b6f0 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -719,6 +719,10 @@
 from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hitrecord import HitRecordIE
+from .hollywoodreporter import (
+    HollywoodReporterIE,
+    HollywoodReporterPlaylistIE,
+)
 from .holodex import HolodexIE
 from .hotnewhiphop import HotNewHipHopIE
 from .hotstar import (
diff --git a/yt_dlp/extractor/hollywoodreporter.py b/yt_dlp/extractor/hollywoodreporter.py
new file mode 100644
index 000000000..1f7eb89bc
--- /dev/null
+++ b/yt_dlp/extractor/hollywoodreporter.py
@@ -0,0 +1,72 @@
+import functools
+import re
+
+from .common import InfoExtractor
+from .jwplatform import JWPlatformIE
+from ..utils import (
+    ExtractorError,
+    OnDemandPagedList,
+    extract_attributes,
+    get_element_by_class,
+    get_element_html_by_class,
+)
+
+
+class HollywoodReporterIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hollywoodreporter\.com/video/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.hollywoodreporter.com/video/chris-pine-michelle-rodriguez-dungeons-dragons-cast-directors-on-what-it-took-to-make-film-sxsw-2023/',
+        'info_dict': {
+            'id': 'zH4jZaR5',
+            'ext': 'mp4',
+            'title': 'md5:a9a1c073770a32f178955997712c4bd9',
+            'description': 'The cast and directors of \'Dungeons & Dragons: Honor Among Thieves\' talk about their new film.',
+            'thumbnail': 'https://cdn.jwplayer.com/v2/media/zH4jZaR5/poster.jpg?width=720',
+            'upload_date': '20230312',
+            'timestamp': 1678586423,
+            'duration': 242.0,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        data = extract_attributes(get_element_html_by_class('vlanding-video-card__link', webpage) or '')
+        video_id = data['data-video-showcase-trigger']
+        showcase_type = data['data-video-showcase-type']
+
+        if showcase_type == 'jwplayer':
+            return self.url_result(f'jwplatform:{video_id}', JWPlatformIE)
+        elif showcase_type == 'youtube':
+            return self.url_result(video_id, 'Youtube')
+        else:
+            raise ExtractorError(f'Unsupported showcase type "{showcase_type}"')
+
+
+class HollywoodReporterPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hollywoodreporter\.com/vcategory/(?P<slug>[\w-]+)-(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.hollywoodreporter.com/vcategory/heat-vision-breakdown-57822/',
+        'playlist_mincount': 109,
+        'info_dict': {
+            'id': '57822',
+            'title': 'heat-vision-breakdown',
+        }
+    }]
+
+    def _fetch_page(self, slug, pl_id, page):
+        page += 1
+        webpage = self._download_webpage(
+            f'https://www.hollywoodreporter.com/vcategory/{slug}-{pl_id}/page/{page}/',
+            pl_id, note=f'Downloading playlist page {page}')
+        section = get_element_by_class('video-playlist-river', webpage) or ''
+
+        for url in re.findall(r'<a[^>]+href="([^"]+)"[^>]+class="c-title__link', section):
+            yield self.url_result(url, HollywoodReporterIE)
+
+    def _real_extract(self, url):
+        slug, pl_id = self._match_valid_url(url).group('slug', 'id')
+        return self.playlist_result(
+            OnDemandPagedList(functools.partial(self._fetch_page, slug, pl_id), 15), pl_id, slug)

From 8ceb07e870424c219dced8f4348729553f05c5cc Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 23 Mar 2023 13:46:33 -0500
Subject: [PATCH 183/871] [extractor/tiktok] Fix mp3 formats (#6615)

Closes #6608
Authored by: bashonly
---
 yt_dlp/extractor/tiktok.py | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index f1696a2fc..fb838d529 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -13,6 +13,7 @@
     LazyList,
     UnsupportedError,
     UserNotLive,
+    determine_ext,
     format_field,
     get_element_by_id,
     get_first,
@@ -204,6 +205,16 @@ def parse_url_key(url_key):
 
         known_resolutions = {}
 
+        def mp3_meta(url):
+            return {
+                'format_note': 'Music track',
+                'ext': 'mp3',
+                'acodec': 'mp3',
+                'vcodec': 'none',
+                'width': None,
+                'height': None,
+            } if determine_ext(url) == 'mp3' else {}
+
         def extract_addr(addr, add_meta={}):
             parsed_meta, res = parse_url_key(addr.get('url_key', ''))
             if res:
@@ -219,7 +230,8 @@ def extract_addr(addr, add_meta={}):
                 'source_preference': -2 if 'aweme/v1' in url else -1,  # Downloads from API might get blocked
                 **add_meta, **parsed_meta,
                 'format_note': join_nonempty(
-                    add_meta.get('format_note'), '(API)' if 'aweme/v1' in url else None, delim=' ')
+                    add_meta.get('format_note'), '(API)' if 'aweme/v1' in url else None, delim=' '),
+                **mp3_meta(url),
             } for url in addr.get('url_list') or []]
 
         # Hack: Add direct video links first to prioritize them when removing duplicate formats
@@ -553,6 +565,28 @@ class TikTokIE(TikTokBaseIE):
             'comment_count': int,
         },
         'skip': 'This video is unavailable',
+    }, {
+        # slideshow audio-only mp3 format
+        'url': 'https://www.tiktok.com/@_le_cannibale_/video/7139980461132074283',
+        'info_dict': {
+            'id': '7139980461132074283',
+            'ext': 'mp3',
+            'title': 'TikTok video #7139980461132074283',
+            'description': '',
+            'creator': 'Antaura',
+            'uploader': '_le_cannibale_',
+            'uploader_id': '6604511138619654149',
+            'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAoShJqaw_5gvy48y3azFeFcT4jeyKWbB0VVYasOCt2tTLwjNFIaDcHAM4D-QGXFOP',
+            'artist': 'nathan !',
+            'track': 'grahamscott canon',
+            'upload_date': '20220905',
+            'timestamp': 1662406249,
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+            'thumbnail': r're:^https://.+\.webp',
+        },
     }, {
         # Auto-captions available
         'url': 'https://www.tiktok.com/@hankgreen1/video/7047596209028074758',

From 9bfe0d15bd7dbdc6b0e6378fa9f5e2e289b2373b Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Thu, 23 Mar 2023 14:28:31 -0500
Subject: [PATCH 184/871] Fix 5cc0a8fd2e9fec50026fb92170b57993af939e4a

Authored by: bashonly
---
 yt_dlp/extractor/generic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index f9fa01feb..75355aeb5 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -2188,7 +2188,6 @@ def report_detected(self, name, num=1, note=None):
     def _extra_manifest_info(self, info, manifest_url):
         fragment_query = self._configuration_arg('fragment_query', [None], casesense=True)[0]
         if fragment_query is not None:
-            fragment_query = self._configuration_arg('fragment_query', casesense=True)[0]
             info['extra_param_to_segment_url'] = (
                 urllib.parse.urlparse(fragment_query).query or fragment_query
                 or urllib.parse.urlparse(manifest_url).query or None)

From baa922b5c74b10e3b86ff5e6cf6529b3aae8efab Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 24 Mar 2023 21:53:45 +0530
Subject: [PATCH 185/871] [extractor] Do not exit early for unsuitable
 `url_result`

---
 yt_dlp/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 2091df7fa..5da12725a 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -3513,8 +3513,8 @@ def _RETURN_TYPE(cls):
     @classmethod
     def is_single_video(cls, url):
         """Returns whether the URL is of a single video, None if unknown"""
-        assert cls.suitable(url), 'The URL must be suitable for the extractor'
-        return {'video': True, 'playlist': False}.get(cls._RETURN_TYPE)
+        if cls.suitable(url):
+            return {'video': True, 'playlist': False}.get(cls._RETURN_TYPE)
 
     @classmethod
     def is_suitable(cls, age_limit):

From f68434cc74cfd3db01b266476a2eac8329fbb267 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 24 Mar 2023 21:53:06 +0530
Subject: [PATCH 186/871] [extractor] Extract more metadata from ISM

Fixes https://github.com/yt-dlp/yt-dlp/commit/81b6102d2099eec78a2db9ae3d101a8503dd4f25#r105892531
---
 test/test_InfoExtractor.py | 138 +++++++++++++------------------------
 yt_dlp/extractor/common.py |   2 +
 2 files changed, 49 insertions(+), 91 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index e8d94a6ac..1f60abfd2 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -1406,6 +1406,7 @@ def test_parse_ism_formats(self):
                     'vcodec': 'none',
                     'acodec': 'AACL',
                     'protocol': 'ism',
+                    'audio_channels': 2,
                     '_download_params': {
                         'stream_type': 'audio',
                         'duration': 8880746666,
@@ -1419,9 +1420,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'audio_ext': 'isma',
-                    'video_ext': 'none',
-                    'abr': 128,
                 }, {
                     'format_id': 'video-100',
                     'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/Manifest',
@@ -1445,9 +1443,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 100,
                 }, {
                     'format_id': 'video-326',
                     'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/Manifest',
@@ -1471,9 +1466,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 326,
                 }, {
                     'format_id': 'video-698',
                     'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/Manifest',
@@ -1497,9 +1489,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 698,
                 }, {
                     'format_id': 'video-1493',
                     'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/Manifest',
@@ -1523,9 +1512,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 1493,
                 }, {
                     'format_id': 'video-4482',
                     'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/Manifest',
@@ -1549,9 +1535,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 4482,
                 }],
                 {
                     'eng': [
@@ -1575,34 +1558,6 @@ def test_parse_ism_formats(self):
                 'ec-3_test',
                 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
                 [{
-                    'format_id': 'audio_deu_1-224',
-                    'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
-                    'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
-                    'ext': 'isma',
-                    'tbr': 224,
-                    'asr': 48000,
-                    'vcodec': 'none',
-                    'acodec': 'EC-3',
-                    'protocol': 'ism',
-                    '_download_params':
-                    {
-                        'stream_type': 'audio',
-                        'duration': 370000000,
-                        'timescale': 10000000,
-                        'width': 0,
-                        'height': 0,
-                        'fourcc': 'EC-3',
-                        'language': 'deu',
-                        'codec_private_data': '00063F000000AF87FBA7022DFB42A4D405CD93843BDD0700200F00',
-                        'sampling_rate': 48000,
-                        'channels': 6,
-                        'bits_per_sample': 16,
-                        'nal_unit_length_field': 4
-                    },
-                    'audio_ext': 'isma',
-                    'video_ext': 'none',
-                    'abr': 224,
-                }, {
                     'format_id': 'audio_deu-127',
                     'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
                     'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
@@ -1612,8 +1567,9 @@ def test_parse_ism_formats(self):
                     'vcodec': 'none',
                     'acodec': 'AACL',
                     'protocol': 'ism',
-                    '_download_params':
-                    {
+                    'language': 'deu',
+                    'audio_channels': 2,
+                    '_download_params': {
                         'stream_type': 'audio',
                         'duration': 370000000,
                         'timescale': 10000000,
@@ -1627,9 +1583,32 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'audio_ext': 'isma',
-                    'video_ext': 'none',
-                    'abr': 127,
+                }, {
+                    'format_id': 'audio_deu_1-224',
+                    'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
+                    'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
+                    'ext': 'isma',
+                    'tbr': 224,
+                    'asr': 48000,
+                    'vcodec': 'none',
+                    'acodec': 'EC-3',
+                    'protocol': 'ism',
+                    'language': 'deu',
+                    'audio_channels': 6,
+                    '_download_params': {
+                        'stream_type': 'audio',
+                        'duration': 370000000,
+                        'timescale': 10000000,
+                        'width': 0,
+                        'height': 0,
+                        'fourcc': 'EC-3',
+                        'language': 'deu',
+                        'codec_private_data': '00063F000000AF87FBA7022DFB42A4D405CD93843BDD0700200F00',
+                        'sampling_rate': 48000,
+                        'channels': 6,
+                        'bits_per_sample': 16,
+                        'nal_unit_length_field': 4
+                    },
                 }, {
                     'format_id': 'video_deu-23',
                     'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
@@ -1641,8 +1620,8 @@ def test_parse_ism_formats(self):
                     'vcodec': 'AVC1',
                     'acodec': 'none',
                     'protocol': 'ism',
-                    '_download_params':
-                    {
+                    'language': 'deu',
+                    '_download_params': {
                         'stream_type': 'video',
                         'duration': 370000000,
                         'timescale': 10000000,
@@ -1655,9 +1634,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 23,
                 }, {
                     'format_id': 'video_deu-403',
                     'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
@@ -1669,8 +1645,8 @@ def test_parse_ism_formats(self):
                     'vcodec': 'AVC1',
                     'acodec': 'none',
                     'protocol': 'ism',
-                    '_download_params':
-                    {
+                    'language': 'deu',
+                    '_download_params': {
                         'stream_type': 'video',
                         'duration': 370000000,
                         'timescale': 10000000,
@@ -1683,9 +1659,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 403,
                 }, {
                     'format_id': 'video_deu-680',
                     'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
@@ -1697,8 +1670,8 @@ def test_parse_ism_formats(self):
                     'vcodec': 'AVC1',
                     'acodec': 'none',
                     'protocol': 'ism',
-                    '_download_params':
-                    {
+                    'language': 'deu',
+                    '_download_params': {
                         'stream_type': 'video',
                         'duration': 370000000,
                         'timescale': 10000000,
@@ -1711,9 +1684,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 680,
                 }, {
                     'format_id': 'video_deu-1253',
                     'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
@@ -1725,8 +1695,9 @@ def test_parse_ism_formats(self):
                     'vcodec': 'AVC1',
                     'acodec': 'none',
                     'protocol': 'ism',
-                    '_download_params':
-                    {
+                    'vbr': 1253,
+                    'language': 'deu',
+                    '_download_params': {
                         'stream_type': 'video',
                         'duration': 370000000,
                         'timescale': 10000000,
@@ -1739,9 +1710,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 1253,
                 }, {
                     'format_id': 'video_deu-2121',
                     'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
@@ -1753,8 +1721,8 @@ def test_parse_ism_formats(self):
                     'vcodec': 'AVC1',
                     'acodec': 'none',
                     'protocol': 'ism',
-                    '_download_params':
-                    {
+                    'language': 'deu',
+                    '_download_params': {
                         'stream_type': 'video',
                         'duration': 370000000,
                         'timescale': 10000000,
@@ -1767,9 +1735,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 2121,
                 }, {
                     'format_id': 'video_deu-3275',
                     'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
@@ -1781,8 +1746,8 @@ def test_parse_ism_formats(self):
                     'vcodec': 'AVC1',
                     'acodec': 'none',
                     'protocol': 'ism',
-                    '_download_params':
-                    {
+                    'language': 'deu',
+                    '_download_params': {
                         'stream_type': 'video',
                         'duration': 370000000,
                         'timescale': 10000000,
@@ -1795,9 +1760,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 3275,
                 }, {
                     'format_id': 'video_deu-5300',
                     'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
@@ -1809,8 +1771,8 @@ def test_parse_ism_formats(self):
                     'vcodec': 'AVC1',
                     'acodec': 'none',
                     'protocol': 'ism',
-                    '_download_params':
-                    {
+                    'language': 'deu',
+                    '_download_params': {
                         'stream_type': 'video',
                         'duration': 370000000,
                         'timescale': 10000000,
@@ -1823,9 +1785,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 5300,
                 }, {
                     'format_id': 'video_deu-8079',
                     'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest',
@@ -1837,8 +1796,8 @@ def test_parse_ism_formats(self):
                     'vcodec': 'AVC1',
                     'acodec': 'none',
                     'protocol': 'ism',
-                    '_download_params':
-                    {
+                    'language': 'deu',
+                    '_download_params': {
                         'stream_type': 'video',
                         'duration': 370000000,
                         'timescale': 10000000,
@@ -1851,9 +1810,6 @@ def test_parse_ism_formats(self):
                         'bits_per_sample': 16,
                         'nal_unit_length_field': 4
                     },
-                    'video_ext': 'ismv',
-                    'audio_ext': 'none',
-                    'vbr': 8079,
                 }],
                 {},
             ),
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 5da12725a..838899052 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -2983,6 +2983,8 @@ def _parse_ism_formats_and_subtitles(self, ism_doc, ism_url, ism_id=None):
                         'protocol': 'ism',
                         'fragments': fragments,
                         'has_drm': ism_doc.find('Protection') is not None,
+                        'language': stream_language,
+                        'audio_channels': int_or_none(track.get('Channels')),
                         '_download_params': {
                             'stream_type': stream_type,
                             'duration': duration,

From 0898c5c8ccadfc404472456a7a7751b72afebadd Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 25 Mar 2023 19:41:28 +0100
Subject: [PATCH 187/871] [utils] `js_to_json`: Implement template strings
 (#6623)

Authored by: Grub4K
---
 test/test_utils.py |  7 +++++++
 yt_dlp/utils.py    | 11 +++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3045b6d7e..d4a301583 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1190,6 +1190,13 @@ def test_js_to_json_malformed(self):
         self.assertEqual(js_to_json('42a1'), '42"a1"')
         self.assertEqual(js_to_json('42a-1'), '42"a"-1')
 
+    def test_js_to_json_template_literal(self):
+        self.assertEqual(js_to_json('`Hello ${name}`', {'name': '"world"'}), '"Hello world"')
+        self.assertEqual(js_to_json('`${name}${name}`', {'name': '"X"'}), '"XX"')
+        self.assertEqual(js_to_json('`${name}${name}`', {'name': '5'}), '"55"')
+        self.assertEqual(js_to_json('`${name}"${name}"`', {'name': '5'}), '"5\\"5\\""')
+        self.assertEqual(js_to_json('`${name}`', {}), '"name"')
+
     def test_extract_attributes(self):
         self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
         self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 8c2c5593c..40533c2cb 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3366,7 +3366,7 @@ def strip_jsonp(code):
 
 def js_to_json(code, vars={}, *, strict=False):
     # vars is a dict of var, val pairs to substitute
-    STRING_QUOTES = '\'"'
+    STRING_QUOTES = '\'"`'
     STRING_RE = '|'.join(rf'{q}(?:\\.|[^\\{q}])*{q}' for q in STRING_QUOTES)
     COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*\n'
     SKIP_RE = fr'\s*(?:{COMMENT_RE})?\s*'
@@ -3384,6 +3384,12 @@ def process_escape(match):
                 else '' if escape == '\n'
                 else escape)
 
+    def template_substitute(match):
+        evaluated = js_to_json(match.group(1), vars, strict=strict)
+        if evaluated[0] == '"':
+            return json.loads(evaluated)
+        return evaluated
+
     def fix_kv(m):
         v = m.group(0)
         if v in ('true', 'false', 'null'):
@@ -3394,7 +3400,8 @@ def fix_kv(m):
             return ''
 
         if v[0] in STRING_QUOTES:
-            escaped = re.sub(r'(?s)(")|\\(.)', process_escape, v[1:-1])
+            v = re.sub(r'(?s)\${([^}]+)}', template_substitute, v[1:-1]) if v[0] == '`' else v[1:-1]
+            escaped = re.sub(r'(?s)(")|\\(.)', process_escape, v)
             return f'"{escaped}"'
 
         for regex, base in INTEGER_TABLE:

From 33b737bedf8383c0d00d4e1d06a5273dcdfdb756 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 26 Mar 2023 17:16:42 -0500
Subject: [PATCH 188/871] [extractor/triller] Support short URLs, detect
 removed videos (#6636)

Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/triller.py     | 307 ++++++++++++++++++--------------
 2 files changed, 174 insertions(+), 134 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 69464b6f0..a97c458fa 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1962,6 +1962,7 @@
 from .triller import (
     TrillerIE,
     TrillerUserIE,
+    TrillerShortIE,
 )
 from .trilulilu import TriluliluIE
 from .trovo import (
diff --git a/yt_dlp/extractor/triller.py b/yt_dlp/extractor/triller.py
index acd9e68d2..6a4dadb9b 100644
--- a/yt_dlp/extractor/triller.py
+++ b/yt_dlp/extractor/triller.py
@@ -1,15 +1,21 @@
 import itertools
 import json
+import re
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    HEADRequest,
+    UnsupportedError,
+    determine_ext,
     int_or_none,
+    parse_resolution,
     str_or_none,
     traverse_obj,
-    unified_strdate,
     unified_timestamp,
     url_basename,
+    urljoin,
+    url_or_none,
 )
 
 
@@ -22,25 +28,22 @@ def _perform_login(self, username, password):
         if self._API_HEADERS.get('Authorization'):
             return
 
-        user_check = self._download_json(
+        headers = {**self._API_HEADERS, 'Content-Type': 'application/json'}
+        user_check = traverse_obj(self._download_json(
             f'{self._API_BASE_URL}/api/user/is-valid-username', None, note='Checking username',
-            fatal=False, expected_status=400, headers={
-                'Content-Type': 'application/json',
-                'Origin': 'https://triller.co',
-            }, data=json.dumps({'username': username}, separators=(',', ':')).encode('utf-8'))
-        if user_check.get('status'):  # endpoint returns "status":false if username exists
+            fatal=False, expected_status=400, headers=headers,
+            data=json.dumps({'username': username}, separators=(',', ':')).encode()), 'status')
+
+        if user_check:  # endpoint returns `"status":false` if username exists
             raise ExtractorError('Unable to login: Invalid username', expected=True)
 
-        credentials = {
-            'username': username,
-            'password': password,
-        }
         login = self._download_json(
-            f'{self._API_BASE_URL}/user/auth', None, note='Logging in',
-            fatal=False, expected_status=400, headers={
-                'Content-Type': 'application/json',
-                'Origin': 'https://triller.co',
-            }, data=json.dumps(credentials, separators=(',', ':')).encode('utf-8'))
+            f'{self._API_BASE_URL}/user/auth', None, note='Logging in', fatal=False,
+            expected_status=400, headers=headers, data=json.dumps({
+                'username': username,
+                'password': password,
+            }, separators=(',', ':')).encode()) or {}
+
         if not login.get('auth_token'):
             if login.get('error') == 1008:
                 raise ExtractorError('Unable to login: Incorrect password', expected=True)
@@ -55,100 +58,100 @@ def _get_comments(self, video_id, limit=15):
             headers=self._API_HEADERS, query={'limit': limit}) or {}
         if not comment_info.get('comments'):
             return
-        for comment_dict in comment_info['comments']:
-            yield {
-                'author': traverse_obj(comment_dict, ('author', 'username')),
-                'author_id': traverse_obj(comment_dict, ('author', 'user_id')),
-                'id': comment_dict.get('id'),
-                'text': comment_dict.get('body'),
-                'timestamp': unified_timestamp(comment_dict.get('timestamp')),
-            }
+        yield from traverse_obj(comment_info, ('comments', ..., {
+            'id': ('id', {str_or_none}),
+            'text': 'body',
+            'author': ('author', 'username'),
+            'author_id': ('author', 'user_id'),
+            'timestamp': ('timestamp', {unified_timestamp}),
+        }))
 
     def _check_user_info(self, user_info):
-        if not user_info:
-            self.report_warning('Unable to extract user info')
-        elif user_info.get('private') and not user_info.get('followed_by_me'):
+        if user_info.get('private') and not user_info.get('followed_by_me'):
             raise ExtractorError('This video is private', expected=True)
         elif traverse_obj(user_info, 'blocked_by_user', 'blocking_user'):
             raise ExtractorError('The author of the video is blocked', expected=True)
         return user_info
 
-    def _parse_video_info(self, video_info, username, user_info=None):
-        video_uuid = video_info.get('video_uuid')
-        video_id = video_info.get('id')
+    def _parse_video_info(self, video_info, username, user_id, display_id=None):
+        video_id = str(video_info['id'])
+        display_id = display_id or video_info.get('video_uuid')
+
+        if traverse_obj(video_info, (
+                None, ('transcoded_url', 'video_url', 'stream_url', 'audio_url'),
+                {lambda x: re.search(r'/copyright/', x)}), get_all=False):
+            self.raise_no_formats('This video has been removed due to licensing restrictions', expected=True)
+
+        def format_info(url):
+            return {
+                'url': url,
+                'ext': determine_ext(url),
+                'format_id': url_basename(url).split('.')[0],
+            }
 
         formats = []
-        video_url = traverse_obj(video_info, 'video_url', 'stream_url')
-        if video_url:
+
+        if determine_ext(video_info.get('transcoded_url')) == 'm3u8':
+            formats.extend(self._extract_m3u8_formats(
+                video_info['transcoded_url'], video_id, 'mp4', m3u8_id='hls', fatal=False))
+
+        for video in traverse_obj(video_info, ('video_set', lambda _, v: url_or_none(v['url']))):
             formats.append({
-                'url': video_url,
-                'ext': 'mp4',
-                'vcodec': 'h264',
-                'width': video_info.get('width'),
-                'height': video_info.get('height'),
-                'format_id': url_basename(video_url).split('.')[0],
-                'filesize': video_info.get('filesize'),
-            })
-        video_set = video_info.get('video_set') or []
-        for video in video_set:
-            resolution = video.get('resolution') or ''
-            formats.append({
-                'url': video['url'],
-                'ext': 'mp4',
+                **format_info(video['url']),
+                **parse_resolution(video.get('resolution')),
                 'vcodec': video.get('codec'),
                 'vbr': int_or_none(video.get('bitrate'), 1000),
-                'width': int_or_none(resolution.split('x')[0]),
-                'height': int_or_none(resolution.split('x')[1]),
-                'format_id': url_basename(video['url']).split('.')[0],
             })
-        audio_url = video_info.get('audio_url')
-        if audio_url:
+
+        video_url = traverse_obj(video_info, 'video_url', 'stream_url', expected_type=url_or_none)
+        if video_url:
             formats.append({
-                'url': audio_url,
-                'ext': 'm4a',
-                'format_id': url_basename(audio_url).split('.')[0],
+                **format_info(video_url),
+                'vcodec': 'h264',
+                **traverse_obj(video_info, {
+                    'width': 'width',
+                    'height': 'height',
+                    'filesize': 'filesize',
+                }, expected_type=int_or_none),
             })
 
-        manifest_url = video_info.get('transcoded_url')
-        if manifest_url:
-            formats.extend(self._extract_m3u8_formats(
-                manifest_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                m3u8_id='hls', fatal=False))
+        audio_url = url_or_none(video_info.get('audio_url'))
+        if audio_url:
+            formats.append(format_info(audio_url))
 
-        comment_count = int_or_none(video_info.get('comment_count'))
-
-        user_info = user_info or traverse_obj(video_info, 'user', default={})
+        comment_count = traverse_obj(video_info, ('comment_count', {int_or_none}))
 
         return {
-            'id': str_or_none(video_id) or video_uuid,
-            'title': video_info.get('description') or f'Video by {username}',
-            'thumbnail': video_info.get('thumbnail_url'),
-            'description': video_info.get('description'),
-            'uploader': str_or_none(username),
-            'uploader_id': str_or_none(user_info.get('user_id')),
-            'creator': str_or_none(user_info.get('name')),
-            'timestamp': unified_timestamp(video_info.get('timestamp')),
-            'upload_date': unified_strdate(video_info.get('timestamp')),
-            'duration': int_or_none(video_info.get('duration')),
-            'view_count': int_or_none(video_info.get('play_count')),
-            'like_count': int_or_none(video_info.get('likes_count')),
-            'artist': str_or_none(video_info.get('song_artist')),
-            'track': str_or_none(video_info.get('song_title')),
-            'webpage_url': f'https://triller.co/@{username}/video/{video_uuid}',
+            'id': video_id,
+            'display_id': display_id,
+            'uploader': username,
+            'uploader_id': user_id or traverse_obj(video_info, ('user', 'user_id', {str_or_none})),
+            'webpage_url': urljoin(f'https://triller.co/@{username}/video/', display_id),
             'uploader_url': f'https://triller.co/@{username}',
             'extractor_key': TrillerIE.ie_key(),
             'extractor': TrillerIE.IE_NAME,
             'formats': formats,
             'comment_count': comment_count,
             '__post_extractor': self.extract_comments(video_id, comment_count),
+            **traverse_obj(video_info, {
+                'title': ('description', {lambda x: x.replace('\r\n', ' ')}),
+                'description': 'description',
+                'creator': ((('user'), ('users', lambda _, v: str(v['user_id']) == user_id)), 'name'),
+                'thumbnail': ('thumbnail_url', {url_or_none}),
+                'timestamp': ('timestamp', {unified_timestamp}),
+                'duration': ('duration', {int_or_none}),
+                'view_count': ('play_count', {int_or_none}),
+                'like_count': ('likes_count', {int_or_none}),
+                'artist': 'song_artist',
+                'track': 'song_title',
+            }, get_all=False),
         }
 
 
 class TrillerIE(TrillerBaseIE):
     _VALID_URL = r'''(?x)
             https?://(?:www\.)?triller\.co/
-            @(?P<username>[\w\._]+)/video/
-            (?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})
+            @(?P<username>[\w.]+)/video/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})
         '''
     _TESTS = [{
         'url': 'https://triller.co/@theestallion/video/2358fcd7-3df2-4c77-84c8-1d091610a6cf',
@@ -165,16 +168,14 @@ class TrillerIE(TrillerBaseIE):
             'timestamp': 1660598222,
             'upload_date': '20220815',
             'duration': 47,
-            'height': 3840,
-            'width': 2160,
             'view_count': int,
             'like_count': int,
             'artist': 'Megan Thee Stallion',
             'track': 'Her',
-            'webpage_url': 'https://triller.co/@theestallion/video/2358fcd7-3df2-4c77-84c8-1d091610a6cf',
             'uploader_url': 'https://triller.co/@theestallion',
             'comment_count': int,
-        }
+        },
+        'skip': 'This video has been removed due to licensing restrictions',
     }, {
         'url': 'https://triller.co/@charlidamelio/video/46c6fcfa-aa9e-4503-a50c-68444f44cddc',
         'md5': '874055f462af5b0699b9dbb527a505a0',
@@ -182,6 +183,7 @@ class TrillerIE(TrillerBaseIE):
             'id': '71621339',
             'ext': 'mp4',
             'title': 'md5:4c91ea82760fe0fffb71b8c3aa7295fc',
+            'display_id': '46c6fcfa-aa9e-4503-a50c-68444f44cddc',
             'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
             'description': 'md5:4c91ea82760fe0fffb71b8c3aa7295fc',
             'uploader': 'charlidamelio',
@@ -190,59 +192,75 @@ class TrillerIE(TrillerBaseIE):
             'timestamp': 1660773354,
             'upload_date': '20220817',
             'duration': 16,
-            'height': 1920,
-            'width': 1080,
             'view_count': int,
             'like_count': int,
             'artist': 'Dixie',
             'track': 'Someone to Blame',
-            'webpage_url': 'https://triller.co/@charlidamelio/video/46c6fcfa-aa9e-4503-a50c-68444f44cddc',
             'uploader_url': 'https://triller.co/@charlidamelio',
             'comment_count': int,
-        }
+        },
+    }, {
+        'url': 'https://triller.co/@theestallion/video/07f35f38-1f51-48e2-8c5f-f7a8e829988f',
+        'md5': 'af7b3553e4b8bfca507636471ee2eb41',
+        'info_dict': {
+            'id': '71837829',
+            'ext': 'mp4',
+            'title': 'UNGRATEFUL VIDEO OUT NOW 👏🏾👏🏾👏🏾 💙💙 link my bio  #womeninhiphop',
+            'display_id': '07f35f38-1f51-48e2-8c5f-f7a8e829988f',
+            'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
+            'description': 'UNGRATEFUL VIDEO OUT NOW 👏🏾👏🏾👏🏾 💙💙 link my bio\r\n #womeninhiphop',
+            'uploader': 'theestallion',
+            'uploader_id': '18992236',
+            'creator': 'Megan Thee Stallion',
+            'timestamp': 1662486178,
+            'upload_date': '20220906',
+            'duration': 30,
+            'view_count': int,
+            'like_count': int,
+            'artist': 'Unknown',
+            'track': 'Unknown',
+            'uploader_url': 'https://triller.co/@theestallion',
+            'comment_count': int,
+        },
     }]
 
     def _real_extract(self, url):
-        username, video_uuid = self._match_valid_url(url).group('username', 'id')
+        username, display_id = self._match_valid_url(url).group('username', 'id')
 
-        video_info = traverse_obj(self._download_json(
-            f'{self._API_BASE_URL}/api/videos/{video_uuid}',
-            video_uuid, note='Downloading video info API JSON',
-            errnote='Unable to download video info API JSON',
-            headers=self._API_HEADERS), ('videos', 0))
-        if not video_info:
-            raise ExtractorError('No video info found in API response')
+        video_info = self._download_json(
+            f'{self._API_BASE_URL}/api/videos/{display_id}', display_id,
+            headers=self._API_HEADERS)['videos'][0]
 
-        user_info = self._check_user_info(video_info.get('user') or {})
-        return self._parse_video_info(video_info, username, user_info)
+        self._check_user_info(video_info.get('user') or {})
+
+        return self._parse_video_info(video_info, username, None, display_id)
 
 
 class TrillerUserIE(TrillerBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?triller\.co/@(?P<id>[\w\._]+)/?(?:$|[#?])'
+    _VALID_URL = r'https?://(?:www\.)?triller\.co/@(?P<id>[\w.]+)/?(?:$|[#?])'
     _TESTS = [{
-        # first videos request only returns 2 videos
         'url': 'https://triller.co/@theestallion',
-        'playlist_mincount': 9,
+        'playlist_mincount': 12,
         'info_dict': {
             'id': '18992236',
             'title': 'theestallion',
             'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
-        }
+        },
     }, {
         'url': 'https://triller.co/@charlidamelio',
-        'playlist_mincount': 25,
+        'playlist_mincount': 150,
         'info_dict': {
             'id': '1875551',
             'title': 'charlidamelio',
             'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
-        }
+        },
     }]
 
     def _real_initialize(self):
         if not self._API_HEADERS.get('Authorization'):
             guest = self._download_json(
-                f'{self._API_BASE_URL}/user/create_guest',
-                None, note='Creating guest session', data=b'', headers=self._API_HEADERS, query={
+                f'{self._API_BASE_URL}/user/create_guest', None,
+                note='Creating guest session', data=b'', headers=self._API_HEADERS, query={
                     'platform': 'Web',
                     'app_version': '',
                 })
@@ -251,44 +269,65 @@ def _real_initialize(self):
 
             self._API_HEADERS['Authorization'] = f'Bearer {guest["auth_token"]}'
 
-    def _extract_video_list(self, username, user_id, limit=6):
-        query = {
-            'limit': limit,
-        }
+    def _entries(self, username, user_id, limit=6):
+        query = {'limit': limit}
         for page in itertools.count(1):
-            for retry in self.RetryManager():
-                try:
-                    video_list = self._download_json(
-                        f'{self._API_BASE_URL}/api/users/{user_id}/videos',
-                        username, note=f'Downloading user video list page {page}',
-                        errnote='Unable to download user video list', headers=self._API_HEADERS,
-                        query=query)
-                except ExtractorError as e:
-                    if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0:
-                        retry.error = e
-                        continue
-                    raise
-            if not video_list.get('videos'):
-                break
-            yield from video_list['videos']
-            query['before_time'] = traverse_obj(video_list, ('videos', -1, 'timestamp'))
+            videos = self._download_json(
+                f'{self._API_BASE_URL}/api/users/{user_id}/videos',
+                username, note=f'Downloading user video list page {page}',
+                headers=self._API_HEADERS, query=query)
+
+            for video in traverse_obj(videos, ('videos', ...)):
+                yield self._parse_video_info(video, username, user_id)
+
+            query['before_time'] = traverse_obj(videos, ('videos', -1, 'timestamp'))
             if not query['before_time']:
                 break
 
-    def _entries(self, videos, username, user_info):
-        for video in videos:
-            yield self._parse_video_info(video, username, user_info)
-
     def _real_extract(self, url):
         username = self._match_id(url)
+
         user_info = self._check_user_info(self._download_json(
             f'{self._API_BASE_URL}/api/users/by_username/{username}',
-            username, note='Downloading user info',
-            errnote='Failed to download user info', headers=self._API_HEADERS).get('user', {}))
+            username, note='Downloading user info', headers=self._API_HEADERS)['user'])
 
         user_id = str_or_none(user_info.get('user_id'))
-        videos = self._extract_video_list(username, user_id)
-        thumbnail = user_info.get('avatar_url')
+        if not user_id:
+            raise ExtractorError('Unable to extract user ID')
 
         return self.playlist_result(
-            self._entries(videos, username, user_info), user_id, username, thumbnail=thumbnail)
+            self._entries(username, user_id), user_id, username, thumbnail=user_info.get('avatar_url'))
+
+
+class TrillerShortIE(InfoExtractor):
+    _VALID_URL = r'https?://v\.triller\.co/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://v.triller.co/WWZNWk',
+        'md5': '5eb8dc2c971bd8cd794ec9e8d5e9d101',
+        'info_dict': {
+            'id': '66210052',
+            'ext': 'mp4',
+            'title': 'md5:2dfc89d154cd91a4a18cd9582ba03e16',
+            'display_id': 'f4480e1f-fb4e-45b9-a44c-9e6c679ce7eb',
+            'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
+            'description': 'md5:2dfc89d154cd91a4a18cd9582ba03e16',
+            'uploader': 'statefairent',
+            'uploader_id': '487545193',
+            'creator': 'Official Summer Fair of LA',
+            'timestamp': 1629655457,
+            'upload_date': '20210822',
+            'duration': 19,
+            'view_count': int,
+            'like_count': int,
+            'artist': 'Unknown',
+            'track': 'Unknown',
+            'uploader_url': 'https://triller.co/@statefairent',
+            'comment_count': int,
+        },
+    }]
+
+    def _real_extract(self, url):
+        real_url = self._request_webpage(HEADRequest(url), self._match_id(url)).geturl()
+        if self.suitable(real_url):  # Prevent infinite loop in case redirect fails
+            raise UnsupportedError(real_url)
+        return self.url_result(real_url)

From 9be0fe1fd967f62cbf3c60bd14e1021a70abc147 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 26 Mar 2023 17:27:39 -0500
Subject: [PATCH 189/871] [extractor/nbc] Fix `NBCStations` direct mp4 formats
 (#6637)

Authored by: bashonly
---
 yt_dlp/extractor/nbc.py | 57 ++++++++++++++++++++++++++++++++---------
 1 file changed, 45 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index b9f65e927..ddc89a7c2 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -12,9 +12,13 @@
     RegexNotFoundError,
     UserNotLive,
     clean_html,
+    determine_ext,
+    float_or_none,
     int_or_none,
+    mimetype2ext,
     parse_age_limit,
     parse_duration,
+    remove_end,
     smuggle_url,
     traverse_obj,
     try_get,
@@ -22,7 +26,6 @@
     unified_timestamp,
     update_url_query,
     url_basename,
-    xpath_attr,
 )
 
 
@@ -660,6 +663,7 @@ class NBCStationsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Large Structure Fire in Downtown LA Prompts Smoke Odor Advisory',
             'description': 'md5:417ed3c2d91fe9d301e6db7b0942f182',
+            'duration': 112.513,
             'timestamp': 1661135892,
             'upload_date': '20220822',
             'uploader': 'NBC 4',
@@ -676,6 +680,7 @@ class NBCStationsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Huracán complica que televidente de Tucson reciba  reembolso',
             'description': 'md5:af298dc73aab74d4fca6abfb12acb6cf',
+            'duration': 172.406,
             'timestamp': 1660886507,
             'upload_date': '20220819',
             'uploader': 'Telemundo Arizona',
@@ -685,6 +690,22 @@ class NBCStationsIE(InfoExtractor):
         'params': {
             'skip_download': 'm3u8',
         },
+    }, {
+        # direct mp4 link
+        'url': 'https://www.nbcboston.com/weather/video-weather/highs-near-freezing-in-boston-on-wednesday/2961135/',
+        'md5': '9bf8c41dc7abbb75b1a44f1491a4cc85',
+        'info_dict': {
+            'id': '2961135',
+            'ext': 'mp4',
+            'title': 'Highs Near Freezing in Boston on Wednesday',
+            'description': 'md5:3ec486609a926c99f00a3512e6c0e85b',
+            'duration': 235.669,
+            'timestamp': 1675268656,
+            'upload_date': '20230201',
+            'uploader': '',
+            'channel_id': 'WBTS',
+            'channel': 'nbcboston',
+        },
     }]
 
     _RESOLUTIONS = {
@@ -711,7 +732,7 @@ def _real_extract(self, url):
         if not video_data:
             raise ExtractorError('No video metadata found in webpage', expected=True)
 
-        info, formats, subtitles = {}, [], {}
+        info, formats = {}, []
         is_live = int_or_none(video_data.get('mpx_is_livestream')) == 1
         query = {
             'formats': 'MPEG-DASH none,M3U none,MPEG-DASH none,MPEG4,MP3',
@@ -747,13 +768,14 @@ def _real_extract(self, url):
 
             video_url = traverse_obj(video_data, ((None, ('video', 'meta')), 'mp4_url'), get_all=False)
             if video_url:
+                ext = determine_ext(video_url)
                 height = self._search_regex(r'\d+-(\d+)p', url_basename(video_url), 'height', default=None)
                 formats.append({
                     'url': video_url,
-                    'ext': 'mp4',
+                    'ext': ext,
                     'width': int_or_none(self._RESOLUTIONS.get(height)),
                     'height': int_or_none(height),
-                    'format_id': 'http-mp4',
+                    'format_id': f'http-{ext}',
                 })
 
             info.update({
@@ -770,14 +792,25 @@ def _real_extract(self, url):
             smil = self._download_xml(
                 f'https://link.theplatform.com/s/{pdk_acct}/{player_id}', video_id,
                 note='Downloading SMIL data', query=query, fatal=is_live)
-        if smil:
-            manifest_url = xpath_attr(smil, f'.//{{{default_ns}}}video', 'src', fatal=is_live)
-            subtitles = self._parse_smil_subtitles(smil, default_ns)
-            fmts, subs = self._extract_m3u8_formats_and_subtitles(
-                manifest_url, video_id, 'mp4', m3u8_id='hls', fatal=is_live,
-                live=is_live, errnote='No HLS formats found')
-            formats.extend(fmts)
-            self._merge_subtitles(subs, target=subtitles)
+        subtitles = self._parse_smil_subtitles(smil, default_ns) if smil else {}
+        for video in smil.findall(self._xpath_ns('.//video', default_ns)) if smil else []:
+            info['duration'] = float_or_none(remove_end(video.get('dur'), 'ms'), 1000)
+            video_src_url = video.get('src')
+            ext = mimetype2ext(video.get('type'), default=determine_ext(video_src_url))
+            if ext == 'm3u8':
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    video_src_url, video_id, 'mp4', m3u8_id='hls', fatal=is_live,
+                    live=is_live, errnote='No HLS formats found')
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            elif video_src_url:
+                formats.append({
+                    'url': video_src_url,
+                    'format_id': f'https-{ext}',
+                    'ext': ext,
+                    'width': int_or_none(video.get('width')),
+                    'height': int_or_none(video.get('height')),
+                })
 
         if not formats:
             self.raise_no_formats('No video content found in webpage', expected=True)

From 95a383be1b6fb00c92ee3fb091732c4f6009acb6 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Mon, 27 Mar 2023 22:39:55 +0900
Subject: [PATCH 190/871] [extractor/iwara] Report private videos (#6641)

Authored by: Lesmiscore
---
 yt_dlp/extractor/iwara.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/iwara.py b/yt_dlp/extractor/iwara.py
index 62a179700..23f92786f 100644
--- a/yt_dlp/extractor/iwara.py
+++ b/yt_dlp/extractor/iwara.py
@@ -4,6 +4,7 @@
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     OnDemandPagedList,
     int_or_none,
     mimetype2ext,
@@ -75,7 +76,13 @@ def _extract_formats(self, video_id, fileurl):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(f'http://api.iwara.tv/video/{video_id}', video_id)
+        video_data = self._download_json(f'http://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True)
+        errmsg = video_data.get('message')
+        # at this point we can actually get uploaded user info, but do we need it?
+        if errmsg == 'errors.privateVideo':
+            self.raise_login_required('Private video. Login if you have permissions to watch')
+        elif errmsg:
+            raise ExtractorError(f'Iwara says: {errmsg}')
 
         return {
             'id': video_id,

From 0f0875ed555514f32522a0f30554fb08825d5124 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Tue, 28 Mar 2023 01:17:42 +0900
Subject: [PATCH 191/871] 
 [postprocessor/EmbedThumbnail,postprocessor/FFmpegMetadata] Fix error on
 attaching thumbnails and info json for mkv/mka (#6647)

Authored by: Lesmiscore

Current yt-dlp code never hit this bug, but would hit once filename sanitization gets better
---
 yt_dlp/postprocessor/embedthumbnail.py | 2 +-
 yt_dlp/postprocessor/ffmpeg.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/postprocessor/embedthumbnail.py b/yt_dlp/postprocessor/embedthumbnail.py
index b02d9d499..88a767132 100644
--- a/yt_dlp/postprocessor/embedthumbnail.py
+++ b/yt_dlp/postprocessor/embedthumbnail.py
@@ -107,7 +107,7 @@ def run(self, info):
                 options.extend(['-map', '-0:%d' % old_stream])
                 new_stream -= 1
             options.extend([
-                '-attach', thumbnail_filename,
+                '-attach', self._ffmpeg_filename_argument(thumbnail_filename),
                 '-metadata:s:%d' % new_stream, 'mimetype=%s' % mimetype,
                 '-metadata:s:%d' % new_stream, 'filename=cover.%s' % thumbnail_ext])
 
diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py
index 0e8f4c70b..63fc9ace6 100644
--- a/yt_dlp/postprocessor/ffmpeg.py
+++ b/yt_dlp/postprocessor/ffmpeg.py
@@ -809,7 +809,7 @@ def _get_infojson_opts(self, info, infofn):
             new_stream -= 1
 
         yield (
-            '-attach', infofn,
+            '-attach', self._ffmpeg_filename_argument(infofn),
             f'-metadata:s:{new_stream}', 'mimetype=application/json',
             f'-metadata:s:{new_stream}', 'filename=info.json',
         )

From ab92d8651c48d247dfb7d3f0a824cc986e47c7ed Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Wed, 29 Mar 2023 15:28:29 +0900
Subject: [PATCH 192/871] [extractor/iwara] Accept old URLs

Authored by: Lesmiscore

Closes #6669
---
 yt_dlp/extractor/iwara.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/iwara.py b/yt_dlp/extractor/iwara.py
index 23f92786f..ae2960af0 100644
--- a/yt_dlp/extractor/iwara.py
+++ b/yt_dlp/extractor/iwara.py
@@ -15,7 +15,7 @@
 
 class IwaraIE(InfoExtractor):
     IE_NAME = 'iwara'
-    _VALID_URL = r'https?://(?:www\.)?iwara\.tv/video/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.|ecchi\.)?iwara\.tv/videos?/(?P<id>[a-zA-Z0-9]+)'
     _TESTS = [{
         # this video cannot be played because of migration
         'only_matching': True,

From 68be95bd0ca3f76aa63c9812935bd826b3a42e53 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Fri, 31 Mar 2023 11:56:49 +0900
Subject: [PATCH 193/871] [extractor/YahooGyaOIE,extactor/YahooGyaOPlayerIE]
 Delete extractors due to website close (#6218)

Authored by: Lesmiscore
---
 yt_dlp/extractor/_extractors.py |   2 -
 yt_dlp/extractor/yahoo.py       | 117 --------------------------------
 2 files changed, 119 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index a97c458fa..77a3c2ce9 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2343,8 +2343,6 @@
 from .yahoo import (
     YahooIE,
     YahooSearchIE,
-    YahooGyaOPlayerIE,
-    YahooGyaOIE,
     YahooJapanNewsIE,
 )
 from .yandexdisk import YandexDiskIE
diff --git a/yt_dlp/extractor/yahoo.py b/yt_dlp/extractor/yahoo.py
index a69715b7c..24148a0bd 100644
--- a/yt_dlp/extractor/yahoo.py
+++ b/yt_dlp/extractor/yahoo.py
@@ -2,7 +2,6 @@
 import itertools
 import urllib.parse
 
-from .brightcove import BrightcoveNewIE
 from .common import InfoExtractor, SearchInfoExtractor
 from .youtube import YoutubeIE
 from ..utils import (
@@ -11,7 +10,6 @@
     int_or_none,
     mimetype2ext,
     parse_iso8601,
-    smuggle_url,
     traverse_obj,
     try_get,
     url_or_none,
@@ -337,121 +335,6 @@ def _search_results(self, query):
                 break
 
 
-class YahooGyaOPlayerIE(InfoExtractor):
-    IE_NAME = 'yahoo:gyao:player'
-    _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:player|episode(?:/[^/]+)?)|streaming\.yahoo\.co\.jp/c/y)/(?P<id>\d+/v\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
-    _TESTS = [{
-        'url': 'https://gyao.yahoo.co.jp/player/00998/v00818/v0000000000000008564/',
-        'info_dict': {
-            'id': '5993125228001',
-            'ext': 'mp4',
-            'title': 'フューリー　【字幕版】',
-            'description': 'md5:21e691c798a15330eda4db17a8fe45a5',
-            'uploader_id': '4235717419001',
-            'upload_date': '20190124',
-            'timestamp': 1548294365,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://streaming.yahoo.co.jp/c/y/01034/v00133/v0000000000000000706/',
-        'only_matching': True,
-    }, {
-        'url': 'https://gyao.yahoo.co.jp/episode/%E3%81%8D%E3%81%AE%E3%81%86%E4%BD%95%E9%A3%9F%E3%81%B9%E3%81%9F%EF%BC%9F%20%E7%AC%AC2%E8%A9%B1%202019%2F4%2F12%E6%94%BE%E9%80%81%E5%88%86/5cb02352-b725-409e-9f8d-88f947a9f682',
-        'only_matching': True,
-    }, {
-        'url': 'https://gyao.yahoo.co.jp/episode/5fa1226c-ef8d-4e93-af7a-fd92f4e30597',
-        'only_matching': True,
-    }]
-    _GEO_BYPASS = False
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url).replace('/', ':')
-        headers = self.geo_verification_headers()
-        headers['Accept'] = 'application/json'
-        resp = self._download_json(
-            'https://gyao.yahoo.co.jp/apis/playback/graphql', video_id, query={
-                'appId': 'dj00aiZpPUNJeDh2cU1RazU3UCZzPWNvbnN1bWVyc2VjcmV0Jng9NTk-',
-                'query': '''{
-  content(parameter: {contentId: "%s", logicaAgent: PC_WEB}) {
-    video {
-      delivery {
-        id
-      }
-      title
-    }
-  }
-}''' % video_id,
-            }, headers=headers)
-        content = resp['data']['content']
-        if not content:
-            msg = resp['errors'][0]['message']
-            if msg == 'not in japan':
-                self.raise_geo_restricted(countries=['JP'])
-            raise ExtractorError(msg)
-        video = content['video']
-        return {
-            '_type': 'url_transparent',
-            'id': video_id,
-            'title': video['title'],
-            'url': smuggle_url(
-                'http://players.brightcove.net/4235717419001/SyG5P0gjb_default/index.html?videoId=' + video['delivery']['id'],
-                {'geo_countries': ['JP']}),
-            'ie_key': BrightcoveNewIE.ie_key(),
-        }
-
-
-class YahooGyaOIE(InfoExtractor):
-    IE_NAME = 'yahoo:gyao'
-    _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:p|title(?:/[^/]+)?)|streaming\.yahoo\.co\.jp/p/y)/(?P<id>\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
-    _TESTS = [{
-        'url': 'https://gyao.yahoo.co.jp/title/%E3%82%BF%E3%82%A4%E3%83%A0%E3%83%9C%E3%82%AB%E3%83%B3%E3%82%B7%E3%83%AA%E3%83%BC%E3%82%BA%20%E3%83%A4%E3%83%83%E3%82%BF%E3%83%BC%E3%83%9E%E3%83%B3/5f60ceb3-6e5e-40ef-ba40-d68b598d067f',
-        'info_dict': {
-            'id': '5f60ceb3-6e5e-40ef-ba40-d68b598d067f',
-        },
-        'playlist_mincount': 80,
-    }, {
-        'url': 'https://gyao.yahoo.co.jp/p/00449/v03102/',
-        'only_matching': True,
-    }, {
-        'url': 'https://streaming.yahoo.co.jp/p/y/01034/v00133/',
-        'only_matching': True,
-    }, {
-        'url': 'https://gyao.yahoo.co.jp/title/%E3%81%97%E3%82%83%E3%81%B9%E3%81%8F%E3%82%8A007/5b025a49-b2e5-4dc7-945c-09c6634afacf',
-        'only_matching': True,
-    }, {
-        'url': 'https://gyao.yahoo.co.jp/title/5b025a49-b2e5-4dc7-945c-09c6634afacf',
-        'only_matching': True,
-    }]
-
-    def _entries(self, program_id):
-        page = 1
-        while True:
-            playlist = self._download_json(
-                f'https://gyao.yahoo.co.jp/api/programs/{program_id}/videos?page={page}&serviceId=gy', program_id,
-                note=f'Downloading JSON metadata page {page}')
-            if not playlist:
-                break
-            for video in playlist['videos']:
-                video_id = video.get('id')
-                if not video_id:
-                    continue
-                if video.get('streamingAvailability') == 'notYet':
-                    continue
-                yield self.url_result(
-                    'https://gyao.yahoo.co.jp/player/%s/' % video_id.replace(':', '/'),
-                    YahooGyaOPlayerIE.ie_key(), video_id)
-            if playlist.get('ended'):
-                break
-            page += 1
-
-    def _real_extract(self, url):
-        program_id = self._match_id(url).replace('/', ':')
-        return self.playlist_result(self._entries(program_id), program_id)
-
-
 class YahooJapanNewsIE(InfoExtractor):
     IE_NAME = 'yahoo:japannews'
     IE_DESC = 'Yahoo! Japan News'

From 141a8dff98874a426d7fbe772e0a8421bb42656f Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Thu, 6 Apr 2023 19:44:22 +1200
Subject: [PATCH 194/871] [extractor/youtube] Fix comment loop detection for
 pinned comments (#6714)

Pinned comments may repeat a second time - this is expected.

Fixes https://github.com/yt-dlp/yt-dlp/issues/6712

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index ca56f112b..6dc36f9b9 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3316,9 +3316,17 @@ def extract_thread(contents):
                 comment = self._extract_comment(comment_renderer, parent)
                 if not comment:
                     continue
+                is_pinned = bool(traverse_obj(comment_renderer, 'pinnedCommentBadge'))
+                comment_id = comment['id']
+                if is_pinned:
+                    tracker['pinned_comment_ids'].add(comment_id)
                 # Sometimes YouTube may break and give us infinite looping comments.
                 # See: https://github.com/yt-dlp/yt-dlp/issues/6290
-                if comment['id'] in tracker['seen_comment_ids']:
+                if comment_id in tracker['seen_comment_ids']:
+                    if comment_id in tracker['pinned_comment_ids'] and not is_pinned:
+                        # Pinned comments may appear a second time in newest first sort
+                        # See: https://github.com/yt-dlp/yt-dlp/issues/6712
+                        continue
                     self.report_warning('Detected YouTube comments looping. Stopping comment extraction as we probably cannot get any more.')
                     yield
                 else:
@@ -3348,7 +3356,9 @@ def extract_thread(contents):
                 current_page_thread=0,
                 total_parent_comments=0,
                 total_reply_comments=0,
-                seen_comment_ids=set())
+                seen_comment_ids=set(),
+                pinned_comment_ids=set()
+            )
 
         # TODO: Deprecated
         # YouTube comments have a max depth of 2

From 0a6918a4a1431960181d8c50e0bbbcb0afbaff9a Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 8 Apr 2023 11:09:05 -0500
Subject: [PATCH 195/871] [extractor/kick] Make initial request non-fatal

Authored by: bashonly
---
 yt_dlp/extractor/kick.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/kick.py b/yt_dlp/extractor/kick.py
index a79ffb7a9..765ffa0c8 100644
--- a/yt_dlp/extractor/kick.py
+++ b/yt_dlp/extractor/kick.py
@@ -14,7 +14,7 @@
 
 class KickBaseIE(InfoExtractor):
     def _real_initialize(self):
-        self._request_webpage(HEADRequest('https://kick.com/'), None, 'Setting up session')
+        self._request_webpage(HEADRequest('https://kick.com/'), None, 'Setting up session', fatal=False)
         xsrf_token = self._get_cookies('https://kick.com/').get('XSRF-TOKEN')
         if not xsrf_token:
             self.write_debug('kick.com did not set XSRF-TOKEN cookie')

From ef0848abd425dfda6db62baa8d72897eefb0007f Mon Sep 17 00:00:00 2001
From: Chris Caruso <carusochrisr@gmail.com>
Date: Tue, 11 Apr 2023 04:45:22 -0700
Subject: [PATCH 196/871] [extractor/youku] Improve error message (#6690)

Authored by: carusocr
Closes #6551
---
 yt_dlp/extractor/youku.py | 44 +++------------------------------------
 1 file changed, 3 insertions(+), 41 deletions(-)

diff --git a/yt_dlp/extractor/youku.py b/yt_dlp/extractor/youku.py
index 404f196f4..7ecd9f183 100644
--- a/yt_dlp/extractor/youku.py
+++ b/yt_dlp/extractor/youku.py
@@ -6,6 +6,7 @@
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    clean_html,
     get_element_by_class,
     js_to_json,
     str_or_none,
@@ -26,48 +27,8 @@ class YoukuIE(InfoExtractor):
     '''
 
     _TESTS = [{
-        # MD5 is unstable
-        'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
-        'info_dict': {
-            'id': 'XMTc1ODE5Njcy',
-            'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
-            'ext': 'mp4',
-            'duration': 74.73,
-            'thumbnail': r're:^https?://.*',
-            'uploader': '。躲猫猫、',
-            'uploader_id': '36017967',
-            'uploader_url': 'http://i.youku.com/u/UMTQ0MDcxODY4',
-            'tags': list,
-        }
-    }, {
         'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
         'only_matching': True,
-    }, {
-        'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html',
-        'info_dict': {
-            'id': 'XODgxNjg1Mzk2',
-            'ext': 'mp4',
-            'title': '武媚娘传奇 85',
-            'duration': 1999.61,
-            'thumbnail': r're:^https?://.*',
-            'uploader': '疯狂豆花',
-            'uploader_id': '62583473',
-            'uploader_url': 'http://i.youku.com/u/UMjUwMzMzODky',
-            'tags': list,
-        },
-    }, {
-        'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
-        'info_dict': {
-            'id': 'XMTI1OTczNDM5Mg',
-            'ext': 'mp4',
-            'title': '花千骨 04',
-            'duration': 2363,
-            'thumbnail': r're:^https?://.*',
-            'uploader': '放剧场-花千骨',
-            'uploader_id': '772849359',
-            'uploader_url': 'http://i.youku.com/u/UMzA5MTM5NzQzNg==',
-            'tags': list,
-        },
     }, {
         'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
         'note': 'Video protected with password',
@@ -81,6 +42,7 @@ class YoukuIE(InfoExtractor):
             'uploader_id': '322014285',
             'uploader_url': 'http://i.youku.com/u/UMTI4ODA1NzE0MA==',
             'tags': list,
+            'skip': '404',
         },
         'params': {
             'videopassword': '100600',
@@ -192,7 +154,7 @@ def _real_extract(self, url):
             else:
                 msg = 'Youku server reported error %i' % error.get('code')
                 if error_note is not None:
-                    msg += ': ' + error_note
+                    msg += ': ' + clean_html(error_note)
                 raise ExtractorError(msg)
 
         # get video title

From 7e35526d5b970a034b9d76215ee3e4bd7631edcd Mon Sep 17 00:00:00 2001
From: "lauren n. liberda" <lauren@selfisekai.rocks>
Date: Tue, 11 Apr 2023 13:54:49 +0200
Subject: [PATCH 197/871] [extractor/hrefli] Add extractor (#6762)

Authored by: selfisekai
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/hrefli.py      | 15 +++++++++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 yt_dlp/extractor/hrefli.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 77a3c2ce9..808b558d1 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -734,6 +734,7 @@
 )
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
+from .hrefli import HrefLiRedirectIE
 from .hrfensehen import HRFernsehenIE
 from .hrti import (
     HRTiIE,
diff --git a/yt_dlp/extractor/hrefli.py b/yt_dlp/extractor/hrefli.py
new file mode 100644
index 000000000..77db2ea68
--- /dev/null
+++ b/yt_dlp/extractor/hrefli.py
@@ -0,0 +1,15 @@
+from .common import InfoExtractor
+
+
+class HrefLiRedirectIE(InfoExtractor):
+    IE_NAME = 'href.li'
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'https?://href\.li/\?(?P<url>.+)'
+
+    _TESTS = [{
+        'url': 'https://href.li/?https://www.reddit.com/r/cats/comments/12bluel/my_cat_helps_me_with_water/?utm_source=share&utm_medium=android_app&utm_name=androidcss&utm_term=1&utm_content=share_button',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        return self.url_result(self._match_valid_url(url).group('url'))

From faa0332ed69e070cf3bd31390589a596e962f392 Mon Sep 17 00:00:00 2001
From: sian1468 <58017832+sian1468@users.noreply.github.com>
Date: Tue, 11 Apr 2023 18:56:39 +0700
Subject: [PATCH 198/871] [extractor/line] Remove extractors (#6734)

Service has shut down - https://archive.ph/txVKy
Authored by: sian1468
---
 yt_dlp/extractor/_extractors.py |   4 -
 yt_dlp/extractor/line.py        | 143 --------------------------------
 2 files changed, 147 deletions(-)
 delete mode 100644 yt_dlp/extractor/line.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 808b558d1..5f4ae7b8d 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -944,10 +944,6 @@
     LimelightChannelIE,
     LimelightChannelListIE,
 )
-from .line import (
-    LineLiveIE,
-    LineLiveChannelIE,
-)
 from .linkedin import (
     LinkedInIE,
     LinkedInLearningIE,
diff --git a/yt_dlp/extractor/line.py b/yt_dlp/extractor/line.py
deleted file mode 100644
index 3fab9c8a5..000000000
--- a/yt_dlp/extractor/line.py
+++ /dev/null
@@ -1,143 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    format_field,
-    int_or_none,
-    str_or_none,
-)
-
-
-class LineLiveBaseIE(InfoExtractor):
-    _API_BASE_URL = 'https://live-api.line-apps.com/web/v4.0/channel/'
-
-    def _parse_broadcast_item(self, item):
-        broadcast_id = compat_str(item['id'])
-        title = item['title']
-        is_live = item.get('isBroadcastingNow')
-
-        thumbnails = []
-        for thumbnail_id, thumbnail_url in (item.get('thumbnailURLs') or {}).items():
-            if not thumbnail_url:
-                continue
-            thumbnails.append({
-                'id': thumbnail_id,
-                'url': thumbnail_url,
-            })
-
-        channel = item.get('channel') or {}
-        channel_id = str_or_none(channel.get('id'))
-
-        return {
-            'id': broadcast_id,
-            'title': title,
-            'thumbnails': thumbnails,
-            'timestamp': int_or_none(item.get('createdAt')),
-            'channel': channel.get('name'),
-            'channel_id': channel_id,
-            'channel_url': format_field(channel_id, None, 'https://live.line.me/channels/%s'),
-            'duration': int_or_none(item.get('archiveDuration')),
-            'view_count': int_or_none(item.get('viewerCount')),
-            'comment_count': int_or_none(item.get('chatCount')),
-            'is_live': is_live,
-        }
-
-
-class LineLiveIE(LineLiveBaseIE):
-    _VALID_URL = r'https?://live\.line\.me/channels/(?P<channel_id>\d+)/broadcast/(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://live.line.me/channels/5833718/broadcast/18373277',
-        'md5': '2c15843b8cb3acd55009ddcb2db91f7c',
-        'info_dict': {
-            'id': '18373277',
-            'title': '2021/12/05 （15分犬）定例譲渡会🐶',
-            'ext': 'mp4',
-            'timestamp': 1638674925,
-            'upload_date': '20211205',
-            'thumbnail': 'md5:e1f5817e60f4a72b7e43377cf308d7ef',
-            'channel_url': 'https://live.line.me/channels/5833718',
-            'channel': 'Yahooニュース掲載🗞プロフ見てね🐕🐕',
-            'channel_id': '5833718',
-            'duration': 937,
-            'view_count': int,
-            'comment_count': int,
-            'is_live': False,
-        }
-    }, {
-        # archiveStatus == 'DELETED'
-        'url': 'https://live.line.me/channels/4778159/broadcast/16378488',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        channel_id, broadcast_id = self._match_valid_url(url).groups()
-        broadcast = self._download_json(
-            self._API_BASE_URL + '%s/broadcast/%s' % (channel_id, broadcast_id),
-            broadcast_id)
-        item = broadcast['item']
-        info = self._parse_broadcast_item(item)
-        protocol = 'm3u8' if info['is_live'] else 'm3u8_native'
-        formats = []
-        for k, v in (broadcast.get(('live' if info['is_live'] else 'archived') + 'HLSURLs') or {}).items():
-            if not v:
-                continue
-            if k == 'abr':
-                formats.extend(self._extract_m3u8_formats(
-                    v, broadcast_id, 'mp4', protocol,
-                    m3u8_id='hls', fatal=False))
-                continue
-            f = {
-                'ext': 'mp4',
-                'format_id': 'hls-' + k,
-                'protocol': protocol,
-                'url': v,
-            }
-            if not k.isdigit():
-                f['vcodec'] = 'none'
-            formats.append(f)
-        if not formats:
-            archive_status = item.get('archiveStatus')
-            if archive_status != 'ARCHIVED':
-                self.raise_no_formats('this video has been ' + archive_status.lower(), expected=True)
-        info['formats'] = formats
-        return info
-
-
-class LineLiveChannelIE(LineLiveBaseIE):
-    _VALID_URL = r'https?://live\.line\.me/channels/(?P<id>\d+)(?!/broadcast/\d+)(?:[/?&#]|$)'
-    _TEST = {
-        'url': 'https://live.line.me/channels/5893542',
-        'info_dict': {
-            'id': '5893542',
-            'title': 'いくらちゃんだよぉ🦒',
-            'description': 'md5:4d418087973ad081ceb1b3481f0b1816',
-        },
-        'playlist_mincount': 29
-    }
-
-    def _archived_broadcasts_entries(self, archived_broadcasts, channel_id):
-        while True:
-            for row in (archived_broadcasts.get('rows') or []):
-                share_url = str_or_none(row.get('shareURL'))
-                if not share_url:
-                    continue
-                info = self._parse_broadcast_item(row)
-                info.update({
-                    '_type': 'url',
-                    'url': share_url,
-                    'ie_key': LineLiveIE.ie_key(),
-                })
-                yield info
-            if not archived_broadcasts.get('hasNextPage'):
-                return
-            archived_broadcasts = self._download_json(
-                self._API_BASE_URL + channel_id + '/archived_broadcasts',
-                channel_id, query={
-                    'lastId': info['id'],
-                })
-
-    def _real_extract(self, url):
-        channel_id = self._match_id(url)
-        channel = self._download_json(self._API_BASE_URL + channel_id, channel_id)
-        return self.playlist_result(
-            self._archived_broadcasts_entries(channel.get('archivedBroadcasts') or {}, channel_id),
-            channel_id, channel.get('title'), channel.get('information'))

From 79c77e85b70ae3b9942d5a88c14d021a9bd24222 Mon Sep 17 00:00:00 2001
From: Shreyas Minocha <11537232+shreyasminocha@users.noreply.github.com>
Date: Tue, 11 Apr 2023 16:05:22 +0000
Subject: [PATCH 199/871] [extractor/zoom] Fix extractor (#6741)

Authored by: shreyasminocha
Closes #6677
---
 yt_dlp/extractor/zoom.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/zoom.py b/yt_dlp/extractor/zoom.py
index ef8b71522..eb0ab795b 100644
--- a/yt_dlp/extractor/zoom.py
+++ b/yt_dlp/extractor/zoom.py
@@ -5,6 +5,7 @@
     str_or_none,
     js_to_json,
     parse_filesize,
+    traverse_obj,
     urlencode_postdata,
     urljoin,
 )
@@ -53,6 +54,9 @@ def _real_extract(self, url):
             r'(?s)window\.__data__\s*=\s*({.+?});',
             webpage, 'data'), play_id, js_to_json)
 
+        data = self._download_json(
+            f'{base_url}nws/recording/1.0/play/info/{data["fileId"]}', play_id)['result']
+
         subtitles = {}
         for _type in ('transcript', 'cc', 'chapter'):
             if data.get('%sUrl' % _type):
@@ -67,11 +71,11 @@ def _real_extract(self, url):
             formats.append({
                 'format_note': 'Camera stream',
                 'url': str_or_none(data.get('viewMp4Url')),
-                'width': int_or_none(data.get('viewResolvtionsWidth')),
-                'height': int_or_none(data.get('viewResolvtionsHeight')),
-                'format_id': str_or_none(data.get('recordingId')),
+                'width': int_or_none(traverse_obj(data, ('viewResolvtions', 0))),
+                'height': int_or_none(traverse_obj(data, ('viewResolvtions', 1))),
+                'format_id': str_or_none(traverse_obj(data, ('recording', 'id'))),
                 'ext': 'mp4',
-                'filesize_approx': parse_filesize(data.get('fileSize')),
+                'filesize_approx': parse_filesize(str_or_none(traverse_obj(data, ('recording', 'fileSizeInMB')))),
                 'preference': 0
             })
 
@@ -79,16 +83,16 @@ def _real_extract(self, url):
             formats.append({
                 'format_note': 'Screen share stream',
                 'url': str_or_none(data.get('shareMp4Url')),
-                'width': int_or_none(data.get('shareResolvtionsWidth')),
-                'height': int_or_none(data.get('shareResolvtionsHeight')),
-                'format_id': str_or_none(data.get('shareVideoId')),
+                'width': int_or_none(traverse_obj(data, ('shareResolvtions', 0))),
+                'height': int_or_none(traverse_obj(data, ('shareResolvtions', 1))),
+                'format_id': str_or_none(traverse_obj(data, ('shareVideo', 'id'))),
                 'ext': 'mp4',
                 'preference': -1
             })
 
         return {
             'id': play_id,
-            'title': data.get('topic'),
+            'title': str_or_none(traverse_obj(data, ('meet', 'topic'))),
             'subtitles': subtitles,
             'formats': formats,
             'http_headers': {

From c6786ff3baaf72a5baa4d56d34058e54cbcf8ceb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 11 Apr 2023 16:11:15 +0530
Subject: [PATCH 200/871] [extractor/youtube] Revert default formats to `https`

---
 yt_dlp/extractor/youtube.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 6dc36f9b9..d6a55e953 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3789,15 +3789,13 @@ def build_fragments(f):
             if single_stream and dct.get('ext'):
                 dct['container'] = dct['ext'] + '_dash'
 
-            if dct['filesize']:
+            if all_formats and dct['filesize']:
                 yield {
                     **dct,
                     'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
                     'protocol': 'http_dash_segments',
                     'fragments': build_fragments(dct),
                 }
-                if not all_formats:
-                    continue
             dct['downloader_options'] = {'http_chunk_size': CHUNK_SIZE}
             yield dct
 

From 26010b5cec50193b98ad7845d1d77450f9f14c2b Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 11 Apr 2023 17:01:22 +0530
Subject: [PATCH 201/871] [postprocessor/FixupDuplicateMoov] Fix bug in
 triggering

---
 yt_dlp/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index a7dced8e8..0d987dbb8 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3312,7 +3312,7 @@ def ffmpeg_fixup(cndn, msg, cls):
                                      or info_dict.get('is_live') and self.params.get('hls_use_mpegts') is None,
                                      'Possible MPEG-TS in MP4 container or malformed AAC timestamps',
                                      FFmpegFixupM3u8PP)
-                        ffmpeg_fixup(info_dict.get('is_live') and downloader == 'DashSegmentsFD',
+                        ffmpeg_fixup(info_dict.get('is_live') and downloader == 'dashsegments',
                                      'Possible duplicate MOOV atoms', FFmpegFixupDuplicateMoovPP)
 
                     ffmpeg_fixup(downloader == 'web_socket_fragment', 'Malformed timestamps detected', FFmpegFixupTimestampPP)

From 52ecc33e221f7de7eb6fed6c22489f0c5fdd2c6d Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Wed, 12 Apr 2023 01:19:34 +0900
Subject: [PATCH 202/871] [extractor/niconico] Download comments from the new
 endpoint (#6773)

Authored by: Lesmiscore
---
 yt_dlp/extractor/niconico.py | 54 ++++++++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py
index 9c3a5a4bc..cacefeb42 100644
--- a/yt_dlp/extractor/niconico.py
+++ b/yt_dlp/extractor/niconico.py
@@ -477,23 +477,32 @@ def _get_subtitles(self, video_id, api_data, session_api_data):
         user_id_str = session_api_data.get('serviceUserId')
 
         thread_ids = traverse_obj(api_data, ('comment', 'threads', lambda _, v: v['isActive']))
-        raw_danmaku = self._extract_all_comments(video_id, thread_ids, user_id_str, comment_user_key)
-        if not raw_danmaku:
+        legacy_danmaku = self._extract_legacy_comments(video_id, thread_ids, user_id_str, comment_user_key) or []
+
+        new_comments = traverse_obj(api_data, ('comment', 'nvComment'))
+        new_danmaku = self._extract_new_comments(
+            new_comments.get('server'), video_id,
+            new_comments.get('params'), new_comments.get('threadKey'))
+
+        if not legacy_danmaku and not new_danmaku:
             self.report_warning(f'Failed to get comments. {bug_reports_message()}')
             return
+
         return {
             'comments': [{
                 'ext': 'json',
-                'data': json.dumps(raw_danmaku),
+                'data': json.dumps(legacy_danmaku + new_danmaku),
             }],
         }
 
-    def _extract_all_comments(self, video_id, threads, user_id, user_key):
+    def _extract_legacy_comments(self, video_id, threads, user_id, user_key):
         auth_data = {
             'user_id': user_id,
             'userkey': user_key,
         } if user_id and user_key else {'user_id': ''}
 
+        api_url = traverse_obj(threads, (..., 'server'), get_all=False)
+
         # Request Start
         post_data = [{'ping': {'content': 'rs:0'}}]
         for i, thread in enumerate(threads):
@@ -532,17 +541,32 @@ def _extract_all_comments(self, video_id, threads, user_id, user_key):
         # Request Final
         post_data.append({'ping': {'content': 'rf:0'}})
 
-        for api_url in self._COMMENT_API_ENDPOINTS:
-            comments = self._download_json(
-                api_url, video_id, data=json.dumps(post_data).encode(), fatal=False,
-                headers={
-                    'Referer': 'https://www.nicovideo.jp/watch/%s' % video_id,
-                    'Origin': 'https://www.nicovideo.jp',
-                    'Content-Type': 'text/plain;charset=UTF-8',
-                },
-                note='Downloading comments', errnote=f'Failed to access endpoint {api_url}')
-            if comments:
-                return comments
+        return self._download_json(
+            f'{api_url}/api.json', video_id, data=json.dumps(post_data).encode(), fatal=False,
+            headers={
+                'Referer': f'https://www.nicovideo.jp/watch/{video_id}',
+                'Origin': 'https://www.nicovideo.jp',
+                'Content-Type': 'text/plain;charset=UTF-8',
+            },
+            note='Downloading comments', errnote=f'Failed to access endpoint {api_url}')
+
+    def _extract_new_comments(self, endpoint, video_id, params, thread_key):
+        comments = self._download_json(
+            f'{endpoint}/v1/threads', video_id, data=json.dumps({
+                'additionals': {},
+                'params': params,
+                'threadKey': thread_key,
+            }).encode(), fatal=False,
+            headers={
+                'Referer': 'https://www.nicovideo.jp/',
+                'Origin': 'https://www.nicovideo.jp',
+                'Content-Type': 'text/plain;charset=UTF-8',
+                'x-client-os-type': 'others',
+                'x-frontend-id': '6',
+                'x-frontend-version': '0',
+            },
+            note='Downloading comments (new)', errnote='Failed to download comments (new)')
+        return traverse_obj(comments, ('data', 'threads', ..., 'comments', ...))
 
 
 class NiconicoPlaylistBaseIE(InfoExtractor):

From c3f624ef0a5d7a6ae1c5ffeb243087e9fc7d79dc Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 12 Apr 2023 05:04:47 +0530
Subject: [PATCH 203/871] Relaxed validation for numeric format filters

Continued from f96bff99cb2cf1d112b099e5149dd2c3a6a76af2

Closes #6782
---
 yt_dlp/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 0d987dbb8..7b6fef204 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1932,7 +1932,7 @@ def _build_format_filter(self, filter_spec):
             '!=': operator.ne,
         }
         operator_rex = re.compile(r'''(?x)\s*
-            (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)\s*
+            (?P<key>[\w.-]+)\s*
             (?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)\s*
             ''' % '|'.join(map(re.escape, OPERATORS.keys())))

From 2d97d154fe4fb84fe2ed3a4e1ed5819e89b71e88 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Thu, 13 Apr 2023 03:19:08 +0900
Subject: [PATCH 204/871] [extractor/gmanetwork] Add extractor (#5945)

Authored by: HobbyistDev
Partially fixes #5770
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/gmanetwork.py  | 83 +++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+)
 create mode 100644 yt_dlp/extractor/gmanetwork.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 5f4ae7b8d..c2043bbd2 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -681,6 +681,7 @@
     GloboIE,
     GloboArticleIE,
 )
+from .gmanetwork import GMANetworkVideoIE
 from .go import GoIE
 from .godtube import GodTubeIE
 from .gofile import GofileIE
diff --git a/yt_dlp/extractor/gmanetwork.py b/yt_dlp/extractor/gmanetwork.py
new file mode 100644
index 000000000..62fff4ead
--- /dev/null
+++ b/yt_dlp/extractor/gmanetwork.py
@@ -0,0 +1,83 @@
+from .common import InfoExtractor
+from .dailymotion import DailymotionIE
+from .youtube import YoutubeIE
+
+
+class GMANetworkVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www)\.gmanetwork\.com/(?:\w+/){3}(?P<id>\d+)/(?P<display_id>[\w-]+)/video'
+    _TESTS = [{
+        'url': 'https://www.gmanetwork.com/fullepisodes/home/running_man_philippines/168677/running-man-philippines-catch-the-thief-full-chapter-2/video?section=home',
+        'info_dict': {
+            'id': '28BqW0AXPe0',
+            'ext': 'mp4',
+            'upload_date': '20220919',
+            'uploader_url': 'http://www.youtube.com/channel/UChsoPNR5x-wdSO2GrOSIWqQ',
+            'like_count': int,
+            'view_count': int,
+            'uploader': 'YoüLOL',
+            'channel_id': 'UChsoPNR5x-wdSO2GrOSIWqQ',
+            'duration': 5313,
+            'comment_count': int,
+            'tags': 'count:22',
+            'uploader_id': 'UChsoPNR5x-wdSO2GrOSIWqQ',
+            'title': 'Running Man Philippines: Catch the Thief (FULL CHAPTER 2)',
+            'channel_url': 'https://www.youtube.com/channel/UChsoPNR5x-wdSO2GrOSIWqQ',
+            'thumbnail': 'https://i.ytimg.com/vi/28BqW0AXPe0/maxresdefault.jpg',
+            'release_timestamp': 1663594212,
+            'age_limit': 0,
+            'channel_follower_count': int,
+            'categories': ['Entertainment'],
+            'description': 'md5:811bdcea74f9c48051824e494756e926',
+            'live_status': 'not_live',
+            'playable_in_embed': True,
+            'channel': 'YoüLOL',
+            'availability': 'public',
+            'release_date': '20220919',
+        }
+    }, {
+        'url': 'https://www.gmanetwork.com/fullepisodes/home/more_than_words/87059/more-than-words-full-episode-80/video?section=home',
+        'info_dict': {
+            'id': 'yiDOExw2aSA',
+            'ext': 'mp4',
+            'live_status': 'not_live',
+            'channel': 'GMANetwork',
+            'like_count': int,
+            'channel_follower_count': int,
+            'description': 'md5:6d00cd658394fa1a5071200d3ed4be05',
+            'duration': 1419,
+            'age_limit': 0,
+            'comment_count': int,
+            'upload_date': '20181003',
+            'thumbnail': 'https://i.ytimg.com/vi_webp/yiDOExw2aSA/maxresdefault.webp',
+            'availability': 'public',
+            'playable_in_embed': True,
+            'channel_id': 'UCKL5hAuzgFQsyrsQKgU0Qng',
+            'title': 'More Than Words: Full Episode 80 (Finale)',
+            'uploader_id': 'GMANETWORK',
+            'categories': ['Entertainment'],
+            'uploader': 'GMANetwork',
+            'channel_url': 'https://www.youtube.com/channel/UCKL5hAuzgFQsyrsQKgU0Qng',
+            'tags': 'count:29',
+            'view_count': int,
+            'uploader_url': 'http://www.youtube.com/user/GMANETWORK',
+        }
+    }]
+
+    def _real_extract(self, url):
+        content_id, display_id = self._match_valid_url(url).group('id', 'display_id')
+        webpage = self._download_webpage(url, display_id)
+        # webpage route
+        youtube_id = self._search_regex(
+            r'var\s*YOUTUBE_VIDEO\s*=\s*[\'"]+(?P<yt_id>[\w-]+)', webpage, 'youtube_id', fatal=False)
+        if youtube_id:
+            return self.url_result(youtube_id, YoutubeIE, youtube_id)
+
+        # api call route
+        # more info at https://aphrodite.gmanetwork.com/fullepisodes/assets/fullepisodes/js/dist/fullepisodes_video.js?v=1.1.11
+        network_url = self._search_regex(
+            r'NETWORK_URL\s*=\s*[\'"](?P<url>[^\'"]+)', webpage, 'network_url')
+        json_data = self._download_json(f'{network_url}api/data/content/video/{content_id}', display_id)
+        if json_data.get('video_file'):
+            return self.url_result(json_data['video_file'], YoutubeIE, json_data['video_file'])
+        else:
+            return self.url_result(json_data['dailymotion_file'], DailymotionIE, json_data['dailymotion_file'])

From b093c38cc9f26b59a8504211d792f053142c847d Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Thu, 13 Apr 2023 03:21:57 +0900
Subject: [PATCH 205/871] [extractor/biliIntl] Add comment extraction  (#6079)

Authored by: HobbyistDev
---
 yt_dlp/extractor/bilibili.py | 111 ++++++++++++++++++++++++++++++++++-
 1 file changed, 109 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index c34439779..91d436dd8 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -26,6 +26,7 @@
     srt_subtitles_timecode,
     str_or_none,
     traverse_obj,
+    unified_timestamp,
     unsmuggle_url,
     url_or_none,
     urlencode_postdata,
@@ -996,6 +997,53 @@ class BiliIntlIE(BiliIntlBaseIE):
             'thumbnail': r're:https?://pic[-\.]bstarstatic.+/ugc/.+\.jpg$',
             'upload_date': '20221212',
             'title': 'Kimetsu no Yaiba Season 3 Official Trailer - Bstation',
+        },
+    }, {
+        # episode comment extraction
+        'url': 'https://www.bilibili.tv/en/play/34580/340317',
+        'info_dict': {
+            'id': '340317',
+            'ext': 'mp4',
+            'timestamp': 1604057820,
+            'upload_date': '20201030',
+            'episode_number': 5,
+            'title': 'E5 - My Own Steel',
+            'description': 'md5:2b17ab10aebb33e3c2a54da9e8e487e2',
+            'thumbnail': r're:https?://pic\.bstarstatic\.com/ogv/.+\.png$',
+            'episode': 'Episode 5',
+            'comment_count': int,
+            'chapters': [{
+                'start_time': 0,
+                'end_time': 61.0,
+                'title': '<Untitled Chapter 1>'
+            }, {
+                'start_time': 61.0,
+                'end_time': 134.0,
+                'title': 'Intro'
+            }, {
+                'start_time': 1290.0,
+                'end_time': 1379.0,
+                'title': 'Outro'
+            }],
+        },
+        'params': {
+            'getcomments': True
+        }
+    }, {
+        # user generated content comment extraction
+        'url': 'https://www.bilibili.tv/en/video/2045730385',
+        'info_dict': {
+            'id': '2045730385',
+            'ext': 'mp4',
+            'description': 'md5:693b6f3967fb4e7e7764ea817857c33a',
+            'timestamp': 1667891924,
+            'upload_date': '20221108',
+            'title': 'That Time I Got Reincarnated as a Slime: Scarlet Bond - Official Trailer 3| AnimeStan - Bstation',
+            'comment_count': int,
+            'thumbnail': 'https://pic.bstarstatic.com/ugc/f6c363659efd2eabe5683fbb906b1582.jpg',
+        },
+        'params': {
+            'getcomments': True
         }
     }, {
         # episode id without intro and outro
@@ -1055,11 +1103,69 @@ def _extract_video_metadata(self, url, video_id, season_id):
 
         # XXX: webpage metadata may not accurate, it just used to not crash when video_data not found
         return merge_dicts(
-            self._parse_video_metadata(video_data), self._search_json_ld(webpage, video_id), {
+            self._parse_video_metadata(video_data), self._search_json_ld(webpage, video_id, fatal=False), {
                 'title': self._html_search_meta('og:title', webpage),
                 'description': self._html_search_meta('og:description', webpage)
             })
 
+    def _get_comments_reply(self, root_id, next_id=0, display_id=None):
+        comment_api_raw_data = self._download_json(
+            'https://api.bilibili.tv/reply/web/detail', display_id,
+            note=f'Downloading reply comment of {root_id} - {next_id}',
+            query={
+                'platform': 'web',
+                'ps': 20,  # comment's reply per page (default: 3)
+                'root': root_id,
+                'next': next_id,
+            })
+
+        for replies in traverse_obj(comment_api_raw_data, ('data', 'replies', ...)):
+            yield {
+                'author': traverse_obj(replies, ('member', 'name')),
+                'author_id': traverse_obj(replies, ('member', 'mid')),
+                'author_thumbnail': traverse_obj(replies, ('member', 'face')),
+                'text': traverse_obj(replies, ('content', 'message')),
+                'id': replies.get('rpid'),
+                'like_count': int_or_none(replies.get('like_count')),
+                'parent': replies.get('parent'),
+                'timestamp': unified_timestamp(replies.get('ctime_text'))
+            }
+
+        if not traverse_obj(comment_api_raw_data, ('data', 'cursor', 'is_end')):
+            yield from self._get_comments_reply(
+                root_id, comment_api_raw_data['data']['cursor']['next'], display_id)
+
+    def _get_comments(self, video_id, ep_id):
+        for i in itertools.count(0):
+            comment_api_raw_data = self._download_json(
+                'https://api.bilibili.tv/reply/web/root', video_id,
+                note=f'Downloading comment page {i + 1}',
+                query={
+                    'platform': 'web',
+                    'pn': i,  # page number
+                    'ps': 20,  # comment per page (default: 20)
+                    'oid': video_id,
+                    'type': 3 if ep_id else 1,  # 1: user generated content, 3: series content
+                    'sort_type': 1,  # 1: best, 2: recent
+                })
+
+            for replies in traverse_obj(comment_api_raw_data, ('data', 'replies', ...)):
+                yield {
+                    'author': traverse_obj(replies, ('member', 'name')),
+                    'author_id': traverse_obj(replies, ('member', 'mid')),
+                    'author_thumbnail': traverse_obj(replies, ('member', 'face')),
+                    'text': traverse_obj(replies, ('content', 'message')),
+                    'id': replies.get('rpid'),
+                    'like_count': int_or_none(replies.get('like_count')),
+                    'timestamp': unified_timestamp(replies.get('ctime_text')),
+                    'author_is_uploader': bool(traverse_obj(replies, ('member', 'type'))),
+                }
+                if replies.get('count'):
+                    yield from self._get_comments_reply(replies.get('rpid'), display_id=video_id)
+
+            if traverse_obj(comment_api_raw_data, ('data', 'cursor', 'is_end')):
+                break
+
     def _real_extract(self, url):
         season_id, ep_id, aid = self._match_valid_url(url).group('season_id', 'ep_id', 'aid')
         video_id = ep_id or aid
@@ -1087,7 +1193,8 @@ def _real_extract(self, url):
             **self._extract_video_metadata(url, video_id, season_id),
             'formats': self._get_formats(ep_id=ep_id, aid=aid),
             'subtitles': self.extract_subtitles(ep_id=ep_id, aid=aid),
-            'chapters': chapters
+            'chapters': chapters,
+            '__post_extractor': self.extract_comments(video_id, ep_id)
         }
 
 

From 979568f26ece80bca72b48f0dd57d676e431059a Mon Sep 17 00:00:00 2001
From: MyNey <20515340+MinePlayersPE@users.noreply.github.com>
Date: Thu, 13 Apr 2023 01:28:33 +0700
Subject: [PATCH 206/871] [extractor/BrainPOP] Add extractors (#6106)

Authored by: MinePlayersPE
Based on https://github.com/ytdl-org/youtube-dl/pull/10025
---
 yt_dlp/extractor/_extractors.py |   8 +
 yt_dlp/extractor/brainpop.py    | 318 ++++++++++++++++++++++++++++++++
 2 files changed, 326 insertions(+)
 create mode 100644 yt_dlp/extractor/brainpop.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index c2043bbd2..09903423d 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -254,6 +254,14 @@
     BRMediathekIE,
 )
 from .bravotv import BravoTVIE
+from .brainpop import (
+    BrainPOPIE,
+    BrainPOPJrIE,
+    BrainPOPELLIE,
+    BrainPOPEspIE,
+    BrainPOPFrIE,
+    BrainPOPIlIE,
+)
 from .breakcom import BreakIE
 from .breitbart import BreitBartIE
 from .brightcove import (
diff --git a/yt_dlp/extractor/brainpop.py b/yt_dlp/extractor/brainpop.py
new file mode 100644
index 000000000..1200437e6
--- /dev/null
+++ b/yt_dlp/extractor/brainpop.py
@@ -0,0 +1,318 @@
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    classproperty,
+    int_or_none,
+    traverse_obj,
+    urljoin
+)
+
+
+class BrainPOPBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'brainpop'
+    _ORIGIN = ''  # So that _VALID_URL doesn't crash
+    _LOGIN_ERRORS = {
+        1502: 'The username and password you entered did not match.',  # LOGIN_FAILED
+        1503: 'Payment method is expired.',  # LOGIN_FAILED_ACCOUNT_NOT_ACTIVE
+        1506: 'Your BrainPOP plan has expired.',  # LOGIN_FAILED_ACCOUNT_EXPIRED
+        1507: 'Terms not accepted.',  # LOGIN_FAILED_TERMS_NOT_ACCEPTED
+        1508: 'Account not activated.',  # LOGIN_FAILED_SUBSCRIPTION_NOT_ACTIVE
+        1512: 'The maximum number of devices permitted are logged in with your account right now.',  # LOGIN_FAILED_LOGIN_LIMIT_REACHED
+        1513: 'You are trying to access your account from outside of its allowed IP range.',  # LOGIN_FAILED_INVALID_IP
+        1514: 'Individual accounts are not included in your plan. Try again with your shared username and password.',  # LOGIN_FAILED_MBP_DISABLED
+        1515: 'Account not activated.',  # LOGIN_FAILED_TEACHER_NOT_ACTIVE
+        1523: 'That username and password won\'t work on this BrainPOP site.',  # LOGIN_FAILED_NO_ACCESS
+        1524: 'You\'ll need to join a class before you can login.',  # LOGIN_FAILED_STUDENT_NO_PERIOD
+        1526: 'Your account is locked. Reset your password, or ask a teacher or administrator for help.',  # LOGIN_FAILED_ACCOUNT_LOCKED
+    }
+
+    @classproperty
+    def _VALID_URL(cls):
+        root = re.escape(cls._ORIGIN).replace(r'https:', r'https?:').replace(r'www\.', r'(?:www\.)?')
+        return rf'{root}/(?P<slug>[^/]+/[^/]+/(?P<id>[^/?#&]+))'
+
+    def _assemble_formats(self, slug, format_id, display_id, token='', extra_fields={}):
+        formats = []
+        formats = self._extract_m3u8_formats(
+            f'{urljoin(self._HLS_URL, slug)}.m3u8?{token}',
+            display_id, 'mp4', m3u8_id=f'{format_id}-hls', fatal=False)
+        formats.append({
+            'format_id': format_id,
+            'url': f'{urljoin(self._VIDEO_URL, slug)}?{token}',
+        })
+        for f in formats:
+            f.update(extra_fields)
+        return formats
+
+    def _extract_adaptive_formats(self, data, token, display_id, key_format='%s', extra_fields={}):
+        formats = []
+        additional_key_formats = {
+            '%s': {},
+            'ad_%s': {
+                'format_note': 'Audio description',
+                'source_preference': -2
+            }
+        }
+        for additional_key_format, additional_key_fields in additional_key_formats.items():
+            for key_quality, key_index in enumerate(('high', 'low')):
+                full_key_index = additional_key_format % (key_format % key_index)
+                if data.get(full_key_index):
+                    formats.extend(self._assemble_formats(data[full_key_index], full_key_index, display_id, token, {
+                        'quality': -1 - key_quality,
+                        **additional_key_fields,
+                        **extra_fields
+                    }))
+        return formats
+
+    def _perform_login(self, username, password):
+        login_res = self._download_json(
+            'https://api.brainpop.com/api/login', None,
+            data=json.dumps({'username': username, 'password': password}).encode(),
+            headers={
+                'Content-Type': 'application/json',
+                'Referer': self._ORIGIN
+            }, note='Logging in', errnote='Unable to log in', expected_status=400)
+        status_code = int_or_none(login_res['status_code'])
+        if status_code != 1505:
+            self.report_warning(
+                f'Unable to login: {self._LOGIN_ERRORS.get(status_code) or login_res.get("message")}'
+                or f'Got status code {status_code}')
+
+
+class BrainPOPIE(BrainPOPBaseIE):
+    _ORIGIN = 'https://www.brainpop.com'
+    _VIDEO_URL = 'https://svideos.brainpop.com'
+    _HLS_URL = 'https://hls.brainpop.com'
+    _CDN_URL = 'https://cdn.brainpop.com'
+    _TESTS = [{
+        'url': 'https://www.brainpop.com/health/conflictresolution/martinlutherkingjr/movie?ref=null',
+        'md5': '3ead374233ae74c7f1b0029a01c972f0',
+        'info_dict': {
+            'id': '1f3259fa457292b4',
+            'ext': 'mp4',
+            'title': 'Martin Luther King, Jr.',
+            'display_id': 'martinlutherkingjr',
+            'description': 'md5:f403dbb2bf3ccc7cf4c59d9e43e3c349',
+        },
+    }, {
+        'url': 'https://www.brainpop.com/science/space/bigbang/',
+        'md5': '9a1ff0e77444dd9e437354eb669c87ec',
+        'info_dict': {
+            'id': 'acae52cd48c99acf',
+            'ext': 'mp4',
+            'title': 'Big Bang',
+            'display_id': 'bigbang',
+            'description': 'md5:3e53b766b0f116f631b13f4cae185d38',
+        },
+        'skip': 'Requires login',
+    }]
+
+    def _real_extract(self, url):
+        slug, display_id = self._match_valid_url(url).group('slug', 'id')
+        movie_data = self._download_json(
+            f'https://api.brainpop.com/api/content/published/bp/en/{slug}/movie?full=1', display_id,
+            'Downloading movie data JSON', 'Unable to download movie data')['data']
+        topic_data = traverse_obj(self._download_json(
+            f'https://api.brainpop.com/api/content/published/bp/en/{slug}?full=1', display_id,
+            'Downloading topic data JSON', 'Unable to download topic data', fatal=False),
+            ('data', 'topic'), expected_type=dict) or movie_data['topic']
+
+        if not traverse_obj(movie_data, ('access', 'allow')):
+            reason = traverse_obj(movie_data, ('access', 'reason'))
+            if 'logged' in reason:
+                self.raise_login_required(reason, metadata_available=True)
+            else:
+                self.raise_no_formats(reason, video_id=display_id)
+        movie_feature = movie_data['feature']
+        movie_feature_data = movie_feature['data']
+
+        formats, subtitles = [], {}
+        formats.extend(self._extract_adaptive_formats(movie_feature_data, movie_feature_data.get('token', ''), display_id, '%s_v2', {
+            'language': movie_feature.get('language') or 'en',
+            'language_preference': 10
+        }))
+        for lang, localized_feature in traverse_obj(movie_feature, 'localization', default={}, expected_type=dict).items():
+            formats.extend(self._extract_adaptive_formats(localized_feature, localized_feature.get('token', ''), display_id, '%s_v2', {
+                'language': lang,
+                'language_preference': -10
+            }))
+
+        # TODO: Do localization fields also have subtitles?
+        for name, url in movie_feature_data.items():
+            lang = self._search_regex(
+                r'^subtitles_(?P<lang>\w+)$', name, 'subtitle metadata', default=None)
+            if lang and url:
+                subtitles.setdefault(lang, []).append({
+                    'url': urljoin(self._CDN_URL, url)
+                })
+
+        return {
+            'id': topic_data['topic_id'],
+            'display_id': display_id,
+            'title': topic_data.get('name'),
+            'description': topic_data.get('synopsis'),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+
+class BrainPOPLegacyBaseIE(BrainPOPBaseIE):
+    def _parse_js_topic_data(self, topic_data, display_id, token):
+        movie_data = topic_data['movies']
+        # TODO: Are there non-burned subtitles?
+        formats = self._extract_adaptive_formats(movie_data, token, display_id)
+
+        return {
+            'id': topic_data['EntryID'],
+            'display_id': display_id,
+            'title': topic_data.get('name'),
+            'alt_title': topic_data.get('title'),
+            'description': topic_data.get('synopsis'),
+            'formats': formats,
+        }
+
+    def _real_extract(self, url):
+        slug, display_id = self._match_valid_url(url).group('slug', 'id')
+        webpage = self._download_webpage(url, display_id)
+        topic_data = self._search_json(
+            r'var\s+content\s*=\s*', webpage, 'content data',
+            display_id, end_pattern=';')['category']['unit']['topic']
+        token = self._search_regex(r'ec_token\s*:\s*[\'"]([^\'"]+)', webpage, 'video token')
+        return self._parse_js_topic_data(topic_data, display_id, token)
+
+
+class BrainPOPJrIE(BrainPOPLegacyBaseIE):
+    _ORIGIN = 'https://jr.brainpop.com'
+    _VIDEO_URL = 'https://svideos-jr.brainpop.com'
+    _HLS_URL = 'https://hls-jr.brainpop.com'
+    _CDN_URL = 'https://cdn-jr.brainpop.com'
+    _TESTS = [{
+        'url': 'https://jr.brainpop.com/health/feelingsandsel/emotions/',
+        'md5': '04e0561bb21770f305a0ce6cf0d869ab',
+        'info_dict': {
+            'id': '347',
+            'ext': 'mp4',
+            'title': 'Emotions',
+            'display_id': 'emotions',
+        },
+    }, {
+        'url': 'https://jr.brainpop.com/science/habitats/arctichabitats/',
+        'md5': 'b0ed063bbd1910df00220ee29340f5d6',
+        'info_dict': {
+            'id': '29',
+            'ext': 'mp4',
+            'title': 'Arctic Habitats',
+            'display_id': 'arctichabitats',
+        },
+        'skip': 'Requires login',
+    }]
+
+
+class BrainPOPELLIE(BrainPOPLegacyBaseIE):
+    _ORIGIN = 'https://ell.brainpop.com'
+    _VIDEO_URL = 'https://svideos-esl.brainpop.com'
+    _HLS_URL = 'https://hls-esl.brainpop.com'
+    _CDN_URL = 'https://cdn-esl.brainpop.com'
+    _TESTS = [{
+        'url': 'https://ell.brainpop.com/level1/unit1/lesson1/',
+        'md5': 'a2012700cfb774acb7ad2e8834eed0d0',
+        'info_dict': {
+            'id': '1',
+            'ext': 'mp4',
+            'title': 'Lesson 1',
+            'display_id': 'lesson1',
+            'alt_title': 'Personal Pronouns',
+        },
+    }, {
+        'url': 'https://ell.brainpop.com/level3/unit6/lesson5/',
+        'md5': 'be19c8292c87b24aacfb5fda2f3f8363',
+        'info_dict': {
+            'id': '101',
+            'ext': 'mp4',
+            'title': 'Lesson 5',
+            'display_id': 'lesson5',
+            'alt_title': 'Review: Unit 6',
+        },
+        'skip': 'Requires login',
+    }]
+
+
+class BrainPOPEspIE(BrainPOPLegacyBaseIE):
+    IE_DESC = 'BrainPOP Español'
+    _ORIGIN = 'https://esp.brainpop.com'
+    _VIDEO_URL = 'https://svideos.brainpop.com'
+    _HLS_URL = 'https://hls.brainpop.com'
+    _CDN_URL = 'https://cdn.brainpop.com/mx'
+    _TESTS = [{
+        'url': 'https://esp.brainpop.com/ciencia/la_diversidad_de_la_vida/ecosistemas/',
+        'md5': 'cb3f062db2b3c5240ddfcfde7108f8c9',
+        'info_dict': {
+            'id': '3893',
+            'ext': 'mp4',
+            'title': 'Ecosistemas',
+            'display_id': 'ecosistemas',
+            'description': 'md5:80fc55b07e241f8c8f2aa8d74deaf3c3',
+        },
+    }, {
+        'url': 'https://esp.brainpop.com/espanol/la_escritura/emily_dickinson/',
+        'md5': '98c1b9559e0e33777209c425cda7dac4',
+        'info_dict': {
+            'id': '7146',
+            'ext': 'mp4',
+            'title': 'Emily Dickinson',
+            'display_id': 'emily_dickinson',
+            'description': 'md5:2795ad87b1d239c9711c1e92ab5a978b',
+        },
+        'skip': 'Requires login',
+    }]
+
+
+class BrainPOPFrIE(BrainPOPLegacyBaseIE):
+    IE_DESC = 'BrainPOP Français'
+    _ORIGIN = 'https://fr.brainpop.com'
+    _VIDEO_URL = 'https://svideos.brainpop.com'
+    _HLS_URL = 'https://hls.brainpop.com'
+    _CDN_URL = 'https://cdn.brainpop.com/fr'
+    _TESTS = [{
+        'url': 'https://fr.brainpop.com/sciencesdelaterre/energie/sourcesdenergie/',
+        'md5': '97e7f48af8af93f8a2be11709f239371',
+        'info_dict': {
+            'id': '1651',
+            'ext': 'mp4',
+            'title': 'Sources d\'énergie',
+            'display_id': 'sourcesdenergie',
+            'description': 'md5:7eece350f019a21ef9f64d4088b2d857',
+        },
+    }, {
+        'url': 'https://fr.brainpop.com/francais/ecrire/plagiat/',
+        'md5': '0cf2b4f89804d0dd4a360a51310d445a',
+        'info_dict': {
+            'id': '5803',
+            'ext': 'mp4',
+            'title': 'Plagiat',
+            'display_id': 'plagiat',
+            'description': 'md5:4496d87127ace28e8b1eda116e77cd2b',
+        },
+        'skip': 'Requires login',
+    }]
+
+
+class BrainPOPIlIE(BrainPOPLegacyBaseIE):
+    IE_DESC = 'BrainPOP Hebrew'
+    _ORIGIN = 'https://il.brainpop.com'
+    _VIDEO_URL = 'https://svideos.brainpop.com'
+    _HLS_URL = 'https://hls.brainpop.com'
+    _CDN_URL = 'https://cdn.brainpop.com/he'
+    _TESTS = [{
+        'url': 'https://il.brainpop.com/category_9/subcategory_150/subjects_3782/',
+        'md5': '9e4ea9dc60ecd385a6e5ca12ccf31641',
+        'info_dict': {
+            'id': '3782',
+            'ext': 'mp4',
+            'title': 'md5:e993632fcda0545d9205602ec314ad67',
+            'display_id': 'subjects_3782',
+            'description': 'md5:4cc084a8012beb01f037724423a4d4ed',
+        },
+    }]

From d1483ec693c79f0b4ddf493870bcb840aca4da08 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Thu, 13 Apr 2023 16:09:20 +0900
Subject: [PATCH 207/871] [extractor/iwara] Fix typo

Authored by: Lesmiscore

Closes #6795
---
 yt_dlp/extractor/iwara.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/iwara.py b/yt_dlp/extractor/iwara.py
index ae2960af0..9dbb141fd 100644
--- a/yt_dlp/extractor/iwara.py
+++ b/yt_dlp/extractor/iwara.py
@@ -76,7 +76,7 @@ def _extract_formats(self, video_id, fileurl):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(f'http://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True)
+        video_data = self._download_json(f'https://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True)
         errmsg = video_data.get('message')
         # at this point we can actually get uploaded user info, but do we need it?
         if errmsg == 'errors.privateVideo':

From 56793f74c36899742d7abd52afb0deca97d469e1 Mon Sep 17 00:00:00 2001
From: hasezoey <hasezoey@gmail.com>
Date: Thu, 13 Apr 2023 19:17:56 +0200
Subject: [PATCH 208/871] [extractor/iwara] Fix format sorting (#6651)

Authored by: hasezoey
---
 yt_dlp/extractor/iwara.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/iwara.py b/yt_dlp/extractor/iwara.py
index 9dbb141fd..a5aad26ee 100644
--- a/yt_dlp/extractor/iwara.py
+++ b/yt_dlp/extractor/iwara.py
@@ -8,6 +8,7 @@
     OnDemandPagedList,
     int_or_none,
     mimetype2ext,
+    qualities,
     traverse_obj,
     unified_timestamp,
 )
@@ -64,13 +65,15 @@ def _extract_formats(self, video_id, fileurl):
         # https://github.com/yt-dlp/yt-dlp/issues/6549#issuecomment-1473771047
         x_version = hashlib.sha1('_'.join((paths[-1], q['expires'][0], '5nFp9kmbNnHdAFhaqMvt')).encode()).hexdigest()
 
+        preference = qualities(['preview', '360', '540', 'Source'])
+
         files = self._download_json(fileurl, video_id, headers={'X-Version': x_version})
         for fmt in files:
             yield traverse_obj(fmt, {
                 'format_id': 'name',
                 'url': ('src', ('view', 'download'), {self._proto_relative_url}),
                 'ext': ('type', {mimetype2ext}),
-                'quality': ('name', {lambda x: int_or_none(x) or 1e4}),
+                'quality': ('name', {preference}),
                 'height': ('name', {int_or_none}),
             }, get_all=False)
 
@@ -84,6 +87,11 @@ def _real_extract(self, url):
         elif errmsg:
             raise ExtractorError(f'Iwara says: {errmsg}')
 
+        if not video_data.get('fileUrl'):
+            if video_data.get('embedUrl'):
+                return self.url_result(video_data.get('embedUrl'))
+            raise ExtractorError('This video is unplayable', expected=True)
+
         return {
             'id': video_id,
             'age_limit': 18 if video_data.get('rating') == 'ecchi' else 0,  # ecchi is 'sexy' in Japanese

From 90c1f5120694105496a6ad9e3ecfc6c25de6cae1 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 13 Apr 2023 13:56:12 -0500
Subject: [PATCH 209/871] [extractor/zoom] Fix share URL extraction (#6789)

Authored by: bashonly
---
 yt_dlp/extractor/zoom.py | 90 +++++++++++++++++++++++++++-------------
 1 file changed, 62 insertions(+), 28 deletions(-)

diff --git a/yt_dlp/extractor/zoom.py b/yt_dlp/extractor/zoom.py
index eb0ab795b..3d7ccca76 100644
--- a/yt_dlp/extractor/zoom.py
+++ b/yt_dlp/extractor/zoom.py
@@ -13,8 +13,8 @@
 
 class ZoomIE(InfoExtractor):
     IE_NAME = 'zoom'
-    _VALID_URL = r'(?P<base_url>https?://(?:[^.]+\.)?zoom.us/)rec(?:ording)?/(?:play|share)/(?P<id>[A-Za-z0-9_.-]+)'
-    _TEST = {
+    _VALID_URL = r'(?P<base_url>https?://(?:[^.]+\.)?zoom.us/)rec(?:ording)?/(?P<type>play|share)/(?P<id>[A-Za-z0-9_.-]+)'
+    _TESTS = [{
         'url': 'https://economist.zoom.us/rec/play/dUk_CNBETmZ5VA2BwEl-jjakPpJ3M1pcfVYAPRsoIbEByGsLjUZtaa4yCATQuOL3der8BlTwxQePl_j0.EImBkXzTIaPvdZO5',
         'md5': 'ab445e8c911fddc4f9adc842c2c5d434',
         'info_dict': {
@@ -23,39 +23,73 @@ class ZoomIE(InfoExtractor):
             'title': 'China\'s "two sessions" and the new five-year plan',
         },
         'skip': 'Recording requires email authentication to access',
-    }
+    }, {
+        # play URL
+        'url': 'https://ffgolf.zoom.us/rec/play/qhEhXbrxq1Zoucx8CMtHzq1Z_2YZRPVCqWK_K-2FkEGRsSLDeOX8Tu4P6jtjZcRry8QhIbvKZdtr4UNo.QcPn2debFskI9whJ',
+        'md5': '2c4b1c4e5213ebf9db293e88d9385bee',
+        'info_dict': {
+            'id': 'qhEhXbrxq1Zoucx8CMtHzq1Z_2YZRPVCqWK_K-2FkEGRsSLDeOX8Tu4P6jtjZcRry8QhIbvKZdtr4UNo.QcPn2debFskI9whJ',
+            'ext': 'mp4',
+            'title': 'Prépa AF2023 - Séance 5 du 11 avril - R20/VM/GO',
+        },
+    }, {
+        # share URL
+        'url': 'https://us02web.zoom.us/rec/share/hkUk5Zxcga0nkyNGhVCRfzkA2gX_mzgS3LpTxEEWJz9Y_QpIQ4mZFOUx7KZRZDQA.9LGQBdqmDAYgiZ_8',
+        'md5': '90fdc7cfcaee5d52d1c817fc03c43c9b',
+        'info_dict': {
+            'id': 'hkUk5Zxcga0nkyNGhVCRfzkA2gX_mzgS3LpTxEEWJz9Y_QpIQ4mZFOUx7KZRZDQA.9LGQBdqmDAYgiZ_8',
+            'ext': 'mp4',
+            'title': 'Timea Andrea Lelik\'s Personal Meeting Room',
+        },
+    }]
 
-    def _real_extract(self, url):
-        base_url, play_id = self._match_valid_url(url).groups()
-        webpage = self._download_webpage(url, play_id)
+    def _get_page_data(self, webpage, video_id):
+        return self._search_json(
+            r'window\.__data__\s*=', webpage, 'data', video_id, transform_source=js_to_json)
 
+    def _get_real_webpage(self, url, base_url, video_id, url_type):
+        webpage = self._download_webpage(url, video_id, note=f'Downloading {url_type} webpage')
         try:
             form = self._form_hidden_inputs('password_form', webpage)
         except ExtractorError:
-            form = None
-        if form:
-            password = self.get_param('videopassword')
-            if not password:
-                raise ExtractorError(
-                    'This video is protected by a passcode, use the --video-password option', expected=True)
-            is_meeting = form.get('useWhichPasswd') == 'meeting'
-            validation = self._download_json(
-                base_url + 'rec/validate%s_passwd' % ('_meet' if is_meeting else ''),
-                play_id, 'Validating passcode', 'Wrong passcode', data=urlencode_postdata({
-                    'id': form[('meet' if is_meeting else 'file') + 'Id'],
-                    'passwd': password,
-                    'action': form.get('action'),
-                }))
-            if not validation.get('status'):
-                raise ExtractorError(validation['errorMessage'], expected=True)
-            webpage = self._download_webpage(url, play_id)
+            return webpage
 
-        data = self._parse_json(self._search_regex(
-            r'(?s)window\.__data__\s*=\s*({.+?});',
-            webpage, 'data'), play_id, js_to_json)
+        password = self.get_param('videopassword')
+        if not password:
+            raise ExtractorError(
+                'This video is protected by a passcode, use the --video-password option', expected=True)
+        is_meeting = form.get('useWhichPasswd') == 'meeting'
+        validation = self._download_json(
+            base_url + 'rec/validate%s_passwd' % ('_meet' if is_meeting else ''),
+            video_id, 'Validating passcode', 'Wrong passcode', data=urlencode_postdata({
+                'id': form[('meet' if is_meeting else 'file') + 'Id'],
+                'passwd': password,
+                'action': form.get('action'),
+            }))
+        if not validation.get('status'):
+            raise ExtractorError(validation['errorMessage'], expected=True)
+        return self._download_webpage(url, video_id, note=f'Re-downloading {url_type} webpage')
+
+    def _real_extract(self, url):
+        base_url, url_type, video_id = self._match_valid_url(url).group('base_url', 'type', 'id')
+
+        if url_type == 'share':
+            webpage = self._get_real_webpage(url, base_url, video_id, 'share')
+            meeting_id = self._get_page_data(webpage, video_id)['meetingId']
+            redirect_path = self._download_json(
+                f'{base_url}nws/recording/1.0/play/share-info/{meeting_id}',
+                video_id, note='Downloading share info JSON')['result']['redirectUrl']
+            url = urljoin(base_url, redirect_path)
+
+        webpage = self._get_real_webpage(url, base_url, video_id, 'play')
+        file_id = self._get_page_data(webpage, video_id)['fileId']
+        if not file_id:
+            # When things go wrong, file_id can be empty string
+            raise ExtractorError('Unable to extract file ID')
 
         data = self._download_json(
-            f'{base_url}nws/recording/1.0/play/info/{data["fileId"]}', play_id)['result']
+            f'{base_url}nws/recording/1.0/play/info/{file_id}', video_id,
+            note='Downloading play info JSON')['result']
 
         subtitles = {}
         for _type in ('transcript', 'cc', 'chapter'):
@@ -91,7 +125,7 @@ def _real_extract(self, url):
             })
 
         return {
-            'id': play_id,
+            'id': video_id,
             'title': str_or_none(traverse_obj(data, ('meet', 'topic'))),
             'subtitles': subtitles,
             'formats': formats,

From 925936908a3c3ee0e508621db14696b9f6a8b563 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 13 Apr 2023 14:05:57 -0500
Subject: [PATCH 210/871] [extractor/tiktok] Fix and improve metadata
 extraction (#6777)

Authored by: bashonly
---
 yt_dlp/extractor/tiktok.py | 187 +++++++++++++++++++++++--------------
 1 file changed, 116 insertions(+), 71 deletions(-)

diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index fb838d529..63708229e 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -287,17 +287,15 @@ def extract_addr(addr, add_meta={}):
         thumbnails = []
         for cover_id in ('cover', 'ai_dynamic_cover', 'animated_cover', 'ai_dynamic_cover_bak',
                          'origin_cover', 'dynamic_cover'):
-            cover = video_info.get(cover_id)
-            if cover:
-                for cover_url in cover['url_list']:
-                    thumbnails.append({
-                        'id': cover_id,
-                        'url': cover_url,
-                    })
+            for cover_url in traverse_obj(video_info, (cover_id, 'url_list', ...)):
+                thumbnails.append({
+                    'id': cover_id,
+                    'url': cover_url,
+                })
 
-        stats_info = aweme_detail.get('statistics', {})
-        author_info = aweme_detail.get('author', {})
-        music_info = aweme_detail.get('music', {})
+        stats_info = aweme_detail.get('statistics') or {}
+        author_info = aweme_detail.get('author') or {}
+        music_info = aweme_detail.get('music') or {}
         user_url = self._UPLOADER_URL_FORMAT % (traverse_obj(author_info,
                                                              'sec_uid', 'id', 'uid', 'unique_id',
                                                              expected_type=str_or_none, get_all=False))
@@ -319,20 +317,27 @@ def extract_addr(addr, add_meta={}):
             'extractor_key': TikTokIE.ie_key(),
             'extractor': TikTokIE.IE_NAME,
             'webpage_url': self._create_url(author_info.get('uid'), aweme_id),
-            'title': aweme_detail.get('desc'),
-            'description': aweme_detail.get('desc'),
-            'view_count': int_or_none(stats_info.get('play_count')),
-            'like_count': int_or_none(stats_info.get('digg_count')),
-            'repost_count': int_or_none(stats_info.get('share_count')),
-            'comment_count': int_or_none(stats_info.get('comment_count')),
-            'uploader': str_or_none(author_info.get('unique_id')),
-            'creator': str_or_none(author_info.get('nickname')),
-            'uploader_id': str_or_none(author_info.get('uid')),
+            **traverse_obj(aweme_detail, {
+                'title': ('desc', {str}),
+                'description': ('desc', {str}),
+                'timestamp': ('create_time', {int_or_none}),
+            }),
+            **traverse_obj(stats_info, {
+                'view_count': 'play_count',
+                'like_count': 'digg_count',
+                'repost_count': 'share_count',
+                'comment_count': 'comment_count',
+            }, expected_type=int_or_none),
+            **traverse_obj(author_info, {
+                'uploader': 'unique_id',
+                'uploader_id': 'uid',
+                'creator': 'nickname',
+                'channel_id': 'sec_uid',
+            }, expected_type=str_or_none),
             'uploader_url': user_url,
             'track': music_track,
             'album': str_or_none(music_info.get('album')) or None,
             'artist': music_author or None,
-            'timestamp': int_or_none(aweme_detail.get('create_time')),
             'formats': formats,
             'subtitles': self.extract_subtitles(aweme_detail, aweme_id),
             'thumbnails': thumbnails,
@@ -344,37 +349,27 @@ def extract_addr(addr, add_meta={}):
             '_format_sort_fields': ('quality', 'codec', 'size', 'br'),
         }
 
-    def _parse_aweme_video_web(self, aweme_detail, webpage_url):
+    def _parse_aweme_video_web(self, aweme_detail, webpage_url, video_id):
         video_info = aweme_detail['video']
         author_info = traverse_obj(aweme_detail, 'authorInfo', 'author', expected_type=dict, default={})
         music_info = aweme_detail.get('music') or {}
         stats_info = aweme_detail.get('stats') or {}
-        user_url = self._UPLOADER_URL_FORMAT % (traverse_obj(author_info,
-                                                             'secUid', 'id', 'uid', 'uniqueId',
-                                                             expected_type=str_or_none, get_all=False)
-                                                or aweme_detail.get('authorSecId'))
+        channel_id = traverse_obj(author_info or aweme_detail, (('authorSecId', 'secUid'), {str}), get_all=False)
+        user_url = self._UPLOADER_URL_FORMAT % channel_id if channel_id else None
 
         formats = []
-        play_url = video_info.get('playAddr')
-        width = video_info.get('width')
-        height = video_info.get('height')
-        if isinstance(play_url, str):
-            formats = [{
+        width = int_or_none(video_info.get('width'))
+        height = int_or_none(video_info.get('height'))
+
+        for play_url in traverse_obj(video_info, ('playAddr', ((..., 'src'), None), {url_or_none})):
+            formats.append({
                 'url': self._proto_relative_url(play_url),
                 'ext': 'mp4',
                 'width': width,
                 'height': height,
-            }]
-        elif isinstance(play_url, list):
-            formats = [{
-                'url': self._proto_relative_url(url),
-                'ext': 'mp4',
-                'width': width,
-                'height': height,
-            } for url in traverse_obj(play_url, (..., 'src'), expected_type=url_or_none) if url]
+            })
 
-        download_url = url_or_none(video_info.get('downloadAddr')) or traverse_obj(video_info, ('download', 'url'), expected_type=url_or_none)
-        if download_url:
+        for download_url in traverse_obj(video_info, (('downloadAddr', ('download', 'url')), {url_or_none})):
             formats.append({
                 'format_id': 'download',
                 'url': self._proto_relative_url(download_url),
@@ -382,38 +377,48 @@ def _parse_aweme_video_web(self, aweme_detail, webpage_url):
                 'width': width,
                 'height': height,
             })
+
         self._remove_duplicate_formats(formats)
 
         thumbnails = []
-        for thumbnail_name in ('thumbnail', 'cover', 'dynamicCover', 'originCover'):
-            if aweme_detail.get(thumbnail_name):
-                thumbnails = [{
-                    'url': self._proto_relative_url(aweme_detail[thumbnail_name]),
-                    'width': width,
-                    'height': height
-                }]
+        for thumb_url in traverse_obj(aweme_detail, (
+                (None, 'video'), ('thumbnail', 'cover', 'dynamicCover', 'originCover'), {url_or_none})):
+            thumbnails.append({
+                'url': self._proto_relative_url(thumb_url),
+                'width': width,
+                'height': height,
+            })
 
         return {
-            'id': traverse_obj(aweme_detail, 'id', 'awemeId', expected_type=str_or_none),
-            'title': aweme_detail.get('desc'),
-            'duration': try_get(aweme_detail, lambda x: x['video']['duration'], int),
-            'view_count': int_or_none(stats_info.get('playCount')),
-            'like_count': int_or_none(stats_info.get('diggCount')),
-            'repost_count': int_or_none(stats_info.get('shareCount')),
-            'comment_count': int_or_none(stats_info.get('commentCount')),
-            'timestamp': int_or_none(aweme_detail.get('createTime')),
-            'creator': str_or_none(author_info.get('nickname')),
-            'uploader': str_or_none(author_info.get('uniqueId') or aweme_detail.get('author')),
-            'uploader_id': str_or_none(traverse_obj(author_info, 'id', 'uid', 'authorId')),
+            'id': video_id,
+            **traverse_obj(aweme_detail, {
+                'title': ('desc', {str}),
+                'description': ('desc', {str}),
+                'duration': ('video', 'duration', {int_or_none}),
+                'timestamp': ('createTime', {int_or_none}),
+            }),
+            **traverse_obj(author_info or aweme_detail, {
+                'creator': ('nickname', {str}),
+                'uploader': (('uniqueId', 'author'), {str}),
+                'uploader_id': (('authorId', 'uid', 'id'), {str_or_none}),
+            }, get_all=False),
+            **traverse_obj(stats_info, {
+                'view_count': 'playCount',
+                'like_count': 'diggCount',
+                'repost_count': 'shareCount',
+                'comment_count': 'commentCount',
+            }, expected_type=int_or_none),
+            **traverse_obj(music_info, {
+                'track': 'title',
+                'album': ('album', {lambda x: x or None}),
+                'artist': 'authorName',
+            }, expected_type=str),
+            'channel_id': channel_id,
             'uploader_url': user_url,
-            'track': str_or_none(music_info.get('title')),
-            'album': str_or_none(music_info.get('album')) or None,
-            'artist': str_or_none(music_info.get('authorName')),
             'formats': formats,
             'thumbnails': thumbnails,
-            'description': str_or_none(aweme_detail.get('desc')),
             'http_headers': {
-                'Referer': webpage_url
+                'Referer': webpage_url,
             }
         }
 
@@ -447,7 +452,8 @@ class TikTokIE(TikTokBaseIE):
             'artist': 'Ysrbeats',
             'album': 'Lehanga',
             'track': 'Lehanga',
-        }
+        },
+        'skip': '404 Not Found',
     }, {
         'url': 'https://www.tiktok.com/@patroxofficial/video/6742501081818877190?langCountry=en',
         'md5': '6f3cf8cdd9b28cb8363fe0a9a160695b',
@@ -462,6 +468,7 @@ class TikTokIE(TikTokBaseIE):
             'uploader': 'patrox',
             'uploader_id': '18702747',
             'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAiFnldaILebi5heDoVU6bn4jBWWycX6-9U3xuNPqZ8Ws',
+            'channel_id': 'MS4wLjABAAAAiFnldaILebi5heDoVU6bn4jBWWycX6-9U3xuNPqZ8Ws',
             'creator': 'patroX',
             'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?',
             'upload_date': '20190930',
@@ -472,7 +479,7 @@ class TikTokIE(TikTokBaseIE):
             'comment_count': int,
             'artist': 'Evan Todd, Jessica Keenan Wynn, Alice Lee, Barrett Wilbert Weed & Jon Eidson',
             'track': 'Big Fun',
-        }
+        },
     }, {
         # Banned audio, only available on the app
         'url': 'https://www.tiktok.com/@barudakhb_/video/6984138651336838402',
@@ -485,6 +492,7 @@ class TikTokIE(TikTokBaseIE):
             'creator': 'md5:29f238c49bc0c176cb3cef1a9cea9fa6',
             'uploader_id': '6974687867511718913',
             'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAbhBwQC-R1iKoix6jDFsF-vBdfx2ABoDjaZrM9fX6arU3w71q3cOWgWuTXn1soZ7d',
+            'channel_id': 'MS4wLjABAAAAbhBwQC-R1iKoix6jDFsF-vBdfx2ABoDjaZrM9fX6arU3w71q3cOWgWuTXn1soZ7d',
             'track': 'Boka Dance',
             'artist': 'md5:29f238c49bc0c176cb3cef1a9cea9fa6',
             'timestamp': 1626121503,
@@ -495,7 +503,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
-        }
+        },
     }, {
         # Sponsored video, only available with feed workaround
         'url': 'https://www.tiktok.com/@MS4wLjABAAAATh8Vewkn0LYM7Fo03iec3qKdeCUOcBIouRk1mkiag6h3o_pQu_dUXvZ2EZlGST7_/video/7042692929109986561',
@@ -508,6 +516,7 @@ class TikTokIE(TikTokBaseIE):
             'creator': 'Slap And Run',
             'uploader_id': '7036055384943690754',
             'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAATh8Vewkn0LYM7Fo03iec3qKdeCUOcBIouRk1mkiag6h3o_pQu_dUXvZ2EZlGST7_',
+            'channel_id': 'MS4wLjABAAAATh8Vewkn0LYM7Fo03iec3qKdeCUOcBIouRk1mkiag6h3o_pQu_dUXvZ2EZlGST7_',
             'track': 'Promoted Music',
             'timestamp': 1639754738,
             'duration': 30,
@@ -518,7 +527,6 @@ class TikTokIE(TikTokBaseIE):
             'repost_count': int,
             'comment_count': int,
         },
-        'expected_warnings': ['trying with webpage', 'Unable to find video in feed']
     }, {
         # Video without title and description
         'url': 'https://www.tiktok.com/@pokemonlife22/video/7059698374567611694',
@@ -531,6 +539,7 @@ class TikTokIE(TikTokBaseIE):
             'creator': 'Pokemon',
             'uploader_id': '6820838815978423302',
             'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAA0tF1nBwQVVMyrGu3CqttkNgM68Do1OXUFuCY0CRQk8fEtSVDj89HqoqvbSTmUP2W',
+            'channel_id': 'MS4wLjABAAAA0tF1nBwQVVMyrGu3CqttkNgM68Do1OXUFuCY0CRQk8fEtSVDj89HqoqvbSTmUP2W',
             'track': 'original sound',
             'timestamp': 1643714123,
             'duration': 6,
@@ -577,6 +586,7 @@ class TikTokIE(TikTokBaseIE):
             'uploader': '_le_cannibale_',
             'uploader_id': '6604511138619654149',
             'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAoShJqaw_5gvy48y3azFeFcT4jeyKWbB0VVYasOCt2tTLwjNFIaDcHAM4D-QGXFOP',
+            'channel_id': 'MS4wLjABAAAAoShJqaw_5gvy48y3azFeFcT4jeyKWbB0VVYasOCt2tTLwjNFIaDcHAM4D-QGXFOP',
             'artist': 'nathan !',
             'track': 'grahamscott canon',
             'upload_date': '20220905',
@@ -587,6 +597,33 @@ class TikTokIE(TikTokBaseIE):
             'comment_count': int,
             'thumbnail': r're:^https://.+\.webp',
         },
+    }, {
+        # only available via web
+        'url': 'https://www.tiktok.com/@moxypatch/video/7206382937372134662',
+        'md5': '8d8c0be14127020cd9f5def4a2e6b411',
+        'info_dict': {
+            'id': '7206382937372134662',
+            'ext': 'mp4',
+            'title': 'md5:1d95c0b96560ca0e8a231af4172b2c0a',
+            'description': 'md5:1d95c0b96560ca0e8a231af4172b2c0a',
+            'creator': 'MoxyPatch',
+            'uploader': 'moxypatch',
+            'uploader_id': '7039142049363379205',
+            'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAFhqKnngMHJSsifL0w1vFOP5kn3Ndo1ODp0XuIBkNMBCkALTvwILdpu12g3pTtL4V',
+            'channel_id': 'MS4wLjABAAAAFhqKnngMHJSsifL0w1vFOP5kn3Ndo1ODp0XuIBkNMBCkALTvwILdpu12g3pTtL4V',
+            'artist': 'your worst nightmare',
+            'track': 'original sound',
+            'upload_date': '20230303',
+            'timestamp': 1677866781,
+            'duration': 10,
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+            'thumbnail': r're:^https://.+',
+            'thumbnails': 'count:3',
+        },
+        'expected_warnings': ['Unable to find video in feed'],
     }, {
         # Auto-captions available
         'url': 'https://www.tiktok.com/@hankgreen1/video/7047596209028074758',
@@ -612,7 +649,7 @@ def _real_extract(self, url):
             video_data = traverse_obj(sigi_data, ('ItemModule', video_id), expected_type=dict)
 
         if status == 0:
-            return self._parse_aweme_video_web(video_data, url)
+            return self._parse_aweme_video_web(video_data, url, video_id)
         elif status == 10216:
             raise ExtractorError('This video is private', expected=True)
         raise ExtractorError('Video not available', video_id=video_id)
@@ -839,6 +876,7 @@ class DouyinIE(TikTokBaseIE):
             'description': '#杨超越  小小水手带你去远航❤️',
             'uploader_id': '110403406559',
             'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
+            'channel_id': 'MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
             'creator': '杨超越',
             'duration': 19782,
             'timestamp': 1620905839,
@@ -848,6 +886,7 @@ class DouyinIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'thumbnail': r're:https?://.+\.jpe?g',
         },
     }, {
         'url': 'https://www.douyin.com/video/6982497745948921092',
@@ -859,8 +898,9 @@ class DouyinIE(TikTokBaseIE):
             'description': '这个夏日和小羊@杨超越 一起遇见白色幻想',
             'uploader_id': '408654318141572',
             'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAZJpnglcjW2f_CMVcnqA_6oVBXKWMpH0F8LIHuUu8-lA',
+            'channel_id': 'MS4wLjABAAAAZJpnglcjW2f_CMVcnqA_6oVBXKWMpH0F8LIHuUu8-lA',
             'creator': '杨超越工作室',
-            'duration': 42608,
+            'duration': 42479,
             'timestamp': 1625739481,
             'upload_date': '20210708',
             'track': '@杨超越工作室创作的原声',
@@ -868,6 +908,7 @@ class DouyinIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'thumbnail': r're:https?://.+\.jpe?g',
         },
     }, {
         'url': 'https://www.douyin.com/video/6953975910773099811',
@@ -879,8 +920,9 @@ class DouyinIE(TikTokBaseIE):
             'description': '#一起看海  出现在你的夏日里',
             'uploader_id': '110403406559',
             'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
+            'channel_id': 'MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
             'creator': '杨超越',
-            'duration': 17228,
+            'duration': 17343,
             'timestamp': 1619098692,
             'upload_date': '20210422',
             'track': '@杨超越创作的原声',
@@ -888,6 +930,7 @@ class DouyinIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'thumbnail': r're:https?://.+\.jpe?g',
         },
     }, {
         'url': 'https://www.douyin.com/video/6950251282489675042',
@@ -916,6 +959,7 @@ class DouyinIE(TikTokBaseIE):
             'description': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
             'uploader_id': '110403406559',
             'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
+            'channel_id': 'MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
             'creator': '杨超越',
             'duration': 15115,
             'timestamp': 1621261163,
@@ -925,6 +969,7 @@ class DouyinIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'thumbnail': r're:https?://.+\.jpe?g',
         },
     }]
     _APP_VERSIONS = [('23.3.0', '230300')]
@@ -956,7 +1001,7 @@ def _real_extract(self, url):
 
         render_data = self._parse_json(
             render_data_json, video_id, transform_source=compat_urllib_parse_unquote)
-        return self._parse_aweme_video_web(get_first(render_data, ('aweme', 'detail')), url)
+        return self._parse_aweme_video_web(get_first(render_data, ('aweme', 'detail')), url, video_id)
 
 
 class TikTokVMIE(InfoExtractor):

From 3f7e2bd80e3c5d8a1682f20a1b245fcd974f295d Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 13 Apr 2023 14:21:09 -0500
Subject: [PATCH 211/871] [FFmpegFixupM3u8PP] Check audio codec before fixup
 (#6778)

Closes #6673
Authored by: bashonly
---
 yt_dlp/postprocessor/ffmpeg.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py
index 63fc9ace6..323f4303c 100644
--- a/yt_dlp/postprocessor/ffmpeg.py
+++ b/yt_dlp/postprocessor/ffmpeg.py
@@ -898,8 +898,11 @@ def _needs_fixup(self, info):
     @PostProcessor._restrict_to(images=False)
     def run(self, info):
         if all(self._needs_fixup(info)):
+            args = ['-f', 'mp4']
+            if self.get_audio_codec(info['filepath']) == 'aac':
+                args.extend(['-bsf:a', 'aac_adtstoasc'])
             self._fixup('Fixing MPEG-TS in MP4 container', info['filepath'], [
-                *self.stream_copy_opts(), '-f', 'mp4', '-bsf:a', 'aac_adtstoasc'])
+                *self.stream_copy_opts(), *args])
         return [], info
 
 

From 93e7c6995e07dafb9dcc06c0d06acf6c5bdfecc5 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 13 Apr 2023 14:36:06 -0500
Subject: [PATCH 212/871] [extractor/generic] Attempt to detect live HLS
 (#6775)

* Extract duration for non-live generic HLS videos
* Add extractor-arg `is_live` to bypass live HLS check

Closes #6705
Authored by: bashonly
---
 README.md                   |  1 +
 yt_dlp/extractor/generic.py | 63 +++++++++++++++++++++++++++++++------
 2 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 3e8484314..35229f728 100644
--- a/README.md
+++ b/README.md
@@ -1800,6 +1800,7 @@ #### generic
 * `fragment_query`: Passthrough any query in mpd/m3u8 manifest URLs to their fragments if no value is provided, or else apply the query string given as `fragment_query=VALUE`. Does not apply to ffmpeg
 * `variant_query`: Passthrough the master m3u8 URL query to its variant playlist URLs if no value is provided, or else apply the query string given as `variant_query=VALUE`
 * `hls_key`: An HLS AES-128 key URI *or* key (as hex), and optionally the IV (as hex), in the form of `(URI|KEY)[,IV]`; e.g. `generic:hls_key=ABCDEF1234567980,0xFEDCBA0987654321`. Passing any of these values will force usage of the native HLS downloader and override the corresponding values found in the m3u8 playlist
+* `is_live`: Bypass live HLS detection and manually set `live_status` - a value of `false` will set `not_live`, any other value (or no value) will set `is_live`
 
 #### funimation
 * `language`: Audio languages to extract, e.g. `funimation:language=english,japanese`
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 75355aeb5..87cf11d6b 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -14,6 +14,7 @@
     ExtractorError,
     UnsupportedError,
     determine_ext,
+    determine_protocol,
     dict_get,
     extract_basic_auth,
     format_field,
@@ -867,7 +868,7 @@ class GenericIE(InfoExtractor):
             },
         },
         {
-            # Video.js embed, multiple formats
+            # Youtube embed, formerly: Video.js embed, multiple formats
             'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
             'info_dict': {
                 'id': 'yygqldloqIk',
@@ -894,6 +895,7 @@ class GenericIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': '404 Not Found',
         },
         # rtl.nl embed
         {
@@ -2169,6 +2171,33 @@ class GenericIE(InfoExtractor):
                 'age_limit': 18,
             },
         },
+        {
+            'note': 'Live HLS direct link',
+            'url': 'https://d18j67ugtrocuq.cloudfront.net/out/v1/2767aec339144787926bd0322f72c6e9/index.m3u8',
+            'info_dict': {
+                'id': 'index',
+                'title': r're:index',
+                'ext': 'mp4',
+                'live_status': 'is_live',
+            },
+            'params': {
+                'skip_download': 'm3u8',
+            },
+        },
+        {
+            'note': 'Video.js VOD HLS',
+            'url': 'https://gist.githubusercontent.com/bashonly/2aae0862c50f4a4b84f220c315767208/raw/e3380d413749dabbe804c9c2d8fd9a45142475c7/videojs_hls_test.html',
+            'info_dict': {
+                'id': 'videojs_hls_test',
+                'title': 'video',
+                'ext': 'mp4',
+                'age_limit': 0,
+                'duration': 1800,
+            },
+            'params': {
+                'skip_download': 'm3u8',
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -2205,6 +2234,22 @@ def _extra_manifest_info(self, info, manifest_url):
             for fmt in self._downloader._get_formats(info):
                 fmt['url'] = update_url_query(fmt['url'], query)
 
+        # Attempt to detect live HLS or set VOD duration
+        m3u8_format = next((f for f in self._downloader._get_formats(info)
+                            if determine_protocol(f) == 'm3u8_native'), None)
+        if m3u8_format:
+            is_live = self._configuration_arg('is_live', [None])[0]
+            if is_live is not None:
+                info['live_status'] = 'not_live' if is_live == 'false' else 'is_live'
+                return
+            headers = m3u8_format.get('http_headers') or info.get('http_headers')
+            duration = self._extract_m3u8_vod_duration(
+                m3u8_format['url'], info.get('id'), note='Checking m3u8 live status',
+                errnote='Failed to download m3u8 media playlist', headers=headers)
+            if not duration:
+                info['live_status'] = 'is_live'
+            info['duration'] = info.get('duration') or duration
+
     def _extract_rss(self, url, video_id, doc):
         NS_MAP = {
             'itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
@@ -2580,8 +2625,7 @@ def _extract_embeds(self, url, webpage, *, urlh=None, info_dict={}):
             varname = mobj.group(1)
             sources = variadic(self._parse_json(
                 mobj.group(2), video_id, transform_source=js_to_json, fatal=False) or [])
-            formats = []
-            subtitles = {}
+            formats, subtitles, src = [], {}, None
             for source in sources:
                 src = source.get('src')
                 if not src or not isinstance(src, str):
@@ -2604,8 +2648,6 @@ def _extract_embeds(self, url, webpage, *, urlh=None, info_dict={}):
                         m3u8_id='hls', fatal=False)
                     formats.extend(fmts)
                     self._merge_subtitles(subs, target=subtitles)
-                for fmt in formats:
-                    self._extra_manifest_info(fmt, src)
 
                 if not formats:
                     formats.append({
@@ -2621,11 +2663,11 @@ def _extract_embeds(self, url, webpage, *, urlh=None, info_dict={}):
             for sub_match in re.finditer(rf'(?s){re.escape(varname)}' r'\.addRemoteTextTrack\(({.+?})\s*,\s*(?:true|false)\)', webpage):
                 sub = self._parse_json(
                     sub_match.group(1), video_id, transform_source=js_to_json, fatal=False) or {}
-                src = str_or_none(sub.get('src'))
-                if not src:
+                sub_src = str_or_none(sub.get('src'))
+                if not sub_src:
                     continue
                 subtitles.setdefault(dict_get(sub, ('language', 'srclang')) or 'und', []).append({
-                    'url': urllib.parse.urljoin(url, src),
+                    'url': urllib.parse.urljoin(url, sub_src),
                     'name': sub.get('label'),
                     'http_headers': {
                         'Referer': actual_url,
@@ -2633,7 +2675,10 @@ def _extract_embeds(self, url, webpage, *, urlh=None, info_dict={}):
                 })
             if formats or subtitles:
                 self.report_detected('video.js embed')
-                return [{'formats': formats, 'subtitles': subtitles}]
+                info_dict = {'formats': formats, 'subtitles': subtitles}
+                if formats:
+                    self._extra_manifest_info(info_dict, src)
+                return [info_dict]
 
         # Look for generic KVS player (before json-ld bc of some urls that break otherwise)
         found = self._search_regex((

From 7666b93604b97e9ada981c6b04ccf5605dd1bd44 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Fri, 14 Apr 2023 07:58:36 +0000
Subject: [PATCH 213/871] [extractor/youtube] Define strict uploader metadata
 mapping (#6384)

New mapping:
```
channel -> channel name
channel_id -> UCID
channel_url -> UCID channel url

uploader -> channel name (same as channel field)
uploader_id -> @handle
uploader_url -> @handle channel url
```

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 709 +++++++++++++++++++-----------------
 1 file changed, 371 insertions(+), 338 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index d6a55e953..2b17751e5 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -458,6 +458,23 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 
+    _YT_HANDLE_RE = r'@[\w.-]{3,30}'  # https://support.google.com/youtube/answer/11585688?hl=en
+    _YT_CHANNEL_UCID_RE = r'UC[\w-]{22}'
+
+    def ucid_or_none(self, ucid):
+        return self._search_regex(rf'^({self._YT_CHANNEL_UCID_RE})$', ucid, 'UC-id', default=None)
+
+    def handle_or_none(self, handle):
+        return self._search_regex(rf'^({self._YT_HANDLE_RE})$', handle, '@-handle', default=None)
+
+    def handle_from_url(self, url):
+        return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_HANDLE_RE})',
+                                  url, 'channel handle', default=None)
+
+    def ucid_from_url(self, url):
+        return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_CHANNEL_UCID_RE})',
+                                  url, 'channel id', default=None)
+
     @functools.cached_property
     def _preferred_lang(self):
         """
@@ -992,6 +1009,8 @@ def _extract_video(self, renderer):
         if not channel_id:
             channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
 
+        channel_id = self.ucid_or_none(channel_id)
+
         overlay_style = traverse_obj(
             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
             get_all=False, expected_type=str)
@@ -1233,9 +1252,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
-                'uploader': 'Philipp Hagemeister',
-                'uploader_id': 'phihag',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
                 'channel': 'Philipp Hagemeister',
                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
@@ -1254,7 +1270,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'start_time': 1,
                 'end_time': 9,
                 'comment_count': int,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Philipp Hagemeister',
+                'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
+                'uploader_id': '@PhilippHagemeister',
             }
         },
         {
@@ -1266,9 +1285,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20120608',
                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
-                'uploader': 'SET India',
-                'uploader_id': 'setindia',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
                 'age_limit': 18,
             },
             'skip': 'Private video',
@@ -1280,9 +1296,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
-                'uploader': 'Philipp Hagemeister',
-                'uploader_id': 'phihag',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
                 'channel': 'Philipp Hagemeister',
                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
@@ -1299,7 +1312,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'live_status': 'not_live',
                 'age_limit': 0,
                 'comment_count': int,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Philipp Hagemeister',
+                'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
+                'uploader_id': '@PhilippHagemeister',
             },
             'params': {
                 'skip_download': True,
@@ -1312,10 +1328,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'a9LDPn-MO4I',
                 'ext': 'm4a',
                 'upload_date': '20121002',
-                'uploader_id': '8KVIDEO',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
                 'description': '',
-                'uploader': '8KVIDEO',
                 'title': 'UHDTV TEST 8K VIDEO.mp4'
             },
             'params': {
@@ -1333,8 +1346,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
                 'duration': 244,
-                'uploader': 'AfrojackVEVO',
-                'uploader_id': 'AfrojackVEVO',
                 'upload_date': '20131011',
                 'abr': 129.495,
                 'like_count': int,
@@ -1346,13 +1357,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'live_status': 'not_live',
                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
                 'channel': 'Afrojack',
-                'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
                 'tags': 'count:19',
                 'availability': 'public',
                 'categories': ['Music'],
                 'age_limit': 0,
                 'alt_title': 'The Spark',
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Afrojack',
+                'uploader_url': 'https://www.youtube.com/@Afrojack',
+                'uploader_id': '@Afrojack',
             },
             'params': {
                 'youtube_include_dash_manifest': True,
@@ -1369,9 +1382,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'duration': 142,
-                'uploader': 'The Witcher',
-                'uploader_id': 'WitcherGame',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
                 'upload_date': '20140605',
                 'age_limit': 18,
                 'categories': ['Gaming'],
@@ -1385,7 +1395,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
                 'playable_in_embed': True,
                 'view_count': int,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'The Witcher',
+                'uploader_url': 'https://www.youtube.com/@thewitcher',
+                'uploader_id': '@thewitcher',
             },
         },
         {
@@ -1397,12 +1410,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Godzilla 2 (Official Video)',
                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
                 'upload_date': '20200408',
-                'uploader_id': 'FlyingKitty900',
-                'uploader': 'FlyingKitty',
                 'age_limit': 18,
                 'availability': 'needs_auth',
                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
-                'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
                 'channel': 'FlyingKitty',
                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
                 'view_count': int,
@@ -1413,7 +1423,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'like_count': int,
                 'duration': 177,
                 'playable_in_embed': True,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'FlyingKitty',
+                'uploader_url': 'https://www.youtube.com/@FlyingKitty900',
+                'uploader_id': '@FlyingKitty900',
             },
         },
         {
@@ -1424,13 +1437,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
                 'ext': 'mp4',
                 'upload_date': '20191228',
-                'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
-                'uploader': 'Projekt Melody',
                 'description': 'md5:17eccca93a786d51bc67646756894066',
                 'age_limit': 18,
                 'like_count': int,
                 'availability': 'needs_auth',
-                'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
                 'view_count': int,
                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
@@ -1442,7 +1452,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 106,
                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
                 'comment_count': int,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Projekt Melody',
+                'uploader_url': 'https://www.youtube.com/@ProjektMelody',
+                'uploader_id': '@ProjektMelody',
             },
         },
         {
@@ -1452,8 +1465,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'MeJVWBSsPAY',
                 'ext': 'mp4',
                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
-                'uploader': 'Herr Lurik',
-                'uploader_id': 'st3in234',
                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
                 'upload_date': '20130730',
                 'track': 'Such mich find mich',
@@ -1470,11 +1481,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
                 'categories': ['Music'],
                 'availability': 'public',
-                'uploader_url': 'http://www.youtube.com/user/st3in234',
                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
                 'live_status': 'not_live',
                 'artist': 'OOMPH!',
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Herr Lurik',
+                'uploader_url': 'https://www.youtube.com/@HerrLurik',
+                'uploader_id': '@HerrLurik',
             },
         },
         {
@@ -1491,11 +1504,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'duration': 266,
                 'upload_date': '20100430',
-                'uploader_id': 'deadmau5',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
                 'creator': 'deadmau5',
                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
-                'uploader': 'deadmau5',
                 'title': 'Deadmau5 - Some Chords (HD)',
                 'alt_title': 'Some Chords',
                 'availability': 'public',
@@ -1513,7 +1523,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
                 'categories': ['Music'],
                 'album': 'Some Chords',
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'deadmau5',
+                'uploader_url': 'https://www.youtube.com/@deadmau5',
+                'uploader_id': '@deadmau5',
             },
             'expected_warnings': [
                 'DASH manifest missing',
@@ -1527,10 +1540,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'duration': 6085,
                 'upload_date': '20150827',
-                'uploader_id': 'olympic',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
-                'uploader': 'Olympics',
                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
                 'like_count': int,
                 'release_timestamp': 1343767800,
@@ -1546,7 +1556,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'live_status': 'was_live',
                 'view_count': int,
                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Olympics',
+                'uploader_url': 'https://www.youtube.com/@Olympics',
+                'uploader_id': '@Olympics',
             },
             'params': {
                 'skip_download': 'requires avconv',
@@ -1561,10 +1574,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'stretched_ratio': 16 / 9.,
                 'duration': 85,
                 'upload_date': '20110310',
-                'uploader_id': 'AllenMeow',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
-                'uploader': '孫ᄋᄅ',
                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
                 'playable_in_embed': True,
                 'channel': '孫ᄋᄅ',
@@ -1579,7 +1589,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'live_status': 'not_live',
                 'availability': 'unlisted',
                 'comment_count': int,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': '孫ᄋᄅ',
+                'uploader_url': 'https://www.youtube.com/@AllenMeow',
+                'uploader_id': '@AllenMeow',
             },
         },
         # url_encoded_fmt_stream_map is empty string
@@ -1591,8 +1604,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
                 'description': '',
                 'upload_date': '20150404',
-                'uploader_id': 'spbelect',
-                'uploader': 'Наблюдатели Петербурга',
             },
             'params': {
                 'skip_download': 'requires avconv',
@@ -1609,9 +1620,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
                 'duration': 220,
                 'upload_date': '20150625',
-                'uploader_id': 'dorappi2000',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
-                'uploader': 'dorappi2000',
                 'formats': 'mincount:31',
             },
             'skip': 'not actual anymore',
@@ -1624,9 +1632,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'CsmdDsKjzN8',
                 'ext': 'mp4',
                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
-                'uploader': 'Airtek',
                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
-                'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
             },
             'params': {
@@ -1637,6 +1643,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         },
         {
             # Multifeed videos (multiple cameras), URL can be of any Camera
+            # TODO: fix multifeed titles
             'url': 'https://www.youtube.com/watch?v=zaPI8MvL8pg',
             'info_dict': {
                 'id': 'zaPI8MvL8pg',
@@ -1648,16 +1655,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'id': 'j5yGuxZ8lLU',
                     'ext': 'mp4',
                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Chris)',
-                    'uploader': 'WiiLikeToPlay',
                     'description': 'md5:563ccbc698b39298481ca3c571169519',
-                    'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
                     'duration': 10120,
                     'channel_follower_count': int,
                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
                     'availability': 'public',
                     'playable_in_embed': True,
                     'upload_date': '20131105',
-                    'uploader_id': 'WiiRikeToPray',
                     'categories': ['Gaming'],
                     'live_status': 'was_live',
                     'tags': 'count:24',
@@ -1670,17 +1674,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'channel': 'WiiLikeToPlay',
                     'view_count': int,
                     'release_date': '20131106',
+                    'uploader': 'WiiLikeToPlay',
+                    'uploader_id': '@WLTP',
+                    'uploader_url': 'https://www.youtube.com/@WLTP',
                 },
             }, {
                 'info_dict': {
                     'id': 'zaPI8MvL8pg',
                     'ext': 'mp4',
                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Tyson)',
-                    'uploader_id': 'WiiRikeToPray',
                     'availability': 'public',
                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
                     'channel': 'WiiLikeToPlay',
-                    'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
                     'channel_follower_count': int,
                     'description': 'md5:563ccbc698b39298481ca3c571169519',
                     'duration': 10108,
@@ -1688,7 +1693,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'like_count': int,
                     'tags': 'count:24',
                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
-                    'uploader': 'WiiLikeToPlay',
                     'release_timestamp': 1383701915,
                     'comment_count': int,
                     'upload_date': '20131105',
@@ -1698,6 +1702,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'live_status': 'was_live',
                     'categories': ['Gaming'],
                     'view_count': int,
+                    'uploader': 'WiiLikeToPlay',
+                    'uploader_id': '@WLTP',
+                    'uploader_url': 'https://www.youtube.com/@WLTP',
                 },
             }, {
                 'info_dict': {
@@ -1711,12 +1718,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'playable_in_embed': True,
                     'upload_date': '20131105',
                     'description': 'md5:563ccbc698b39298481ca3c571169519',
-                    'uploader_id': 'WiiRikeToPray',
-                    'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
                     'channel_follower_count': int,
                     'tags': 'count:24',
                     'release_date': '20131106',
-                    'uploader': 'WiiLikeToPlay',
                     'comment_count': int,
                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
                     'channel': 'WiiLikeToPlay',
@@ -1726,6 +1730,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'age_limit': 0,
                     'duration': 10128,
                     'view_count': int,
+                    'uploader': 'WiiLikeToPlay',
+                    'uploader_id': '@WLTP',
+                    'uploader_url': 'https://www.youtube.com/@WLTP',
                 },
             }],
             'params': {'skip_download': True},
@@ -1762,9 +1769,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
                 'duration': 133,
                 'upload_date': '20151119',
-                'uploader_id': 'IronSoulElf',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
-                'uploader': 'IronSoulElf',
                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
                 'track': 'Dark Walk',
                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
@@ -1801,8 +1805,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
                 'upload_date': '20151107',
-                'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
-                'uploader': 'CH GAMER DROID',
             },
             'params': {
                 'skip_download': True,
@@ -1824,9 +1826,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
                 'duration': 721,
                 'upload_date': '20150128',
-                'uploader_id': 'BerkmanCenter',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
-                'uploader': 'The Berkman Klein Center for Internet & Society',
                 'license': 'Creative Commons Attribution license (reuse allowed)',
                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
@@ -1840,16 +1839,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
                 'live_status': 'not_live',
                 'playable_in_embed': True,
-                'comment_count': int,
                 'channel_follower_count': int,
                 'chapters': list,
+                'uploader': 'The Berkman Klein Center for Internet & Society',
+                'uploader_id': '@BKCHarvard',
+                'uploader_url': 'https://www.youtube.com/@BKCHarvard',
             },
             'params': {
                 'skip_download': True,
             },
         },
         {
-            # Channel-like uploader_url
             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
             'info_dict': {
                 'id': 'eQcmzGIKrzg',
@@ -1858,9 +1858,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
                 'duration': 4060,
                 'upload_date': '20151120',
-                'uploader': 'Bernie Sanders',
-                'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
                 'license': 'Creative Commons Attribution license (reuse allowed)',
                 'playable_in_embed': True,
                 'tags': 'count:12',
@@ -1877,6 +1874,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'comment_count': int,
                 'channel_follower_count': int,
                 'chapters': list,
+                'uploader': 'Bernie Sanders',
+                'uploader_url': 'https://www.youtube.com/@BernieSanders',
+                'uploader_id': '@BernieSanders',
             },
             'params': {
                 'skip_download': True,
@@ -1900,9 +1900,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Piku - Trailer',
                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
                 'upload_date': '20150811',
-                'uploader': 'FlixMatrix',
-                'uploader_id': 'FlixMatrixKaravan',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
                 'license': 'Standard YouTube License',
             },
             'params': {
@@ -1920,9 +1917,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
                 'duration': 2085,
                 'upload_date': '20170118',
-                'uploader': 'Vsauce',
-                'uploader_id': 'Vsauce',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
                 'series': 'Mind Field',
                 'season_number': 1,
                 'episode_number': 1,
@@ -1940,7 +1934,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'like_count': int,
                 'playable_in_embed': True,
                 'live_status': 'not_live',
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Vsauce',
+                'uploader_url': 'https://www.youtube.com/@Vsauce',
+                'uploader_id': '@Vsauce',
             },
             'params': {
                 'skip_download': True,
@@ -1960,9 +1957,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
                 'duration': 965,
                 'upload_date': '20140124',
-                'uploader': 'New Century Foundation',
-                'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
             },
             'params': {
                 'skip_download': True,
@@ -2007,9 +2001,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
                 'duration': 433,
                 'upload_date': '20130923',
-                'uploader': 'Amelia Putri Harwita',
-                'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
                 'formats': 'maxcount:10',
             },
             'params': {
@@ -2020,6 +2011,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         },
         {
             # Youtube Music Auto-generated description
+            # TODO: fix metadata extraction
             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
             'info_dict': {
                 'id': 'MgNrAu2pzNs',
@@ -2027,8 +2019,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Voyeur Girl',
                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
                 'upload_date': '20190312',
-                'uploader': 'Stephen - Topic',
-                'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
                 'artist': 'Stephen',
                 'track': 'Voyeur Girl',
                 'album': 'it\'s too much love to know my dear',
@@ -2036,12 +2026,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'release_year': 2019,
                 'alt_title': 'Voyeur Girl',
                 'view_count': int,
-                'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
                 'playable_in_embed': True,
                 'like_count': int,
                 'categories': ['Music'],
                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
-                'channel': 'Stephen',
+                'channel': 'Stephen',  # TODO: should be "Stephen - Topic"
+                'uploader': 'Stephen',
                 'availability': 'public',
                 'creator': 'Stephen',
                 'duration': 169,
@@ -2069,9 +2059,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
                 'upload_date': '20090125',
-                'uploader': 'Prochorowka',
-                'uploader_id': 'Prochorowka',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
                 'artist': 'Panjabi MC',
                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
@@ -2090,11 +2077,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'IMG 3456',
                 'description': '',
                 'upload_date': '20170613',
-                'uploader_id': 'ElevageOrVert',
-                'uploader': 'ElevageOrVert',
                 'view_count': int,
                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
-                'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
                 'like_count': int,
                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
                 'tags': [],
@@ -2105,8 +2089,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 7,
                 'playable_in_embed': True,
                 'live_status': 'not_live',
-                'channel': 'ElevageOrVert',
-                'channel_follower_count': int
+                'channel': 'l\'Or Vert asbl',
+                'channel_follower_count': int,
+                'uploader': 'l\'Or Vert asbl',
+                'uploader_url': 'https://www.youtube.com/@ElevageOrVert',
+                'uploader_id': '@ElevageOrVert',
             },
             'params': {
                 'skip_download': True,
@@ -2124,11 +2111,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Part 77   Sort a list of simple types in c#',
                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
                 'upload_date': '20130831',
-                'uploader_id': 'kudvenkat',
-                'uploader': 'kudvenkat',
                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
                 'like_count': int,
-                'uploader_url': 'http://www.youtube.com/user/kudvenkat',
                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
                 'live_status': 'not_live',
                 'categories': ['Education'],
@@ -2143,6 +2127,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'comment_count': int,
                 'channel_follower_count': int,
                 'chapters': list,
+                'uploader': 'kudvenkat',
+                'uploader_url': 'https://www.youtube.com/@Csharp-video-tutorialsBlogspot',
+                'uploader_id': '@Csharp-video-tutorialsBlogspot',
             },
             'params': {
                 'skip_download': True,
@@ -2166,9 +2153,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Burn Out',
                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
                 'upload_date': '20141120',
-                'uploader': 'The Cinematic Orchestra - Topic',
-                'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
                 'artist': 'The Cinematic Orchestra',
                 'track': 'Burn Out',
                 'album': 'Every Day',
@@ -2187,7 +2171,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
                 'categories': ['Music'],
                 'playable_in_embed': True,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'The Cinematic Orchestra',
+                'comment_count': int,
             },
             'params': {
                 'skip_download': True,
@@ -2206,13 +2192,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
-                'uploader': 'CBS Mornings',
-                'uploader_id': 'CBSThisMorning',
                 'upload_date': '20140716',
                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
                 'duration': 170,
                 'categories': ['News & Politics'],
-                'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
                 'view_count': int,
                 'channel': 'CBS Mornings',
                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
@@ -2223,7 +2206,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'like_count': int,
                 'live_status': 'not_live',
                 'playable_in_embed': True,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'CBS Mornings',
+                'uploader_url': 'https://www.youtube.com/@CBSMornings',
+                'uploader_id': '@CBSMornings',
             }
         },
         {
@@ -2235,9 +2221,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
                 'upload_date': '20201120',
-                'uploader': 'Walk around Japan',
-                'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
                 'duration': 1456,
                 'categories': ['Travel & Events'],
                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
@@ -2250,7 +2233,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
                 'live_status': 'not_live',
                 'playable_in_embed': True,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Walk around Japan',
+                'uploader_url': 'https://www.youtube.com/@walkaroundjapan7124',
+                'uploader_id': '@walkaroundjapan7124',
             },
             'params': {
                 'skip_download': True,
@@ -2276,13 +2262,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': '3gp',
                 'upload_date': '20210624',
                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
-                'uploader': 'colinfurze',
-                'uploader_id': 'colinfurze',
                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
                 'duration': 596,
                 'categories': ['Entertainment'],
-                'uploader_url': 'http://www.youtube.com/user/colinfurze',
                 'view_count': int,
                 'channel': 'colinfurze',
                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
@@ -2294,6 +2277,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'playable_in_embed': True,
                 'channel_follower_count': int,
                 'chapters': list,
+                'uploader': 'colinfurze',
+                'uploader_url': 'https://www.youtube.com/@colinfurze',
+                'uploader_id': '@colinfurze',
             },
             'params': {
                 'format': '17',  # 3gp format available on android
@@ -2319,10 +2305,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mhtml',
                 'format_id': 'sb0',
                 'title': 'Your Brain is Plastic',
-                'uploader_id': 'scishow',
                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
                 'upload_date': '20140324',
-                'uploader': 'SciShow',
                 'like_count': int,
                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
@@ -2330,7 +2314,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
                 'playable_in_embed': True,
                 'tags': 'count:12',
-                'uploader_url': 'http://www.youtube.com/user/scishow',
                 'availability': 'public',
                 'channel': 'SciShow',
                 'live_status': 'not_live',
@@ -2339,6 +2322,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'age_limit': 0,
                 'channel_follower_count': int,
                 'chapters': list,
+                'uploader': 'SciShow',
+                'uploader_url': 'https://www.youtube.com/@SciShow',
+                'uploader_id': '@SciShow',
             }, 'params': {'format': 'mhtml', 'skip_download': True}
         }, {
             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
@@ -2348,9 +2334,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'The NP that test your phone performance 🙂',
                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
-                'uploader': 'Leon Nguyen',
-                'uploader_id': 'VNSXIII',
-                'uploader_url': 'http://www.youtube.com/user/VNSXIII',
                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
                 'duration': 21,
@@ -2366,7 +2349,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel': 'Leon Nguyen',
                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
                 'comment_count': int,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Leon Nguyen',
+                'uploader_url': 'https://www.youtube.com/@LeonNguyen',
+                'uploader_id': '@LeonNguyen',
             }
         }, {
             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
@@ -2376,9 +2362,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'The NP that test your phone performance 🙂',
                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
-                'uploader': 'Leon Nguyen',
-                'uploader_id': 'VNSXIII',
-                'uploader_url': 'http://www.youtube.com/user/VNSXIII',
                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
                 'duration': 21,
@@ -2394,7 +2377,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel': 'Leon Nguyen',
                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
                 'comment_count': int,
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Leon Nguyen',
+                'uploader_url': 'https://www.youtube.com/@LeonNguyen',
+                'uploader_id': '@LeonNguyen',
             },
             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
         }, {
@@ -2404,10 +2390,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'mzZzzBU6lrM',
                 'ext': 'mp4',
                 'title': 'I Met GeorgeNotFound In Real Life...',
-                'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
-                'uploader': 'Quackity',
-                'uploader_id': 'QuackityHQ',
-                'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
+                'description': 'md5:978296ec9783a031738b684d4ebf302d',
                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
                 'duration': 955,
@@ -2424,7 +2407,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'availability': 'public',
                 'channel': 'Quackity',
                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
-                'channel_follower_count': int
+                'channel_follower_count': int,
+                'uploader': 'Quackity',
+                'uploader_id': '@Quackity',
+                'uploader_url': 'https://www.youtube.com/@Quackity',
             }
         },
         {   # continuous livestream. Microformat upload date should be preferred.
@@ -2442,19 +2428,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
                 'live_status': 'is_live',
                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
-                'uploader': '阿鲍Abao',
-                'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
                 'channel': 'Abao in Tokyo',
                 'channel_follower_count': int,
                 'release_date': '20211127',
                 'tags': 'count:39',
                 'categories': ['People & Blogs'],
                 'like_count': int,
-                'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
                 'view_count': int,
                 'playable_in_embed': True,
                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
                 'concurrent_view_count': int,
+                'uploader': 'Abao in Tokyo',
+                'uploader_url': 'https://www.youtube.com/@abaointokyo',
+                'uploader_id': '@abaointokyo',
             },
             'params': {'skip_download': True}
         }, {
@@ -2471,15 +2457,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Story',
                 'channel': 'IT\'S HISTORY',
                 'description': '',
-                'uploader_id': 'BlastfromthePast',
                 'duration': 12,
-                'uploader': 'IT\'S HISTORY',
                 'playable_in_embed': True,
                 'age_limit': 0,
                 'live_status': 'not_live',
                 'tags': [],
                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
-                'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
             },
             'skip': 'stories get removed after some period of time',
@@ -2492,11 +2475,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20220323',
                 'like_count': int,
                 'availability': 'unlisted',
-                'channel': 'nao20010128nao',
-                'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
+                'channel': 'Lesmiscore',
+                'thumbnail': r're:^https?://.*\.jpg',
                 'age_limit': 0,
-                'uploader': 'nao20010128nao',
-                'uploader_id': 'nao20010128nao',
                 'categories': ['Music'],
                 'view_count': int,
                 'description': '',
@@ -2507,7 +2488,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel_follower_count': int,
                 'duration': 6,
                 'tags': [],
-                'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
+                'uploader_id': '@lesmiscore',
+                'uploader': 'Lesmiscore',
+                'uploader_url': 'https://www.youtube.com/@lesmiscore',
             }
         }, {
             # Prefer primary title+description language metadata by default
@@ -2525,16 +2508,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'like_count': int,
                 'playable_in_embed': True,
                 'availability': 'unlisted',
-                'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
+                'thumbnail': r're:^https?://.*\.jpg',
                 'age_limit': 0,
                 'duration': 5,
-                'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
-                'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
                 'live_status': 'not_live',
                 'upload_date': '20220908',
                 'categories': ['People & Blogs'],
-                'uploader': 'cole-dlp-test-acc',
                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
+                'uploader_url': 'https://www.youtube.com/@coletdjnz',
+                'uploader_id': '@coletdjnz',
+                'uploader': 'cole-dlp-test-acc',
             },
             'params': {'skip_download': True}
         }, {
@@ -2549,18 +2532,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'live_status': 'not_live',
                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
                 'upload_date': '20220728',
-                'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
                 'view_count': int,
                 'categories': ['People & Blogs'],
-                'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
+                'thumbnail': r're:^https?://.*\.jpg',
                 'title': 'dlp test video title translated (fr)',
                 'availability': 'public',
-                'uploader': 'cole-dlp-test-acc',
                 'age_limit': 0,
                 'description': 'dlp test video description translated (fr)',
                 'playable_in_embed': True,
                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
-                'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
+                'uploader_url': 'https://www.youtube.com/@coletdjnz',
+                'uploader_id': '@coletdjnz',
+                'uploader': 'cole-dlp-test-acc',
             },
             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
             'expected_warnings': [r'Preferring "fr" translated fields'],
@@ -2576,7 +2559,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'categories': ['Entertainment'],
                 'description': 'md5:e8031ff6e426cdb6a77670c9b81f6fa6',
-                'uploader_url': 'http://www.youtube.com/user/MrBeast6000',
                 'live_status': 'not_live',
                 'duration': 937,
                 'channel_follower_count': int,
@@ -2586,14 +2568,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'playable_in_embed': True,
                 'view_count': int,
                 'upload_date': '20221112',
-                'uploader': 'MrBeast',
-                'uploader_id': 'MrBeast6000',
                 'channel_url': 'https://www.youtube.com/channel/UCX6OQ3DkcsbYNE6H8uQQuVA',
                 'age_limit': 0,
                 'availability': 'public',
                 'channel_id': 'UCX6OQ3DkcsbYNE6H8uQQuVA',
                 'like_count': int,
                 'tags': [],
+                'uploader': 'MrBeast',
+                'uploader_url': 'https://www.youtube.com/@MrBeast',
+                'uploader_id': '@MrBeast',
             },
             'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
         }, {
@@ -2601,14 +2584,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'https://www.youtube.com/watch?v=Tq92D6wQ1mg',
             'info_dict': {
                 'id': 'Tq92D6wQ1mg',
-                'ext': 'weba',
+                'ext': 'webm',
                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
                 'channel_follower_count': int,
                 'description': 'md5:17eccca93a786d51bc67646756894066',
                 'upload_date': '20191228',
-                'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
                 'playable_in_embed': True,
                 'like_count': int,
@@ -2616,13 +2598,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
                 'age_limit': 18,
                 'channel': 'Projekt Melody',
-                'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
                 'view_count': int,
                 'availability': 'needs_auth',
                 'comment_count': int,
                 'live_status': 'not_live',
-                'uploader': 'Projekt Melody',
                 'duration': 106,
+                'uploader': 'Projekt Melody',
+                'uploader_id': '@ProjektMelody',
+                'uploader_url': 'https://www.youtube.com/@ProjektMelody',
             },
             'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
         },
@@ -2632,13 +2615,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'qVv6vCqciTM',
                 'ext': 'mp4',
                 'age_limit': 0,
-                'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
                 'comment_count': int,
                 'chapters': 'count:13',
                 'upload_date': '20221223',
                 'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
                 'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
-                'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
                 'like_count': int,
                 'release_date': '20221223',
                 'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
@@ -2655,6 +2636,40 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel': 'さなちゃんねる',
                 'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
                 'uploader': 'さなちゃんねる',
+                'uploader_url': 'https://www.youtube.com/@sana_natori',
+                'uploader_id': '@sana_natori',
+            },
+        },
+        {
+            # Fallbacks when webpage and web client is unavailable
+            'url': 'https://www.youtube.com/watch?v=wSSmNUl9Snw',
+            'info_dict': {
+                'id': 'wSSmNUl9Snw',
+                'ext': 'mp4',
+                # 'categories': ['Science & Technology'],
+                'view_count': int,
+                'chapters': 'count:2',
+                'channel': 'Scott Manley',
+                'like_count': int,
+                'age_limit': 0,
+                # 'availability': 'public',
+                'channel_follower_count': int,
+                'live_status': 'not_live',
+                'upload_date': '20170831',
+                'duration': 682,
+                'tags': 'count:8',
+                'uploader_url': 'https://www.youtube.com/@scottmanley',
+                'description': 'md5:f4bed7b200404b72a394c2f97b782c02',
+                'uploader': 'Scott Manley',
+                'uploader_id': '@scottmanley',
+                'title': 'The Computer Hack That Saved Apollo 14',
+                'channel_id': 'UCxzC4EngIsMrPmbm6Nxvb-A',
+                'thumbnail': r're:^https?://.*\.webp',
+                'channel_url': 'https://www.youtube.com/channel/UCxzC4EngIsMrPmbm6Nxvb-A',
+                'playable_in_embed': True,
+            },
+            'params': {
+                'extractor_args': {'youtube': {'player_client': ['android'], 'player_skip': ['webpage']}},
             },
         },
     ]
@@ -2670,8 +2685,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
                 'upload_date': '20080526',
                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
-                'uploader': 'Christopher Sykes',
-                'uploader_id': 'ChristopherJSykes',
                 'age_limit': 0,
                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
@@ -2687,7 +2700,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'view_count': int,
                 'categories': ['Science & Technology'],
                 'channel_follower_count': int,
-                'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
+                'uploader': 'Christopher Sykes',
+                'uploader_url': 'https://www.youtube.com/@ChristopherSykesDocumentaries',
+                'uploader_id': '@ChristopherSykesDocumentaries',
             },
             'params': {
                 'skip_download': True,
@@ -4097,10 +4112,10 @@ def feed_entry(name):
         self._downloader._sort_thumbnails(original_thumbnails)
 
         category = get_first(microformats, 'category') or search_meta('genre')
-        channel_id = str_or_none(
+        channel_id = self.ucid_or_none(str_or_none(
             get_first(video_details, 'channelId')
             or get_first(microformats, 'externalChannelId')
-            or search_meta('channelId'))
+            or search_meta('channelId')))
         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
 
         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
@@ -4127,6 +4142,8 @@ def is_bad_format(fmt):
 
         formats.extend(self._extract_storyboard(player_responses, duration))
 
+        channel_handle = self.handle_from_url(owner_profile_url)
+
         info = {
             'id': video_id,
             'title': video_title,
@@ -4136,11 +4153,8 @@ def is_bad_format(fmt):
             # URL checking if user don't care about getting the best possible thumbnail
             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
             'description': video_description,
-            'uploader': get_first(video_details, 'author'),
-            'uploader_id': self._search_regex(r'/(?:channel/|user/|(?=@))([^/?&#]+)', owner_profile_url, 'uploader id', default=None),
-            'uploader_url': owner_profile_url,
             'channel_id': channel_id,
-            'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
+            'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None),
             'duration': duration,
             'view_count': int_or_none(
                 get_first((video_details, microformats), (..., 'viewCount'))
@@ -4369,6 +4383,13 @@ def process_language(container, base_url, lang_code, sub_name, query):
                 'channel': self._get_text(vor, 'title'),
                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
 
+            if not channel_handle:
+                channel_handle = self.handle_from_url(
+                    traverse_obj(vor, (
+                        ('navigationEndpoint', ('title', 'runs', ..., 'navigationEndpoint')),
+                        (('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl')),
+                        {str}), get_all=False))
+
             rows = try_get(
                 vsir,
                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
@@ -4395,12 +4416,11 @@ def process_language(container, base_url, lang_code, sub_name, query):
                     elif mrr_title == 'Song':
                         info['track'] = mrr_contents_text
 
-        fallbacks = {
-            'channel': 'uploader',
-            'channel_id': 'uploader_id',
-            'channel_url': 'uploader_url',
-        }
-
+        info.update({
+            'uploader': info.get('channel'),
+            'uploader_id': channel_handle,
+            'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
+        })
         # The upload date for scheduled, live and past live streams / premieres in microformats
         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
@@ -4415,10 +4435,6 @@ def process_language(container, base_url, lang_code, sub_name, query):
                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
         info['upload_date'] = upload_date
 
-        for to, frm in fallbacks.items():
-            if not info.get(to):
-                info[to] = info.get(frm)
-
         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
             v = info.get(s_k)
             if v:
@@ -4494,18 +4510,28 @@ def _extract_basic_item_renderer(item):
                 return renderer
 
     def _extract_channel_renderer(self, renderer):
-        channel_id = renderer['channelId']
+        channel_id = self.ucid_or_none(renderer['channelId'])
         title = self._get_text(renderer, 'title')
-        channel_url = f'https://www.youtube.com/channel/{channel_id}'
+        channel_url = format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None)
+        # As of 2023-03-01 YouTube doesn't use the channel handles on these renderers yet.
+        # However we can expect them to change that in the future.
+        channel_handle = self.handle_from_url(
+            traverse_obj(renderer, (
+                'navigationEndpoint', (('commandMetadata', 'webCommandMetadata', 'url'),
+                                       ('browseEndpoint', 'canonicalBaseUrl')),
+                {str}), get_all=False))
         return {
             '_type': 'url',
             'url': channel_url,
             'id': channel_id,
             'ie_key': YoutubeTabIE.ie_key(),
             'channel': title,
+            'uploader': title,
             'channel_id': channel_id,
             'channel_url': channel_url,
             'title': title,
+            'uploader_id': channel_handle,
+            'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
             'channel_follower_count': self._get_count(renderer, 'subscriberCountText'),
             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
             'playlist_count': self._get_count(renderer, 'videoCountText'),
@@ -4845,13 +4871,14 @@ def _extract_metadata_from_tabs(self, item_id, data):
 
         metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
         if metadata_renderer:
+            channel_id = traverse_obj(metadata_renderer, ('externalId', {self.ucid_or_none}),
+                                                         ('channelUrl', {self.ucid_from_url}))
             info.update({
-                'uploader': metadata_renderer.get('title'),
-                'uploader_id': metadata_renderer.get('externalId'),
-                'uploader_url': metadata_renderer.get('channelUrl'),
+                'channel': metadata_renderer.get('title'),
+                'channel_id': channel_id,
             })
-            if info['uploader_id']:
-                info['id'] = info['uploader_id']
+            if info['channel_id']:
+                info['id'] = info['channel_id']
         else:
             metadata_renderer = traverse_obj(data, ('metadata', 'playlistMetadataRenderer'), expected_type=dict)
 
@@ -4904,6 +4931,15 @@ def _get_uncropped(url):
             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
         })
 
+        channel_handle = (
+            traverse_obj(metadata_renderer, (('vanityChannelUrl', ('ownerUrls', ...)), {self.handle_from_url}), get_all=False)
+            or traverse_obj(data, ('header', ..., 'channelHandleText', {self.handle_or_none}), get_all=False))
+
+        if channel_handle:
+            info.update({
+                'uploader_id': channel_handle,
+                'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
+            })
         # Playlist stats is a text runs array containing [video count, view count, last updated].
         # last updated or (view count and last updated) may be missing.
         playlist_stats = get_first(
@@ -4926,7 +4962,7 @@ def _get_uncropped(url):
         if info['playlist_count'] is None:  # 0 is allowed
             info['playlist_count'] = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
 
-        if not info.get('uploader_id'):
+        if not info.get('channel_id'):
             owner = traverse_obj(playlist_header_renderer, 'ownerText')
             if not owner:  # Deprecated
                 owner = traverse_obj(
@@ -4935,16 +4971,17 @@ def _get_uncropped(url):
             owner_text = self._get_text(owner)
             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
             info.update({
-                'uploader': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
-                'uploader_id': browse_ep.get('browseId'),
-                'uploader_url': urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))
+                'channel': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
+                'channel_id': self.ucid_or_none(browse_ep.get('browseId')),
+                'uploader_id': self.handle_from_url(urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl')))
             })
 
         info.update({
-            'channel': info['uploader'],
-            'channel_id': info['uploader_id'],
-            'channel_url': info['uploader_url']
+            'uploader': info['channel'],
+            'channel_url': format_field(info.get('channel_id'), None, 'https://www.youtube.com/channel/%s', default=None),
+            'uploader_url': format_field(info.get('uploader_id'), None, 'https://www.youtube.com/%s', default=None),
         })
+
         return info
 
     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
@@ -5221,12 +5258,12 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'title': 'Igor Kleiner - Playlists',
             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
             'uploader': 'Igor Kleiner',
-            'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
+            'uploader_id': '@IgorDataScience',
+            'uploader_url': 'https://www.youtube.com/@IgorDataScience',
             'channel': 'Igor Kleiner',
             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
-            'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
             'channel_follower_count': int
         },
     }, {
@@ -5237,9 +5274,9 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
             'title': 'Igor Kleiner - Playlists',
             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
-            'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
             'uploader': 'Igor Kleiner',
-            'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
+            'uploader_id': '@IgorDataScience',
+            'uploader_url': 'https://www.youtube.com/@IgorDataScience',
             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
             'channel': 'Igor Kleiner',
@@ -5254,12 +5291,12 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'id': 'UCYO_jab_esuFRV4b17AJtAw',
             'title': '3Blue1Brown - Playlists',
             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
-            'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
-            'uploader': '3Blue1Brown',
             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
-            'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
             'channel': '3Blue1Brown',
             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
+            'uploader_id': '@3blue1brown',
+            'uploader_url': 'https://www.youtube.com/@3blue1brown',
+            'uploader': '3Blue1Brown',
             'tags': ['Mathematics'],
             'channel_follower_count': int
         },
@@ -5272,10 +5309,10 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'title': 'ThirstForScience - Playlists',
             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
             'uploader': 'ThirstForScience',
-            'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
-            'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
-            'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
+            'uploader_url': 'https://www.youtube.com/@ThirstForScience',
+            'uploader_id': '@ThirstForScience',
             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
+            'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
             'tags': 'count:13',
             'channel': 'ThirstForScience',
             'channel_follower_count': int
@@ -5287,8 +5324,6 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'note': 'basic, single video playlist',
         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
         'info_dict': {
-            'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
-            'uploader': 'Sergey M.',
             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
             'title': 'youtube-dl public playlist',
             'description': '',
@@ -5297,17 +5332,17 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'modified_date': '20201130',
             'channel': 'Sergey M.',
             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
-            'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
             'availability': 'public',
+            'uploader': 'Sergey M.',
+            'uploader_url': 'https://www.youtube.com/@sergeym.6173',
+            'uploader_id': '@sergeym.6173',
         },
         'playlist_count': 1,
     }, {
         'note': 'empty playlist',
         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
         'info_dict': {
-            'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
-            'uploader': 'Sergey M.',
             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
             'title': 'youtube-dl empty playlist',
             'tags': [],
@@ -5316,8 +5351,10 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'modified_date': '20160902',
             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
-            'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
             'availability': 'public',
+            'uploader_url': 'https://www.youtube.com/@sergeym.6173',
+            'uploader_id': '@sergeym.6173',
+            'uploader': 'Sergey M.',
         },
         'playlist_count': 0,
     }, {
@@ -5328,10 +5365,10 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'title': 'lex will - Home',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
             'uploader': 'lex will',
-            'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+            'uploader_id': '@lexwill718',
             'channel': 'lex will',
             'tags': ['bible', 'history', 'prophesy'],
-            'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+            'uploader_url': 'https://www.youtube.com/@lexwill718',
             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'channel_follower_count': int
@@ -5345,11 +5382,11 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'title': 'lex will - Videos',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
             'uploader': 'lex will',
-            'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+            'uploader_id': '@lexwill718',
             'tags': ['bible', 'history', 'prophesy'],
             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
-            'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+            'uploader_url': 'https://www.youtube.com/@lexwill718',
             'channel': 'lex will',
             'channel_follower_count': int
         },
@@ -5362,9 +5399,9 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'title': 'lex will - Videos',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
             'uploader': 'lex will',
-            'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+            'uploader_id': '@lexwill718',
             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
-            'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+            'uploader_url': 'https://www.youtube.com/@lexwill718',
             'channel': 'lex will',
             'tags': ['bible', 'history', 'prophesy'],
             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
@@ -5379,8 +5416,8 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'title': 'lex will - Playlists',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
             'uploader': 'lex will',
-            'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
-            'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+            'uploader_id': '@lexwill718',
+            'uploader_url': 'https://www.youtube.com/@lexwill718',
             'channel': 'lex will',
             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
@@ -5395,14 +5432,14 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'lex will - Community',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
-            'uploader': 'lex will',
-            'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
-            'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
             'channel': 'lex will',
             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'tags': ['bible', 'history', 'prophesy'],
-            'channel_follower_count': int
+            'channel_follower_count': int,
+            'uploader_url': 'https://www.youtube.com/@lexwill718',
+            'uploader_id': '@lexwill718',
+            'uploader': 'lex will',
         },
         'playlist_mincount': 18,
     }, {
@@ -5412,14 +5449,14 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'lex will - Channels',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
-            'uploader': 'lex will',
-            'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
-            'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
             'channel': 'lex will',
             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'tags': ['bible', 'history', 'prophesy'],
-            'channel_follower_count': int
+            'channel_follower_count': int,
+            'uploader_url': 'https://www.youtube.com/@lexwill718',
+            'uploader_id': '@lexwill718',
+            'uploader': 'lex will',
         },
         'playlist_mincount': 12,
     }, {
@@ -5430,14 +5467,14 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'id': 'UCYO_jab_esuFRV4b17AJtAw',
             'title': '3Blue1Brown - Search - linear algebra',
             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
-            'uploader': '3Blue1Brown',
-            'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
-            'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
             'tags': ['Mathematics'],
             'channel': '3Blue1Brown',
             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
-            'channel_follower_count': int
+            'channel_follower_count': int,
+            'uploader_url': 'https://www.youtube.com/@3blue1brown',
+            'uploader_id': '@3blue1brown',
+            'uploader': '3Blue1Brown',
         },
     }, {
         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
@@ -5454,17 +5491,17 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'title': '29C3: Not my department',
             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
-            'uploader': 'Christiaan008',
-            'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
             'tags': [],
-            'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
             'view_count': int,
             'modified_date': '20150605',
             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
-            'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
+            'channel_url': 'https://www.youtube.com/channel/UCEPzS1rYsrkqzSLNp76nrcg',
             'channel': 'Christiaan008',
             'availability': 'public',
+            'uploader_id': '@ChRiStIaAn008',
+            'uploader': 'Christiaan008',
+            'uploader_url': 'https://www.youtube.com/@ChRiStIaAn008',
         },
         'playlist_count': 96,
     }, {
@@ -5473,17 +5510,17 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'title': 'Uploads from Cauchemar',
             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
-            'uploader': 'Cauchemar',
-            'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
-            'channel_url': 'https://www.youtube.com/c/Cauchemar89',
+            'channel_url': 'https://www.youtube.com/channel/UCBABnxM4Ar9ten8Mdjj1j0Q',
             'tags': [],
             'modified_date': r're:\d{8}',
             'channel': 'Cauchemar',
-            'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
             'view_count': int,
             'description': '',
             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
             'availability': 'public',
+            'uploader_id': '@Cauchemar89',
+            'uploader': 'Cauchemar',
+            'uploader_url': 'https://www.youtube.com/@Cauchemar89',
         },
         'playlist_mincount': 1123,
         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
@@ -5497,17 +5534,17 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'title': 'Uploads from Interstellar Movie',
             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
-            'uploader': 'Interstellar Movie',
-            'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
-            'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
             'tags': [],
             'view_count': int,
             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
-            'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
+            'channel_url': 'https://www.youtube.com/channel/UCXw-G3eDE9trcvY2sBMM_aA',
             'channel': 'Interstellar Movie',
             'description': '',
             'modified_date': r're:\d{8}',
             'availability': 'public',
+            'uploader_id': '@InterstellarMovie',
+            'uploader': 'Interstellar Movie',
+            'uploader_url': 'https://www.youtube.com/@InterstellarMovie',
         },
         'playlist_mincount': 21,
     }, {
@@ -5516,17 +5553,17 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
-            'uploader': 'Phim Siêu Nhân Nhật Bản',
-            'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
             'view_count': int,
             'channel': 'Phim Siêu Nhân Nhật Bản',
             'tags': [],
-            'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
             'description': '',
             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
             'modified_date': r're:\d{8}',
             'availability': 'public',
+            'uploader_url': 'https://www.youtube.com/@phimsieunhannhatban',
+            'uploader_id': '@phimsieunhannhatban',
+            'uploader': 'Phim Siêu Nhân Nhật Bản',
         },
         'playlist_mincount': 200,
         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
@@ -5536,17 +5573,17 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'title': 'Uploads from BlankTV',
             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
-            'uploader': 'BlankTV',
-            'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
             'channel': 'BlankTV',
-            'channel_url': 'https://www.youtube.com/c/blanktv',
+            'channel_url': 'https://www.youtube.com/channel/UC8l9frL61Yl5KFOl87nIm2w',
             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
             'view_count': int,
             'tags': [],
-            'uploader_url': 'https://www.youtube.com/c/blanktv',
             'modified_date': r're:\d{8}',
             'description': '',
             'availability': 'public',
+            'uploader_id': '@blanktv',
+            'uploader': 'BlankTV',
+            'uploader_url': 'https://www.youtube.com/@blanktv',
         },
         'playlist_mincount': 1000,
         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
@@ -5556,17 +5593,17 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'title': 'Data Analysis with Dr Mike Pound',
             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
-            'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
-            'uploader': 'Computerphile',
             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
-            'uploader_url': 'https://www.youtube.com/user/Computerphile',
             'tags': [],
             'view_count': int,
             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
-            'channel_url': 'https://www.youtube.com/user/Computerphile',
+            'channel_url': 'https://www.youtube.com/channel/UC9-y-6csu5WGm29I7JiwpnA',
             'channel': 'Computerphile',
             'availability': 'public',
             'modified_date': '20190712',
+            'uploader_id': '@Computerphile',
+            'uploader': 'Computerphile',
+            'uploader_url': 'https://www.youtube.com/@Computerphile',
         },
         'playlist_mincount': 11,
     }, {
@@ -5579,9 +5616,6 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'id': 'FqZTN594JQw',
             'ext': 'webm',
             'title': "Smiley's People 01 detective, Adventure Series, Action",
-            'uploader': 'STREEM',
-            'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
-            'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
             'upload_date': '20150526',
             'license': 'Standard YouTube License',
             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
@@ -5604,12 +5638,9 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
     }, {
         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
         'info_dict': {
-            'id': 'Wq15eF5vCbI',  # This will keep changing
+            'id': 'AlTsmyW4auo',  # This will keep changing
             'ext': 'mp4',
             'title': str,
-            'uploader': 'Sky News',
-            'uploader_id': 'skynews',
-            'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
             'upload_date': r're:\d{8}',
             'description': str,
             'categories': ['News & Politics'],
@@ -5628,6 +5659,9 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
             'channel_follower_count': int,
             'concurrent_view_count': int,
+            'uploader_url': 'https://www.youtube.com/@SkyNews',
+            'uploader_id': '@SkyNews',
+            'uploader': 'Sky News',
         },
         'params': {
             'skip_download': True,
@@ -5639,9 +5673,6 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'id': 'a48o2S1cPoo',
             'ext': 'mp4',
             'title': 'The Young Turks - Live Main Show',
-            'uploader': 'The Young Turks',
-            'uploader_id': 'TheYoungTurks',
-            'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
             'upload_date': '20150715',
             'license': 'Standard YouTube License',
             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
@@ -5722,41 +5753,40 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
         'info_dict': {
             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
-            'uploader': 'NoCopyrightSounds',
             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
-            'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
             'title': 'NCS : All Releases 💿',
-            'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
-            'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
+            'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
             'modified_date': r're:\d{8}',
             'view_count': int,
             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
             'tags': [],
             'channel': 'NoCopyrightSounds',
             'availability': 'public',
+            'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
+            'uploader': 'NoCopyrightSounds',
+            'uploader_id': '@NoCopyrightSounds',
         },
         'playlist_mincount': 166,
-        'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
+        'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden', 'YouTube Music is not directly supported'],
     }, {
+        # TODO: fix 'unviewable' issue with this playlist when reloading with unavailable videos
         'note': 'Topic, should redirect to playlist?list=UU...',
         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
         'info_dict': {
             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
-            'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
             'title': 'Uploads from Royalty Free Music - Topic',
-            'uploader': 'Royalty Free Music - Topic',
             'tags': [],
             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
             'channel': 'Royalty Free Music - Topic',
             'view_count': int,
             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
-            'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
             'modified_date': r're:\d{8}',
-            'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
             'description': '',
             'availability': 'public',
+            'uploader': 'Royalty Free Music - Topic',
         },
         'playlist_mincount': 101,
+        'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
     }, {
         # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
         # Treat as a general feed
@@ -5780,12 +5810,11 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'modified_date': r're:\d{8}',
         },
         'playlist_count': 50,
+        'expected_warnings': ['YouTube Music is not directly supported'],
     }, {
         'note': 'unlisted single video playlist',
         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
         'info_dict': {
-            'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
-            'uploader': 'colethedj',
             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
             'title': 'yt-dlp unlisted playlist test',
             'availability': 'unlisted',
@@ -5794,9 +5823,11 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'channel': 'colethedj',
             'view_count': int,
             'description': '',
-            'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
+            'uploader_url': 'https://www.youtube.com/@colethedj1894',
+            'uploader_id': '@colethedj1894',
+            'uploader': 'colethedj',
         },
         'playlist_count': 1,
     }, {
@@ -5819,13 +5850,10 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
             'title': 'Cody\'sLab - Videos',
             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
-            'uploader': 'Cody\'sLab',
-            'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
             'channel': 'Cody\'sLab',
             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
             'tags': [],
             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
-            'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
             'channel_follower_count': int
         },
         'playlist_mincount': 650,
@@ -5839,9 +5867,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
         'info_dict': {
             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
-            'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
             'title': 'Uploads from Royalty Free Music - Topic',
-            'uploader': 'Royalty Free Music - Topic',
             'modified_date': r're:\d{8}',
             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
             'description': '',
@@ -5849,14 +5875,15 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'tags': [],
             'channel': 'Royalty Free Music - Topic',
             'view_count': int,
-            'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
             'availability': 'public',
+            'uploader': 'Royalty Free Music - Topic',
         },
         'playlist_mincount': 101,
         'params': {
             'skip_download': True,
             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
         },
+        'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
     }, {
         'note': 'non-standard redirect to regional channel',
         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
@@ -5869,15 +5896,15 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'modified_date': '20220407',
             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
             'tags': [],
-            'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
-            'uploader': 'pukkandan',
             'availability': 'unlisted',
             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
             'channel': 'pukkandan',
             'description': 'Test for collaborative playlist',
             'title': 'yt-dlp test - collaborative playlist',
             'view_count': int,
-            'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
+            'uploader_url': 'https://www.youtube.com/@pukkandan',
+            'uploader_id': '@pukkandan',
+            'uploader': 'pukkandan',
         },
         'playlist_mincount': 2
     }, {
@@ -5886,14 +5913,14 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'id': 'UCiu-3thuViMebBjw_5nWYrA',
             'tags': [],
-            'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
             'description': 'test description',
             'title': 'cole-dlp-test-acc - 再生リスト',
-            'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
-            'uploader': 'cole-dlp-test-acc',
             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
             'channel': 'cole-dlp-test-acc',
+            'uploader_url': 'https://www.youtube.com/@coletdjnz',
+            'uploader_id': '@coletdjnz',
+            'uploader': 'cole-dlp-test-acc',
         },
         'playlist_mincount': 1,
         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
@@ -5907,14 +5934,14 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'tags': [],
             'view_count': int,
             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
-            'uploader': 'cole-dlp-test-acc',
-            'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
             'channel': 'cole-dlp-test-acc',
             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
             'description': 'test',
-            'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
             'title': 'dlp test playlist',
             'availability': 'public',
+            'uploader_url': 'https://www.youtube.com/@coletdjnz',
+            'uploader_id': '@coletdjnz',
+            'uploader': 'cole-dlp-test-acc',
         },
         'playlist_mincount': 1,
         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
@@ -5966,29 +5993,30 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'channel_follower_count': int,
             'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
             'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
-            'uploader': 'Polka Ch. 尾丸ポルカ',
-            'description': 'md5:3b8df1ac5af337aa206e37ee3d181ec9',
+            'description': 'md5:e56b74b5bb7e9c701522162e9abfb822',
             'channel': 'Polka Ch. 尾丸ポルカ',
             'tags': 'count:35',
-            'uploader_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
-            'uploader_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
+            'uploader_url': 'https://www.youtube.com/@OmaruPolka',
+            'uploader': 'Polka Ch. 尾丸ポルカ',
+            'uploader_id': '@OmaruPolka',
         },
         'playlist_count': 3,
     }, {
         # Shorts tab with channel with handle
+        # TODO: fix channel description
         'url': 'https://www.youtube.com/@NotJustBikes/shorts',
         'info_dict': {
             'id': 'UC0intLFzLaudFG-xAvUEO-A',
             'title': 'Not Just Bikes - Shorts',
             'tags': 'count:12',
-            'uploader': 'Not Just Bikes',
             'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
-            'description': 'md5:7513148b1f02b924783157d84c4ea555',
+            'description': 'md5:26bc55af26855a608a5cf89dfa595c8d',
             'channel_follower_count': int,
-            'uploader_id': 'UC0intLFzLaudFG-xAvUEO-A',
             'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
-            'uploader_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
             'channel': 'Not Just Bikes',
+            'uploader_url': 'https://www.youtube.com/@NotJustBikes',
+            'uploader': 'Not Just Bikes',
+            'uploader_id': '@NotJustBikes',
         },
         'playlist_mincount': 10,
     }, {
@@ -6000,12 +6028,12 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'tags': 'count:7',
             'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
             'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
-            'uploader_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
             'channel': '中村悠一',
-            'uploader_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
             'channel_follower_count': int,
-            'uploader': '中村悠一',
             'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
+            'uploader_url': 'https://www.youtube.com/@Yuichi-Nakamura',
+            'uploader_id': '@Yuichi-Nakamura',
+            'uploader': '中村悠一',
         },
         'playlist_mincount': 60,
     }, {
@@ -6024,15 +6052,15 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
             'title': 'Shorts Break - Shorts',
-            'tags': 'count:32',
+            'tags': 'count:48',
             'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
             'channel': 'Shorts Break',
-            'description': 'md5:a6c234cf3d50d878ef8721e34457cd11',
-            'uploader': 'Shorts Break',
+            'description': 'md5:6de33c5e7ba686e5f3efd4e19c7ef499',
             'channel_follower_count': int,
-            'uploader_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
-            'uploader_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
             'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
+            'uploader_url': 'https://www.youtube.com/@ShortsBreak_Official',
+            'uploader': 'Shorts Break',
+            'uploader_id': '@ShortsBreak_Official',
         },
         'playlist_mincount': 30,
     }, {
@@ -6055,30 +6083,28 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'playlist_mincount': 30,
     }, {
         # Shorts url result in shorts tab
+        # TODO: Fix channel id extraction
         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
         'info_dict': {
             'id': 'UCiu-3thuViMebBjw_5nWYrA',
             'title': 'cole-dlp-test-acc - Shorts',
-            'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
             'channel': 'cole-dlp-test-acc',
             'description': 'test description',
             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
             'tags': [],
+            'uploader_url': 'https://www.youtube.com/@coletdjnz',
+            'uploader_id': '@coletdjnz',
             'uploader': 'cole-dlp-test-acc',
-            'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
-
         },
         'playlist': [{
             'info_dict': {
+                # Channel data is not currently available for short renderers (as of 2023-03-01)
                 '_type': 'url',
                 'ie_key': 'Youtube',
                 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
                 'id': 'sSM9J5YH_60',
-                'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
                 'title': 'SHORT short',
-                'channel': 'cole-dlp-test-acc',
-                'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
                 'view_count': int,
                 'thumbnails': list,
             }
@@ -6114,15 +6140,14 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'id': 'UCiu-3thuViMebBjw_5nWYrA',
             'title': 'cole-dlp-test-acc - Channels',
-            'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
             'channel': 'cole-dlp-test-acc',
             'description': 'test description',
             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
             'tags': [],
+            'uploader_url': 'https://www.youtube.com/@coletdjnz',
+            'uploader_id': '@coletdjnz',
             'uploader': 'cole-dlp-test-acc',
-            'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
-
         },
         'playlist': [{
             'info_dict': {
@@ -6136,7 +6161,10 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
                 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
                 'thumbnails': list,
                 'channel_follower_count': int,
-                'playlist_count': int
+                'playlist_count': int,
+                'uploader': 'PewDiePie',
+                'uploader_url': 'https://www.youtube.com/@PewDiePie',
+                'uploader_id': '@PewDiePie',
             }
         }],
         'params': {'extract_flat': True},
@@ -6146,15 +6174,15 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'id': 'UCYO_jab_esuFRV4b17AJtAw',
             'tags': ['Mathematics'],
             'title': '3Blue1Brown - About',
-            'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
             'channel_follower_count': int,
             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
-            'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
             'channel': '3Blue1Brown',
-            'uploader': '3Blue1Brown',
             'view_count': int,
             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
+            'uploader_url': 'https://www.youtube.com/@3blue1brown',
+            'uploader_id': '@3blue1brown',
+            'uploader': '3Blue1Brown',
         },
         'playlist_count': 0,
     }]
@@ -6373,15 +6401,15 @@ class YoutubePlaylistIE(InfoExtractor):
             'title': '[OLD]Team Fortress 2 (Class-based LP)',
             'id': 'PLBB231211A4F62143',
             'uploader': 'Wickman',
-            'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
+            'uploader_id': '@WickmanVT',
             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
             'view_count': int,
-            'uploader_url': 'https://www.youtube.com/c/WickmanVT',
+            'uploader_url': 'https://www.youtube.com/@WickmanVT',
             'modified_date': r're:\d{8}',
             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
             'channel': 'Wickman',
             'tags': [],
-            'channel_url': 'https://www.youtube.com/c/WickmanVT',
+            'channel_url': 'https://www.youtube.com/channel/UCKSpbfbl5kRQpTdL7kMc-1Q',
             'availability': 'public',
         },
         'playlist_mincount': 29,
@@ -6401,7 +6429,7 @@ class YoutubePlaylistIE(InfoExtractor):
             'title': 'JODA15',
             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
             'uploader': 'milan',
-            'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
+            'uploader_id': '@milan5503',
             'description': '',
             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
             'tags': [],
@@ -6409,7 +6437,7 @@ class YoutubePlaylistIE(InfoExtractor):
             'view_count': int,
             'channel': 'milan',
             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
-            'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
+            'uploader_url': 'https://www.youtube.com/@milan5503',
             'availability': 'public',
         },
         'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'],
@@ -6420,13 +6448,13 @@ class YoutubePlaylistIE(InfoExtractor):
             'title': '2018 Chinese New Singles (11/6 updated)',
             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
             'uploader': 'LBK',
-            'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
+            'uploader_id': '@music_king',
             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
             'channel': 'LBK',
             'view_count': int,
-            'channel_url': 'https://www.youtube.com/c/愛低音的國王',
+            'channel_url': 'https://www.youtube.com/channel/UC21nz3_MesPLqtDqwdvnoxA',
             'tags': [],
-            'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
+            'uploader_url': 'https://www.youtube.com/@music_king',
             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
             'modified_date': r're:\d{8}',
             'availability': 'public',
@@ -6472,8 +6500,8 @@ class YoutubeYtBeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Small Scale Baler and Braiding Rugs',
             'uploader': 'Backus-Page House Museum',
-            'uploader_id': 'backuspagemuseum',
-            'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
+            'uploader_id': '@backuspagemuseum',
+            'uploader_url': r're:https?://(?:www\.)?youtube\.com/@backuspagemuseum',
             'upload_date': '20161008',
             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
             'categories': ['Nonprofits & Activism'],
@@ -6481,7 +6509,7 @@ class YoutubeYtBeIE(InfoExtractor):
             'like_count': int,
             'age_limit': 0,
             'playable_in_embed': True,
-            'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
+            'thumbnail': r're:^https?://.*\.webp',
             'channel': 'Backus-Page House Museum',
             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
             'live_status': 'not_live',
@@ -6597,7 +6625,7 @@ def _extract_notification_renderer(self, notification):
         if not video_id:
             browse_ep = traverse_obj(
                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
-            channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
+            channel_id = self.ucid_or_none(traverse_obj(browse_ep, 'browseId', expected_type=str))
             post_id = self._search_regex(
                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
                 'post id', default=None)
@@ -6627,6 +6655,7 @@ def _extract_notification_renderer(self, notification):
             'title': title,
             'channel_id': channel_id,
             'channel': channel,
+            'uploader': channel,
             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
             'timestamp': timestamp,
         }
@@ -6731,7 +6760,10 @@ class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
                 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
                 'playlist_count': int,  # XXX: should have a way of saying > 1
                 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
-                'thumbnails': list
+                'thumbnails': list,
+                'uploader_id': '@kurzgesagt',
+                'uploader_url': 'https://www.youtube.com/@kurzgesagt',
+                'uploader': 'Kurzgesagt – In a Nutshell',
             }
         }],
         'params': {'extract_flat': True, 'playlist_items': '1'},
@@ -6989,11 +7021,12 @@ class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
             'title': 'Mobile Games on Console - Scott The Woz',
             'upload_date': '20210920',
             'uploader': 'Scott The Woz',
-            'uploader_id': 'scottthewoz',
-            'uploader_url': 'http://www.youtube.com/user/scottthewoz',
+            'uploader_id': '@ScottTheWoz',
+            'uploader_url': 'https://www.youtube.com/@ScottTheWoz',
             'view_count': int,
             'live_status': 'not_live',
-            'channel_follower_count': int
+            'channel_follower_count': int,
+            'chapters': 'count:20',
         }
     }]
 
@@ -7031,13 +7064,13 @@ class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
             'id': 'qVv6vCqciTM',
             'ext': 'mp4',
             'age_limit': 0,
-            'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
+            'uploader_id': '@sana_natori',
             'comment_count': int,
             'chapters': 'count:13',
             'upload_date': '20221223',
             'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
             'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
-            'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
+            'uploader_url': 'https://www.youtube.com/@sana_natori',
             'like_count': int,
             'release_date': '20221223',
             'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],

From 84ffeb7d5e72e3829319ba7720a8480fc4c7503b Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 16 Apr 2023 03:16:23 +0530
Subject: [PATCH 214/871] [extractor] Do not warn for invalid chapter data in
 description

Fixes https://github.com/yt-dlp/yt-dlp/issues/6811#issuecomment-1509876209
---
 yt_dlp/extractor/common.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 838899052..78288f809 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -3658,18 +3658,22 @@ def _extract_chapters_helper(self, chapter_list, start_function, title_function,
             'start_time': start_function(chapter),
             'title': title_function(chapter),
         } for chapter in chapter_list or []]
-        if not strict:
+        if strict:
+            warn = self.report_warning
+        else:
+            warn = self.write_debug
             chapter_list.sort(key=lambda c: c['start_time'] or 0)
 
         chapters = [{'start_time': 0}]
         for idx, chapter in enumerate(chapter_list):
             if chapter['start_time'] is None:
-                self.report_warning(f'Incomplete chapter {idx}')
+                warn(f'Incomplete chapter {idx}')
             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
                 chapters.append(chapter)
             elif chapter not in chapters:
-                self.report_warning(
-                    f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
+                issue = (f'{chapter["start_time"]} > {duration}' if chapter['start_time'] > duration
+                         else f'{chapter["start_time"]} < {chapters[-1]["start_time"]}')
+                warn(f'Invalid start time ({issue}) for chapter "{chapter["title"]}"')
         return chapters[1:]
 
     def _extract_chapters_from_description(self, description, duration):

From 9874e82b5a61582169300bea561b3e8899ad1ef7 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 16 Apr 2023 08:54:48 +0530
Subject: [PATCH 215/871] Do not translate newlines in `--print-to-file`

Fixes https://github.com/yt-dlp/yt-dlp/issues/6808#issuecomment-1509361107
---
 yt_dlp/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 7b6fef204..31f7645dc 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2890,8 +2890,8 @@ def format_tmpl(tmpl):
             tmpl = format_tmpl(tmpl)
             self.to_screen(f'[info] Writing {tmpl!r} to: {filename}')
             if self._ensure_dir_exists(filename):
-                with open(filename, 'a', encoding='utf-8') as f:
-                    f.write(self.evaluate_outtmpl(tmpl, info_copy) + '\n')
+                with open(filename, 'a', encoding='utf-8', newline='') as f:
+                    f.write(self.evaluate_outtmpl(tmpl, info_copy) + os.linesep)
 
     def __forced_printings(self, info_dict, filename, incomplete):
         def print_mandatory(field, actual_field=None):

From ea0570820336a0fe9c3b530d1b0d1e59313274f4 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 16 Apr 2023 12:01:19 -0500
Subject: [PATCH 216/871] [extractor/adobepass] Handle `Charter_Direct` MSO as
 `Spectrum` (#6824)

Authored by: bashonly
---
 yt_dlp/extractor/adobepass.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py
index e5944f714..68a970f68 100644
--- a/yt_dlp/extractor/adobepass.py
+++ b/yt_dlp/extractor/adobepass.py
@@ -1573,7 +1573,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                         }), headers={
                             'Content-Type': 'application/x-www-form-urlencoded'
                         })
-                elif mso_id == 'Spectrum':
+                elif mso_id in ('Spectrum', 'Charter_Direct'):
                     # Spectrum's login for is dynamically loaded via JS so we need to hardcode the flow
                     # as a one-off implementation.
                     provider_redirect_page, urlh = provider_redirect_page_res

From 7a6f6f24592a8065376f11a58e44878807732cf6 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 16 Apr 2023 12:07:55 -0500
Subject: [PATCH 217/871] [extractor/reddit] Support cookies and short URLs
 (#6825)

Closes #6665, Closes #6753
Authored by: bashonly
---
 yt_dlp/extractor/reddit.py | 72 ++++++++++++++++++++++++++++++--------
 1 file changed, 58 insertions(+), 14 deletions(-)

diff --git a/yt_dlp/extractor/reddit.py b/yt_dlp/extractor/reddit.py
index 9dba3eca8..3e458456c 100644
--- a/yt_dlp/extractor/reddit.py
+++ b/yt_dlp/extractor/reddit.py
@@ -1,4 +1,3 @@
-import random
 import urllib.parse
 
 from .common import InfoExtractor
@@ -14,7 +13,7 @@
 
 
 class RedditIE(InfoExtractor):
-    _VALID_URL = r'https?://(?P<subdomain>[^/]+\.)?reddit(?:media)?\.com/(?P<slug>(?:r|user)/[^/]+/comments/(?P<id>[^/?#&]+))'
+    _VALID_URL = r'https?://(?P<host>(?:\w+\.)?reddit(?:media)?\.com)/(?P<slug>(?:(?:r|user)/[^/]+/)?comments/(?P<id>[^/?#&]+))'
     _TESTS = [{
         'url': 'https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/',
         'info_dict': {
@@ -109,6 +108,46 @@ class RedditIE(InfoExtractor):
             'age_limit': 0,
             'channel_id': 'dumbfuckers_club',
         },
+    }, {
+        # post link without subreddit
+        'url': 'https://www.reddit.com/comments/124pp33',
+        'md5': '15eec9d828adcef4468b741a7e45a395',
+        'info_dict': {
+            'id': 'antsenjc2jqa1',
+            'ext': 'mp4',
+            'display_id': '124pp33',
+            'title': 'Harmless prank of some old friends',
+            'uploader': 'Dudezila',
+            'channel_id': 'ContagiousLaughter',
+            'duration': 17,
+            'upload_date': '20230328',
+            'timestamp': 1680012043,
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'age_limit': 0,
+            'comment_count': int,
+            'dislike_count': int,
+            'like_count': int,
+        },
+    }, {
+        # quarantined subreddit post
+        'url': 'https://old.reddit.com/r/GenZedong/comments/12fujy3/based_hasan/',
+        'md5': '3156ea69e3c1f1b6259683c5abd36e71',
+        'info_dict': {
+            'id': '8bwtclfggpsa1',
+            'ext': 'mp4',
+            'display_id': '12fujy3',
+            'title': 'Based Hasan?',
+            'uploader': 'KingNigelXLII',
+            'channel_id': 'GenZedong',
+            'duration': 16,
+            'upload_date': '20230408',
+            'timestamp': 1680979138,
+            'age_limit': 0,
+            'comment_count': int,
+            'dislike_count': int,
+            'like_count': int,
+        },
+        'skip': 'Requires account that has opted-in to the GenZedong subreddit',
     }, {
         'url': 'https://www.reddit.com/r/videos/comments/6rrwyj',
         'only_matching': True,
@@ -137,21 +176,26 @@ class RedditIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def _gen_session_id():
-        id_length = 16
-        rand_max = 1 << (id_length * 4)
-        return '%0.*x' % (id_length, random.randrange(rand_max))
-
     def _real_extract(self, url):
-        subdomain, slug, video_id = self._match_valid_url(url).group('subdomain', 'slug', 'id')
+        host, slug, video_id = self._match_valid_url(url).group('host', 'slug', 'id')
 
-        self._set_cookie('.reddit.com', 'reddit_session', self._gen_session_id())
-        self._set_cookie('.reddit.com', '_options', '%7B%22pref_quarantine_optin%22%3A%20true%7D')
-        data = self._download_json(f'https://{subdomain}reddit.com/{slug}/.json', video_id, fatal=False)
+        data = self._download_json(
+            f'https://{host}/{slug}/.json', video_id, fatal=False, expected_status=403)
         if not data:
-            # Fall back to old.reddit.com in case the requested subdomain fails
-            data = self._download_json(f'https://old.reddit.com/{slug}/.json', video_id)
+            fallback_host = 'old.reddit.com' if host != 'old.reddit.com' else 'www.reddit.com'
+            self.to_screen(f'{host} request failed, retrying with {fallback_host}')
+            data = self._download_json(
+                f'https://{fallback_host}/{slug}/.json', video_id, expected_status=403)
+
+        if traverse_obj(data, 'error') == 403:
+            reason = data.get('reason')
+            if reason == 'quarantined':
+                self.raise_login_required('Quarantined subreddit; an account that has opted in is required')
+            elif reason == 'private':
+                self.raise_login_required('Private subreddit; an account that has been approved is required')
+            else:
+                raise ExtractorError(f'HTTP Error 403 Forbidden; reason given: {reason}')
+
         data = data[0]['data']['children'][0]['data']
         video_url = data['url']
 

From 9c92b803fa24e48543ce969468d5404376e315b7 Mon Sep 17 00:00:00 2001
From: satan1st <satan1st@users.noreply.github.com>
Date: Sun, 16 Apr 2023 19:20:10 +0200
Subject: [PATCH 218/871] [extractor/gronkh] Extract duration and chapters
 (#6817)

Authored by: satan1st
---
 yt_dlp/extractor/gronkh.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/gronkh.py b/yt_dlp/extractor/gronkh.py
index b9370e36c..1ae0a6893 100644
--- a/yt_dlp/extractor/gronkh.py
+++ b/yt_dlp/extractor/gronkh.py
@@ -3,6 +3,7 @@
 from .common import InfoExtractor
 from ..utils import (
     OnDemandPagedList,
+    float_or_none,
     traverse_obj,
     unified_strdate,
 )
@@ -19,7 +20,9 @@ class GronkhIE(InfoExtractor):
             'title': 'H.O.R.D.E. - DAS ZWEiTE ZEiTALTER 🎲 Session 1',
             'view_count': int,
             'thumbnail': 'https://01.cdn.vod.farm/preview/9e2555d3a23bf4e5c5b7c6b3b70a9d84.jpg',
-            'upload_date': '20221111'
+            'upload_date': '20221111',
+            'chapters': 'count:3',
+            'duration': 31463,
         },
         'params': {'skip_download': True}
     }, {
@@ -30,7 +33,8 @@ class GronkhIE(InfoExtractor):
             'title': 'GTV0536, 2021-10-01 - MARTHA IS DEAD  #FREiAB1830  !FF7 !horde !archiv',
             'view_count': int,
             'thumbnail': 'https://01.cdn.vod.farm/preview/6436746cce14e25f751260a692872b9b.jpg',
-            'upload_date': '20211001'
+            'upload_date': '20211001',
+            'duration': 32058,
         },
         'params': {'skip_download': True}
     }, {
@@ -56,6 +60,12 @@ def _real_extract(self, url):
             'upload_date': unified_strdate(data_json.get('created_at')),
             'formats': formats,
             'subtitles': subtitles,
+            'duration': float_or_none(data_json.get('source_length')),
+            'chapters': traverse_obj(data_json, (
+                'chapters', lambda _, v: float_or_none(v['offset']) is not None, {
+                    'title': 'title',
+                    'start_time': ('offset', {float_or_none}),
+                })) or None,
         }
 
 

From 2c566ed14101673c651c08c306c30fa5b4010b85 Mon Sep 17 00:00:00 2001
From: CoryTibbettsDev <70112527+CoryTibbettsDev@users.noreply.github.com>
Date: Sun, 16 Apr 2023 17:26:37 +0000
Subject: [PATCH 219/871] [extractor/whyp] Add extractor (#6803)

Authored by: CoryTibbettsDev
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/whyp.py        | 50 +++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 yt_dlp/extractor/whyp.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 09903423d..b08b3095e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2295,6 +2295,7 @@
     WeiboMobileIE
 )
 from .weiqitv import WeiqiTVIE
+from .whyp import WhypIE
 from .wikimedia import WikimediaIE
 from .willow import WillowIE
 from .wimtv import WimTVIE
diff --git a/yt_dlp/extractor/whyp.py b/yt_dlp/extractor/whyp.py
new file mode 100644
index 000000000..fef89c351
--- /dev/null
+++ b/yt_dlp/extractor/whyp.py
@@ -0,0 +1,50 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    str_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class WhypIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?whyp\.it/tracks/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.whyp.it/tracks/18337/home-page-example-track-b4kq7',
+        'md5': 'c1187b42ebf8605284e3dc92aeb33d16',
+        'info_dict': {
+            'url': 'https://cdn.whyp.it/50eb17cc-e9ff-4e18-b89b-dc9206a95cb1.mp3',
+            'id': '18337',
+            'title': 'Home Page Example Track',
+            'description': 'md5:bd758000fb93f3159339c852b5b9133c',
+            'ext': 'mp3',
+            'duration': 52.82,
+            'uploader': 'Brad',
+            'uploader_id': '1',
+            'thumbnail': 'https://cdn.whyp.it/a537bb36-3373-4c61-96c8-27fc1b2f427a.jpg',
+        },
+    }, {
+        'url': 'https://www.whyp.it/tracks/18337',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        unique_id = self._match_id(url)
+        webpage = self._download_webpage(url, unique_id)
+        data = self._search_nuxt_data(webpage, unique_id)['rawTrack']
+
+        return {
+            'url': data['audio_url'],
+            'id': unique_id,
+            **traverse_obj(data, {
+                'title': 'title',
+                'description': 'description',
+                'duration': ('duration', {float_or_none}),
+                'uploader': ('user', 'username'),
+                'uploader_id': ('user', 'id', {str_or_none}),
+                'thumbnail': ('artwork_url', {url_or_none}),
+            }),
+            'ext': 'mp3',
+            'vcodec': 'none',
+            'http_headers': {'Referer': 'https://whyp.it/'},
+        }

From cbdf9408e6f1e35e98fd6477b3d6902df5b8a47f Mon Sep 17 00:00:00 2001
From: zhgwn <130610452+zhgwn@users.noreply.github.com>
Date: Tue, 18 Apr 2023 04:18:29 +0200
Subject: [PATCH 220/871] [extractor/pornez] Support new URL formats (#6792)

Closes #6791, Closes #6298
Authored by: zhgwn
---
 yt_dlp/extractor/pornez.py | 64 ++++++++++++++++++++++++--------------
 1 file changed, 41 insertions(+), 23 deletions(-)

diff --git a/yt_dlp/extractor/pornez.py b/yt_dlp/extractor/pornez.py
index 3a22cb821..bc45f865e 100644
--- a/yt_dlp/extractor/pornez.py
+++ b/yt_dlp/extractor/pornez.py
@@ -1,42 +1,60 @@
 from .common import InfoExtractor
-from ..utils import int_or_none, urljoin
+from ..utils import (
+    clean_html,
+    int_or_none,
+    get_element_by_class,
+    urljoin,
+)
 
 
 class PornezIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornez\.net/video(?P<id>[0-9]+)/'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?pornez\.net/(?:video(?P<id>\w+)|watch)/'
+    _TESTS = [{
         'url': 'https://pornez.net/video344819/mistresst-funny_penis_names-wmv/',
-        'md5': '2e19a0a1cff3a5dbea0ef1b9e80bcbbc',
         'info_dict': {
             'id': '344819',
             'ext': 'mp4',
-            'title': r'mistresst funny_penis_names wmv',
+            'title': 'mistresst funny_penis_names wmv',
             'thumbnail': r're:^https?://.*\.jpg$',
             'age_limit': 18,
-        }
-    }
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://pornez.net/watch/leana+lovings+stiff+for+stepdaughter/',
+        'info_dict': {
+            'id': '156161',
+            'ext': 'mp4',
+            'title': 'Watch leana lovings stiff for stepdaughter porn video.',
+            'age_limit': 18,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://pornez.net/videovzs27fj/tutor4k-e14-blue-wave-1080p-nbq-tutor4k-e14-blue-wave/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        iframe_src = self._html_search_regex(
-            r'<iframe[^>]+src="([^"]+)"', webpage, 'iframe', fatal=True)
-        iframe_src = urljoin('https://pornez.net', iframe_src)
-        title = self._html_search_meta(['name', 'twitter:title', 'og:title'], webpage, 'title', default=None)
-        if title is None:
-            title = self._search_regex(r'<h1>(.*?)</h1>', webpage, 'title', fatal=True)
-        thumbnail = self._html_search_meta(['thumbnailUrl'], webpage, 'title', default=None)
-        webpage = self._download_webpage(iframe_src, video_id)
-        entries = self._parse_html5_media_entries(iframe_src, webpage, video_id)[0]
-        for format in entries['formats']:
-            height = self._search_regex(r'_(\d+)\.m3u8', format['url'], 'height')
-            format['format_id'] = '%sp' % height
-            format['height'] = int_or_none(height)
+        if not video_id:
+            video_id = self._search_regex(
+                r'<link[^>]+\bhref=["\']https?://pornez.net/\?p=(\w+)["\']', webpage, 'id')
+
+        iframe_src = self._html_search_regex(r'<iframe[^>]+src="([^"]+)"', webpage, 'iframe')
+        iframe = self._download_webpage(urljoin('https://pornez.net', iframe_src), video_id)
+
+        entries = self._parse_html5_media_entries(iframe_src, iframe, video_id)[0]
+        for fmt in entries['formats']:
+            height = self._search_regex(r'_(\d+)\.m3u8', fmt['url'], 'height')
+            fmt['format_id'] = '%sp' % height
+            fmt['height'] = int_or_none(height)
 
         entries.update({
             'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'age_limit': 18
+            'title': (clean_html(get_element_by_class('video-title', webpage))
+                      or self._html_search_meta(
+                      ['twitter:title', 'og:title', 'description'], webpage, 'title', default=None)),
+            'thumbnail': self._html_search_meta(['thumbnailUrl'], webpage, 'thumb', default=None),
+            'age_limit': 18,
         })
         return entries

From e5265dc6517478e589ee3c1ff0cb19bdf4e35ce1 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Mon, 17 Apr 2023 21:27:33 -0500
Subject: [PATCH 221/871] [extractor/stageplus] Add extractor (#6838)

Closes #6806
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/stageplus.py   | 518 ++++++++++++++++++++++++++++++++
 2 files changed, 519 insertions(+)
 create mode 100644 yt_dlp/extractor/stageplus.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b08b3095e..deb92b5fc 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1786,6 +1786,7 @@
     BellatorIE,
     ParamountNetworkIE,
 )
+from .stageplus import StagePlusVODConcertIE
 from .startrek import StarTrekIE
 from .stitcher import (
     StitcherIE,
diff --git a/yt_dlp/extractor/stageplus.py b/yt_dlp/extractor/stageplus.py
new file mode 100644
index 000000000..adb4ebbc2
--- /dev/null
+++ b/yt_dlp/extractor/stageplus.py
@@ -0,0 +1,518 @@
+import json
+import uuid
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    traverse_obj,
+    try_call,
+    unified_timestamp,
+    url_or_none,
+)
+
+
+class StagePlusVODConcertIE(InfoExtractor):
+    _NETRC_MACHINE = 'stageplus'
+    _VALID_URL = r'https?://(?:www\.)?stage-plus\.com/video/(?P<id>vod_concert_\w+)'
+    _TESTS = [{
+        'url': 'https://www.stage-plus.com/video/vod_concert_APNM8GRFDPHMASJKBSPJACG',
+        'playlist_count': 6,
+        'info_dict': {
+            'id': 'vod_concert_APNM8GRFDPHMASJKBSPJACG',
+            'title': 'Yuja Wang plays Rachmaninoff\'s Piano Concerto No. 2 – from Odeonsplatz',
+            'description': 'md5:50f78ec180518c9bdb876bac550996fc',
+            'artist': ['Yuja Wang', 'Lorenzo Viotti'],
+            'upload_date': '20230331',
+            'timestamp': 1680249600,
+            'release_date': '20210709',
+            'release_timestamp': 1625788800,
+            'thumbnails': 'count:3',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'performance_work_A1IN4PJFE9MM2RJ3CLBMUSJBBSOJAD9O',
+                'ext': 'mp4',
+                'title': 'Piano Concerto No. 2 in C Minor, Op. 18',
+                'description': 'md5:50f78ec180518c9bdb876bac550996fc',
+                'upload_date': '20230331',
+                'timestamp': 1680249600,
+                'release_date': '20210709',
+                'release_timestamp': 1625788800,
+                'duration': 2207,
+                'chapters': 'count:5',
+                'artist': ['Yuja Wang'],
+                'composer': ['Sergei Rachmaninoff'],
+                'album': 'Yuja Wang plays Rachmaninoff\'s Piano Concerto No. 2 – from Odeonsplatz',
+                'album_artist': ['Yuja Wang', 'Lorenzo Viotti'],
+                'track': 'Piano Concerto No. 2 in C Minor, Op. 18',
+                'track_number': 1,
+                'genre': 'Instrumental Concerto',
+            },
+        }],
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    # TODO: Prune this after livestream and/or album extractors are added
+    _GRAPHQL_QUERY = '''query videoDetailPage($videoId: ID!, $sliderItemsFirst: Int = 24) {
+  node(id: $videoId) {
+    __typename
+    ...LiveConcertFields
+    ... on LiveConcert {
+      artists {
+        edges {
+          role {
+            ...RoleFields
+          }
+          node {
+            id
+            name
+            sortName
+          }
+        }
+      }
+      isAtmos
+      maxResolution
+      groups {
+        id
+        name
+        typeDisplayName
+      }
+      shortDescription
+      performanceWorks {
+        ...livePerformanceWorkFields
+      }
+      totalDuration
+      sliders {
+        ...contentContainerFields
+      }
+      vodConcert {
+        __typename
+        id
+      }
+    }
+    ...VideoFields
+    ... on Video {
+      artists {
+        edges {
+          role {
+            ...RoleFields
+          }
+          node {
+            id
+            name
+            sortName
+          }
+        }
+      }
+      isAtmos
+      maxResolution
+      isLossless
+      description
+      productionDate
+      takedownDate
+      sliders {
+        ...contentContainerFields
+      }
+    }
+    ...VodConcertFields
+    ... on VodConcert {
+      artists {
+        edges {
+          role {
+            ...RoleFields
+          }
+          node {
+            id
+            name
+            sortName
+          }
+        }
+      }
+      isAtmos
+      maxResolution
+      groups {
+        id
+        name
+        typeDisplayName
+      }
+      performanceWorks {
+        ...PerformanceWorkFields
+      }
+      shortDescription
+      productionDate
+      takedownDate
+      sliders {
+        ...contentContainerFields
+      }
+    }
+  }
+}
+
+fragment LiveConcertFields on LiveConcert {
+  endTime
+  id
+  pictures {
+    ...PictureFields
+  }
+  reruns {
+    ...liveConcertRerunFields
+  }
+  publicationLevel
+  startTime
+  streamStartTime
+  subtitle
+  title
+  typeDisplayName
+  stream {
+    ...liveStreamFields
+  }
+  trailerStream {
+    ...streamFields
+  }
+  geoAccessCountries
+  geoAccessMode
+}
+
+fragment PictureFields on Picture {
+  id
+  url
+  type
+}
+
+fragment liveConcertRerunFields on LiveConcertRerun {
+  streamStartTime
+  endTime
+  startTime
+  stream {
+    ...rerunStreamFields
+  }
+}
+
+fragment rerunStreamFields on RerunStream {
+  publicationLevel
+  streamType
+  url
+}
+
+fragment liveStreamFields on LiveStream {
+  publicationLevel
+  streamType
+  url
+}
+
+fragment streamFields on Stream {
+  publicationLevel
+  streamType
+  url
+}
+
+fragment RoleFields on Role {
+  __typename
+  id
+  type
+  displayName
+}
+
+fragment livePerformanceWorkFields on LivePerformanceWork {
+  __typename
+  id
+  artists {
+    ...artistWithRoleFields
+  }
+  groups {
+    edges {
+      node {
+        id
+        name
+        typeDisplayName
+      }
+    }
+  }
+  work {
+    ...workFields
+  }
+}
+
+fragment artistWithRoleFields on ArtistWithRoleConnection {
+  edges {
+    role {
+      ...RoleFields
+    }
+    node {
+      id
+      name
+      sortName
+    }
+  }
+}
+
+fragment workFields on Work {
+  id
+  title
+  movements {
+    id
+    title
+  }
+  composers {
+    id
+    name
+  }
+  genre {
+    id
+    title
+  }
+}
+
+fragment contentContainerFields on CuratedContentContainer {
+  __typename
+  ...SliderFields
+  ...BannerFields
+}
+
+fragment SliderFields on Slider {
+  id
+  headline
+  items(first: $sliderItemsFirst) {
+    edges {
+      node {
+        id
+        __typename
+        ...AlbumFields
+        ...ArtistFields
+        ...EpochFields
+        ...GenreFields
+        ...GroupFields
+        ...LiveConcertFields
+        ...PartnerFields
+        ...PerformanceWorkFields
+        ...VideoFields
+        ...VodConcertFields
+      }
+    }
+  }
+}
+
+fragment AlbumFields on Album {
+  artistAndGroupDisplayInfo
+  id
+  pictures {
+    ...PictureFields
+  }
+  title
+}
+
+fragment ArtistFields on Artist {
+  id
+  name
+  roles {
+    ...RoleFields
+  }
+  pictures {
+    ...PictureFields
+  }
+}
+
+fragment EpochFields on Epoch {
+  id
+  endYear
+  pictures {
+    ...PictureFields
+  }
+  startYear
+  title
+}
+
+fragment GenreFields on Genre {
+  id
+  pictures {
+    ...PictureFields
+  }
+  title
+}
+
+fragment GroupFields on Group {
+  id
+  name
+  typeDisplayName
+  pictures {
+    ...PictureFields
+  }
+}
+
+fragment PartnerFields on Partner {
+  id
+  name
+  typeDisplayName
+  subtypeDisplayName
+  pictures {
+    ...PictureFields
+  }
+}
+
+fragment PerformanceWorkFields on PerformanceWork {
+  __typename
+  id
+  artists {
+    ...artistWithRoleFields
+  }
+  groups {
+    edges {
+      node {
+        id
+        name
+        typeDisplayName
+      }
+    }
+  }
+  work {
+    ...workFields
+  }
+  stream {
+    ...streamFields
+  }
+  vodConcert {
+    __typename
+    id
+  }
+  duration
+  cuePoints {
+    mark
+    title
+  }
+}
+
+fragment VideoFields on Video {
+  id
+  archiveReleaseDate
+  title
+  subtitle
+  pictures {
+    ...PictureFields
+  }
+  stream {
+    ...streamFields
+  }
+  trailerStream {
+    ...streamFields
+  }
+  duration
+  typeDisplayName
+  duration
+  geoAccessCountries
+  geoAccessMode
+  publicationLevel
+  takedownDate
+}
+
+fragment VodConcertFields on VodConcert {
+  id
+  archiveReleaseDate
+  pictures {
+    ...PictureFields
+  }
+  subtitle
+  title
+  typeDisplayName
+  totalDuration
+  geoAccessCountries
+  geoAccessMode
+  trailerStream {
+   ...streamFields
+  }
+  publicationLevel
+  takedownDate
+}
+
+fragment BannerFields on Banner {
+  description
+  link
+  pictures {
+    ...PictureFields
+  }
+  title
+}'''
+
+    _TOKEN = None
+
+    def _perform_login(self, username, password):
+        auth = self._download_json('https://audience.api.stageplus.io/oauth/token', None, headers={
+            'Content-Type': 'application/json',
+            'Origin': 'https://www.stage-plus.com',
+        }, data=json.dumps({
+            'grant_type': 'password',
+            'username': username,
+            'password': password,
+            'device_info': 'Chrome (Windows)',
+            'client_device_id': str(uuid.uuid4()),
+        }, separators=(',', ':')).encode(), note='Logging in')
+
+        if auth.get('access_token'):
+            self._TOKEN = auth['access_token']
+
+    def _real_initialize(self):
+        if self._TOKEN:
+            return
+
+        self._TOKEN = try_call(
+            lambda: self._get_cookies('https://www.stage-plus.com/')['dgplus_access_token'].value)
+        if not self._TOKEN:
+            self.raise_login_required()
+
+    def _real_extract(self, url):
+        concert_id = self._match_id(url)
+
+        data = self._download_json('https://audience.api.stageplus.io/graphql', concert_id, headers={
+            'authorization': f'Bearer {self._TOKEN}',
+            'content-type': 'application/json',
+            'Origin': 'https://www.stage-plus.com',
+        }, data=json.dumps({
+            'query': self._GRAPHQL_QUERY,
+            'variables': {'videoId': concert_id},
+            'operationName': 'videoDetailPage'
+        }, separators=(',', ':')).encode())['data']['node']
+
+        metadata = traverse_obj(data, {
+            'title': 'title',
+            'description': ('shortDescription', {str}),
+            'artist': ('artists', 'edges', ..., 'node', 'name'),
+            'timestamp': ('archiveReleaseDate', {unified_timestamp}),
+            'release_timestamp': ('productionDate', {unified_timestamp}),
+        })
+
+        thumbnails = traverse_obj(data, ('pictures', lambda _, v: url_or_none(v['url']), {
+            'id': 'name',
+            'url': 'url',
+        })) or None
+
+        m3u8_headers = {'jwt': self._TOKEN}
+
+        entries = []
+        for idx, video in enumerate(traverse_obj(data, (
+                'performanceWorks', lambda _, v: v['id'] and url_or_none(v['stream']['url']))), 1):
+            formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+                video['stream']['url'], video['id'], 'mp4', m3u8_id='hls', headers=m3u8_headers)
+            entries.append({
+                'id': video['id'],
+                'formats': formats,
+                'subtitles': subtitles,
+                'http_headers': m3u8_headers,
+                'album': metadata.get('title'),
+                'album_artist': metadata.get('artist'),
+                'track_number': idx,
+                **metadata,
+                **traverse_obj(video, {
+                    'title': ('work', 'title'),
+                    'track': ('work', 'title'),
+                    'duration': ('duration', {float_or_none}),
+                    'chapters': (
+                        'cuePoints', lambda _, v: float_or_none(v['mark']) is not None, {
+                            'title': 'title',
+                            'start_time': ('mark', {float_or_none}),
+                        }),
+                    'artist': ('artists', 'edges', ..., 'node', 'name'),
+                    'composer': ('work', 'composers', ..., 'name'),
+                    'genre': ('work', 'genre', 'title'),
+                }),
+            })
+
+        return self.playlist_result(entries, concert_id, thumbnails=thumbnails, **metadata)

From ab29e47029e2f5b48abbbab78e82faf7cf6e9506 Mon Sep 17 00:00:00 2001
From: qbnu <93988953+qbnu@users.noreply.github.com>
Date: Tue, 18 Apr 2023 02:37:37 +0000
Subject: [PATCH 222/871] [extractor/bilibili] Support festival videos (#6547)

Closes #6138
Authored by: qbnu
---
 yt_dlp/extractor/bilibili.py | 86 ++++++++++++++++++++++++++++++------
 1 file changed, 73 insertions(+), 13 deletions(-)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 91d436dd8..faa2218ce 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -134,7 +134,7 @@ def _get_all_children(self, reply):
 
 
 class BiliBiliIE(BilibiliBaseIE):
-    _VALID_URL = r'https?://www\.bilibili\.com/video/[aAbB][vV](?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://www\.bilibili\.com/(?:video/|festival/\w+\?(?:[^#]*&)?bvid=)[aAbB][vV](?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'https://www.bilibili.com/video/BV13x41117TL',
@@ -282,19 +282,60 @@ class BiliBiliIE(BilibiliBaseIE):
             'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
         },
         'params': {'skip_download': True},
+    }, {
+        'note': 'video redirects to festival page',
+        'url': 'https://www.bilibili.com/video/BV1wP4y1P72h',
+        'info_dict': {
+            'id': 'BV1wP4y1P72h',
+            'ext': 'mp4',
+            'title': '牛虎年相交之际，一首传统民族打击乐《牛斗虎》祝大家新春快乐，虎年大吉！【bilibili音乐虎闹新春】',
+            'timestamp': 1643947497,
+            'upload_date': '20220204',
+            'description': 'md5:8681a0d4d2c06b4ae27e59c8080a7fe6',
+            'uploader': '叨叨冯聊音乐',
+            'duration': 246.719,
+            'uploader_id': '528182630',
+            'view_count': int,
+            'like_count': int,
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+        },
+        'params': {'skip_download': True},
+    }, {
+        'note': 'newer festival video',
+        'url': 'https://www.bilibili.com/festival/2023honkaiimpact3gala?bvid=BV1ay4y1d77f',
+        'info_dict': {
+            'id': 'BV1ay4y1d77f',
+            'ext': 'mp4',
+            'title': '【崩坏3新春剧场】为特别的你送上祝福！',
+            'timestamp': 1674273600,
+            'upload_date': '20230121',
+            'description': 'md5:58af66d15c6a0122dc30c8adfd828dd8',
+            'uploader': '果蝇轰',
+            'duration': 1111.722,
+            'uploader_id': '8469526',
+            'view_count': int,
+            'like_count': int,
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+        },
+        'params': {'skip_download': True},
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', video_id)
-        play_info = self._search_json(r'window\.__playinfo__\s*=', webpage, 'play info', video_id)['data']
 
-        video_data = initial_state['videoData']
+        is_festival = 'videoData' not in initial_state
+        if is_festival:
+            video_data = initial_state['videoInfo']
+        else:
+            play_info = self._search_json(r'window\.__playinfo__\s*=', webpage, 'play info', video_id)['data']
+            video_data = initial_state['videoData']
+
         video_id, title = video_data['bvid'], video_data.get('title')
 
         # Bilibili anthologies are similar to playlists but all videos share the same video ID as the anthology itself.
-        page_list_json = traverse_obj(
+        page_list_json = not is_festival and traverse_obj(
             self._download_json(
                 'https://api.bilibili.com/x/player/pagelist', video_id,
                 fatal=False, query={'bvid': video_id, 'jsonp': 'jsonp'},
@@ -317,20 +358,39 @@ def _real_extract(self, url):
 
         cid = traverse_obj(video_data, ('pages', part_id - 1, 'cid')) if part_id else video_data.get('cid')
 
+        festival_info = {}
+        if is_festival:
+            play_info = self._download_json(
+                'https://api.bilibili.com/x/player/playurl', video_id,
+                query={'bvid': video_id, 'cid': cid, 'fnval': 4048},
+                note='Extracting festival video formats')['data']
+
+            festival_info = traverse_obj(initial_state, {
+                'uploader': ('videoInfo', 'upName'),
+                'uploader_id': ('videoInfo', 'upMid', {str_or_none}),
+                'like_count': ('videoStatus', 'like', {int_or_none}),
+                'thumbnail': ('sectionEpisodes', lambda _, v: v['bvid'] == video_id, 'cover'),
+            }, get_all=False)
+
         return {
+            **traverse_obj(initial_state, {
+                'uploader': ('upData', 'name'),
+                'uploader_id': ('upData', 'mid', {str_or_none}),
+                'like_count': ('videoData', 'stat', 'like', {int_or_none}),
+                'tags': ('tags', ..., 'tag_name'),
+                'thumbnail': ('videoData', 'pic', {url_or_none}),
+            }),
+            **festival_info,
+            **traverse_obj(video_data, {
+                'description': 'desc',
+                'timestamp': ('pubdate', {int_or_none}),
+                'view_count': (('viewCount', ('stat', 'view')), {int_or_none}),
+                'comment_count': ('stat', 'reply', {int_or_none}),
+            }, get_all=False),
             'id': f'{video_id}{format_field(part_id, None, "_p%d")}',
             'formats': self.extract_formats(play_info),
             '_old_archive_ids': [make_archive_id(self, old_video_id)] if old_video_id else None,
             'title': title,
-            'description': traverse_obj(initial_state, ('videoData', 'desc')),
-            'view_count': traverse_obj(initial_state, ('videoData', 'stat', 'view')),
-            'uploader': traverse_obj(initial_state, ('upData', 'name')),
-            'uploader_id': traverse_obj(initial_state, ('upData', 'mid')),
-            'like_count': traverse_obj(initial_state, ('videoData', 'stat', 'like')),
-            'comment_count': traverse_obj(initial_state, ('videoData', 'stat', 'reply')),
-            'tags': traverse_obj(initial_state, ('tags', ..., 'tag_name')),
-            'thumbnail': traverse_obj(initial_state, ('videoData', 'pic')),
-            'timestamp': traverse_obj(initial_state, ('videoData', 'pubdate')),
             'duration': float_or_none(play_info.get('timelength'), scale=1000),
             'chapters': self._get_chapters(aid, cid),
             'subtitles': self.extract_subtitles(video_id, aid, cid),

From 6a765f135ccb654861336ea27a2c1c24ea8e286f Mon Sep 17 00:00:00 2001
From: vidiot720 <128325907+vidiot720@users.noreply.github.com>
Date: Wed, 19 Apr 2023 09:46:57 +1000
Subject: [PATCH 223/871] [extractor/sbs] Overhaul extractor for new API
 (#6839)

Closes #6543
Authored by: vidiot720, dirkf, bashonly
---
 yt_dlp/extractor/sbs.py | 109 ++++++++++++++++++++++++++++++----------
 yt_dlp/utils.py         |   4 ++
 2 files changed, 86 insertions(+), 27 deletions(-)

diff --git a/yt_dlp/extractor/sbs.py b/yt_dlp/extractor/sbs.py
index 45320339d..ac0b6de20 100644
--- a/yt_dlp/extractor/sbs.py
+++ b/yt_dlp/extractor/sbs.py
@@ -1,7 +1,13 @@
 from .common import InfoExtractor
 from ..utils import (
-    smuggle_url,
-    ExtractorError,
+    HEADRequest,
+    float_or_none,
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+    traverse_obj,
+    update_url_query,
+    url_or_none,
 )
 
 
@@ -11,7 +17,7 @@ class SBSIE(InfoExtractor):
         https?://(?:www\.)?sbs\.com\.au/(?:
             ondemand(?:
                 /video/(?:single/)?|
-                /movie/[^/]+/|
+                /(?:movie|tv-program)/[^/]+/|
                 /(?:tv|news)-series/(?:[^/]+/){3}|
                 .*?\bplay=|/watch/
             )|news/(?:embeds/)?video/
@@ -27,18 +33,21 @@ class SBSIE(InfoExtractor):
         # Original URL is handled by the generic IE which finds the iframe:
         # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation
         'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed',
-        'md5': '3150cf278965eeabb5b4cea1c963fe0a',
+        'md5': '31f84a7a19b53635db63c73f8ab0c4a7',
         'info_dict': {
-            'id': '_rFBPRPO4pMR',
+            'id': '320403011771',  # '_rFBPRPO4pMR',
             'ext': 'mp4',
             'title': 'Dingo Conservation (The Feed)',
             'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5',
-            'thumbnail': r're:http://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
             'duration': 308,
             'timestamp': 1408613220,
             'upload_date': '20140821',
             'uploader': 'SBSC',
+            'tags': None,
+            'categories': None,
         },
+        'expected_warnings': ['Unable to download JSON metadata'],
     }, {
         'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
         'only_matching': True,
@@ -70,34 +79,80 @@ class SBSIE(InfoExtractor):
     }, {
         'url': 'https://www.sbs.com.au/ondemand/tv-series/the-handmaids-tale/season-5/the-handmaids-tale-s5-ep1/2065631811776',
         'only_matching': True,
+    }, {
+        'url': 'https://www.sbs.com.au/ondemand/tv-program/autun-romes-forgotten-sister/2116212803602',
+        'only_matching': True,
     }]
 
+    _GEO_COUNTRIES = ['AU']
+    _AUS_TV_PARENTAL_GUIDELINES = {
+        'P': 0,
+        'C': 7,
+        'G': 0,
+        'PG': 0,
+        'M': 14,
+        'MA15+': 15,
+        'MAV15+': 15,
+        'R18+': 18,
+    }
+    _PLAYER_API = 'https://www.sbs.com.au/api/v3'
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        player_params = self._download_json(
-            'http://www.sbs.com.au/api/video_pdkvars/id/%s?form=json' % video_id, video_id)
+        formats, subtitles = self._extract_smil_formats_and_subtitles(
+            update_url_query(f'{self._PLAYER_API}/video_smil', {'id': video_id}), video_id)
 
-        error = player_params.get('error')
-        if error:
-            error_message = 'Sorry, The video you are looking for does not exist.'
-            video_data = error.get('results') or {}
-            error_code = error.get('errorCode')
-            if error_code == 'ComingSoon':
-                error_message = '%s is not yet available.' % video_data.get('title', '')
-            elif error_code in ('Forbidden', 'intranetAccessOnly'):
-                error_message = 'Sorry, This video cannot be accessed via this website'
-            elif error_code == 'Expired':
-                error_message = 'Sorry, %s is no longer available.' % video_data.get('title', '')
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
+        if not formats:
+            urlh = self._request_webpage(
+                HEADRequest('https://sbs-vod-prod-01.akamaized.net/'), video_id,
+                note='Checking geo-restriction', fatal=False, expected_status=403)
+            if urlh:
+                error_reasons = urlh.headers.get_all('x-error-reason') or []
+                if 'geo-blocked' in error_reasons:
+                    self.raise_geo_restricted(countries=['AU'])
+            self.raise_no_formats('No formats are available', video_id=video_id)
 
-        urls = player_params['releaseUrls']
-        theplatform_url = (urls.get('progressive') or urls.get('html')
-                           or urls.get('standard') or player_params['relatedItemsURL'])
+        media = traverse_obj(self._download_json(
+            f'{self._PLAYER_API}/video_stream', video_id, fatal=False,
+            query={'id': video_id, 'context': 'tv'}), ('video_object', {dict})) or {}
+
+        media.update(self._download_json(
+            f'https://catalogue.pr.sbsod.com/mpx-media/{video_id}',
+            video_id, fatal=not media) or {})
+
+        # For named episodes, use the catalogue's title to set episode, rather than generic 'Episode N'.
+        if traverse_obj(media, ('partOfSeries', {dict})):
+            media['epName'] = traverse_obj(media, ('title', {str}))
 
         return {
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
             'id': video_id,
-            'url': smuggle_url(self._proto_relative_url(theplatform_url), {'force_smil_url': True}),
-            'is_live': player_params.get('streamType') == 'live',
+            **traverse_obj(media, {
+                'title': ('name', {str}),
+                'description': ('description', {str}),
+                'channel': ('taxonomy', 'channel', 'name', {str}),
+                'series': ((('partOfSeries', 'name'), 'seriesTitle'), {str}),
+                'series_id': ((('partOfSeries', 'uuid'), 'seriesID'), {str}),
+                'season_number': ('seasonNumber', {int_or_none}),
+                'episode': ('epName', {str}),
+                'episode_number': ('episodeNumber', {int_or_none}),
+                'timestamp': (('datePublished', ('publication', 'startDate')), {parse_iso8601}),
+                'release_year': ('releaseYear', {int_or_none}),
+                'duration': ('duration', ({float_or_none}, {parse_duration})),
+                'is_live': ('liveStream', {bool}),
+                'age_limit': (
+                    ('classificationID', 'contentRating'), {str.upper}, {self._AUS_TV_PARENTAL_GUIDELINES.get}),
+            }, get_all=False),
+            **traverse_obj(media, {
+                'categories': (('genres', ...), ('taxonomy', ('genre', 'subgenre'), 'name'), {str}),
+                'tags': (('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice')), ..., {str}),
+                'thumbnails': ('thumbnails', lambda _, v: url_or_none(v['contentUrl']), {
+                    'id': ('name', {str}),
+                    'url': 'contentUrl',
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                }),
+            }),
+            'formats': formats,
+            'subtitles': subtitles,
+            'uploader': 'SBSC',
         }
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 40533c2cb..746a2885d 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -4093,6 +4093,10 @@ def data(self, data):
         def close(self):
             return self._out.strip()
 
+    # Fix UTF-8 encoded file wrongly marked as UTF-16. See https://github.com/yt-dlp/yt-dlp/issues/6543#issuecomment-1477169870
+    # This will not trigger false positives since only UTF-8 text is being replaced
+    dfxp_data = dfxp_data.replace(b'encoding=\'UTF-16\'', b'encoding=\'UTF-8\'')
+
     def parse_node(node):
         target = TTMLPElementParser()
         parser = xml.etree.ElementTree.XMLParser(target=target)

From 8f0be90ecb3b8d862397177bb226f17b245ef933 Mon Sep 17 00:00:00 2001
From: garret <76261416+garret1317@users.noreply.github.com>
Date: Wed, 19 Apr 2023 05:21:24 +0100
Subject: [PATCH 224/871] [extractor/nhk] Add `NhkRadiru` extractor (#6819)

* Add `NhkRadioNewsPage` extractor

Authored by: garret1317
---
 yt_dlp/extractor/_extractors.py |   2 +
 yt_dlp/extractor/nhk.py         | 140 +++++++++++++++++++++++++++++++-
 2 files changed, 141 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index deb92b5fc..58137d7f6 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1232,6 +1232,8 @@
     NhkForSchoolBangumiIE,
     NhkForSchoolSubjectIE,
     NhkForSchoolProgramListIE,
+    NhkRadioNewsPageIE,
+    NhkRadiruIE,
 )
 from .nhl import NHLIE
 from .nick import (
diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py
index 59702b247..1597962ac 100644
--- a/yt_dlp/extractor/nhk.py
+++ b/yt_dlp/extractor/nhk.py
@@ -6,7 +6,8 @@
     traverse_obj,
     unescapeHTML,
     unified_timestamp,
-    urljoin
+    urljoin,
+    url_or_none
 )
 
 
@@ -334,3 +335,140 @@ def _real_extract(self, url):
             for x in traverse_obj(bangumi_list, ('part', ..., 'part-video-dasid')) or []]
 
         return self.playlist_result(bangumis, program_id, title, description)
+
+
+class NhkRadiruIE(InfoExtractor):
+    _GEO_COUNTRIES = ['JP']
+    IE_DESC = 'NHK らじる (Radiru/Rajiru)'
+    _VALID_URL = r'https?://www\.nhk\.or\.jp/radio/(?:player/ondemand|ondemand/detail)\.html\?p=(?P<site>[\da-zA-Z]+)_(?P<corner>[\da-zA-Z]+)(?:_(?P<headline>[\da-zA-Z]+))?'
+    _TESTS = [{
+        'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=0449_01_3853544',
+        'skip': 'Episode expired on 2023-04-16',
+        'info_dict': {
+            'channel': 'NHK-FM',
+            'description': 'md5:94b08bdeadde81a97df4ec882acce3e9',
+            'ext': 'm4a',
+            'id': '0449_01_3853544',
+            'series': 'ジャズ・トゥナイト',
+            'thumbnail': 'https://www.nhk.or.jp/prog/img/449/g449.jpg',
+            'timestamp': 1680969600,
+            'title': 'ジャズ・トゥナイト　ＮＥＷジャズ特集',
+            'upload_date': '20230408',
+            'release_timestamp': 1680962400,
+            'release_date': '20230408',
+            'was_live': True,
+        },
+    }, {
+        # playlist, airs every weekday so it should _hopefully_ be okay forever
+        'url': 'https://www.nhk.or.jp/radio/ondemand/detail.html?p=0458_01',
+        'info_dict': {
+            'id': '0458_01',
+            'title': 'ベストオブクラシック',
+            'description': '世界中の上質な演奏会をじっくり堪能する本格派クラシック番組。',
+            'channel': 'NHK-FM',
+            'thumbnail': 'https://www.nhk.or.jp/prog/img/458/g458.jpg',
+        },
+        'playlist_mincount': 3,
+    }, {
+        # one with letters in the id
+        'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=F300_06_3738470',
+        'note': 'Expires on 2024-03-31',
+        'info_dict': {
+            'id': 'F300_06_3738470',
+            'ext': 'm4a',
+            'title': '有島武郎「一房のぶどう」',
+            'description': '朗読：川野一宇（ラジオ深夜便アンカー）\r\n\r\n（2016年12月8日放送「ラジオ深夜便『アンカー朗読シリーズ』」より）',
+            'channel': 'NHKラジオ第1、NHK-FM',
+            'timestamp': 1635757200,
+            'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F300/img/corner/box_109_thumbnail.jpg',
+            'release_date': '20161207',
+            'series': 'らじる文庫 by ラジオ深夜便 ',
+            'release_timestamp': 1481126700,
+            'upload_date': '20211101',
+        }
+    }, {
+        # news
+        'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=F261_01_3855109',
+        'skip': 'Expires on 2023-04-17',
+        'info_dict': {
+            'id': 'F261_01_3855109',
+            'ext': 'm4a',
+            'channel': 'NHKラジオ第1',
+            'timestamp': 1681635900,
+            'release_date': '20230416',
+            'series': 'NHKラジオニュース',
+            'title': '午後６時のNHKニュース',
+            'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F261/img/RADIONEWS_640.jpg',
+            'upload_date': '20230416',
+            'release_timestamp': 1681635600,
+        },
+    }]
+
+    def _extract_episode_info(self, headline, programme_id, series_meta):
+        episode_id = f'{programme_id}_{headline["headline_id"]}'
+        episode = traverse_obj(headline, ('file_list', 0, {dict}))
+
+        return {
+            **series_meta,
+            'id': episode_id,
+            'formats': self._extract_m3u8_formats(episode.get('file_name'), episode_id, fatal=False),
+            'container': 'm4a_dash',  # force fixup, AAC-only HLS
+            'was_live': True,
+            'series': series_meta.get('title'),
+            'thumbnail': url_or_none(headline.get('headline_image')) or series_meta.get('thumbnail'),
+            **traverse_obj(episode, {
+                'title': 'file_title',
+                'description': 'file_title_sub',
+                'timestamp': ('open_time', {unified_timestamp}),
+                'release_timestamp': ('aa_vinfo4', {lambda x: x.split('_')[0]}, {unified_timestamp}),
+            }),
+        }
+
+    def _real_extract(self, url):
+        site_id, corner_id, headline_id = self._match_valid_url(url).group('site', 'corner', 'headline')
+        programme_id = f'{site_id}_{corner_id}'
+
+        if site_id == 'F261':
+            json_url = 'https://www.nhk.or.jp/s-media/news/news-site/list/v1/all.json'
+        else:
+            json_url = f'https://www.nhk.or.jp/radioondemand/json/{site_id}/bangumi_{programme_id}.json'
+
+        meta = self._download_json(json_url, programme_id)['main']
+
+        series_meta = traverse_obj(meta, {
+            'title': 'program_name',
+            'channel': 'media_name',
+            'thumbnail': (('thumbnail_c', 'thumbnail_p'), {url_or_none}),
+        }, get_all=False)
+
+        if headline_id:
+            return self._extract_episode_info(
+                traverse_obj(meta, (
+                    'detail_list', lambda _, v: v['headline_id'] == headline_id), get_all=False),
+                programme_id, series_meta)
+
+        def entries():
+            for headline in traverse_obj(meta, ('detail_list', ..., {dict})):
+                yield self._extract_episode_info(headline, programme_id, series_meta)
+
+        return self.playlist_result(
+            entries(), programme_id, playlist_description=meta.get('site_detail'), **series_meta)
+
+
+class NhkRadioNewsPageIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.nhk\.or\.jp/radionews/?(?:$|[?#])'
+    _TESTS = [{
+        # airs daily, on-the-hour most hours
+        'url': 'https://www.nhk.or.jp/radionews/',
+        'playlist_mincount': 5,
+        'info_dict': {
+            'id': 'F261_01',
+            'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F261/img/RADIONEWS_640.jpg',
+            'description': 'md5:bf2c5b397e44bc7eb26de98d8f15d79d',
+            'channel': 'NHKラジオ第1',
+            'title': 'NHKラジオニュース',
+        }
+    }]
+
+    def _real_extract(self, url):
+        return self.url_result('https://www.nhk.or.jp/radio/ondemand/detail.html?p=F261_01', NhkRadiruIE)

From 1ea15603d852971ed7d92f4de12808b27b3d9370 Mon Sep 17 00:00:00 2001
From: truedread <truedread11@gmail.com>
Date: Fri, 21 Apr 2023 20:11:51 -0400
Subject: [PATCH 225/871] [extractor/wevidi] Add extractor (#6868)

Closes #6129
Authored by: truedread
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/wevidi.py      | 108 ++++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 yt_dlp/extractor/wevidi.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 58137d7f6..a81682e43 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2298,6 +2298,7 @@
     WeiboMobileIE
 )
 from .weiqitv import WeiqiTVIE
+from .wevidi import WeVidiIE
 from .whyp import WhypIE
 from .wikimedia import WikimediaIE
 from .willow import WillowIE
diff --git a/yt_dlp/extractor/wevidi.py b/yt_dlp/extractor/wevidi.py
new file mode 100644
index 000000000..3b6d03238
--- /dev/null
+++ b/yt_dlp/extractor/wevidi.py
@@ -0,0 +1,108 @@
+from .common import InfoExtractor
+from ..utils import clean_html, float_or_none, get_element_by_class, js_to_json, traverse_obj
+
+
+class WeVidiIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?wevidi\.net/watch/(?P<id>[\w-]{11})'
+    _TESTS = [{
+        'url': 'https://wevidi.net/watch/2th7UO5F4KV',
+        'md5': 'b913d1ff5bbad499e2c7ef4aa6d829d7',
+        'info_dict': {
+            'id': '2th7UO5F4KV',
+            'ext': 'mp4',
+            'title': 'YouTube Alternative: WeVidi - customizable channels & more',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:73a27d0a87d49fbcc5584566326ebeed',
+            'uploader': 'eclecRC',
+            'duration': 932.098,
+        }
+    }, {
+        'url': 'https://wevidi.net/watch/ievRuuQHbPS',
+        'md5': 'ce8a94989a959bff9003fa27ee572935',
+        'info_dict': {
+            'id': 'ievRuuQHbPS',
+            'ext': 'mp4',
+            'title': 'WeVidi Playlists',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:32cdfca272687390d9bd9b0c9c6153ee',
+            'uploader': 'WeVidi',
+            'duration': 36.1999,
+        }
+    }, {
+        'url': 'https://wevidi.net/watch/PcMzDWaQSWb',
+        'md5': '55ee0d3434be5d9e5cc76b83f2bb57ec',
+        'info_dict': {
+            'id': 'PcMzDWaQSWb',
+            'ext': 'mp4',
+            'title': 'Cat blep',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:e2c9e2b54b8bb424cc64937c8fdc068f',
+            'uploader': 'WeVidi',
+            'duration': 41.972,
+        }
+    }, {
+        'url': 'https://wevidi.net/watch/wJnRqDHNe_u',
+        'md5': 'c8f263dd47e66cc17546b3abf47b5a77',
+        'info_dict': {
+            'id': 'wJnRqDHNe_u',
+            'ext': 'mp4',
+            'title': 'Gissy Talks: YouTube Alternatives',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:e65036f0d4af80e0af191bd11af5195e',
+            'uploader': 'GissyEva',
+            'duration': 630.451,
+        }
+    }, {
+        'url': 'https://wevidi.net/watch/4m1c4yJR_yc',
+        'md5': 'c63ce5ca6990dce86855fc02ca5bc1ed',
+        'info_dict': {
+            'id': '4m1c4yJR_yc',
+            'ext': 'mp4',
+            'title': 'Enough of that! - Awesome Exilez Podcast',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:96af99dd63468b2dfab3020560e3e9b2',
+            'uploader': 'eclecRC',
+            'duration': 6.804,
+        }
+    }]
+
+    def _extract_formats(self, wvplayer_props):
+        # Taken from WeVidi player JS: https://wevidi.net/layouts/default/static/player.min.js
+        resolution_map = {
+            1: 144,
+            2: 240,
+            3: 360,
+            4: 480,
+            5: 720,
+            6: 1080
+        }
+
+        src_path = f'{wvplayer_props["srcVID"]}/{wvplayer_props["srcUID"]}/{wvplayer_props["srcNAME"]}'
+        for res in traverse_obj(wvplayer_props, ('resolutions', ..., {int}, {lambda x: x or None})):
+            format_id = str(-(res // -2) - 1)
+            yield {
+                'acodec': 'mp4a.40.2',
+                'ext': 'mp4',
+                'format_id': format_id,
+                'height': resolution_map.get(res),
+                'url': f'https://www.wevidi.net/videoplayback/{src_path}/{format_id}',
+                'vcodec': 'avc1.42E01E',
+            }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        wvplayer_props = self._search_json(
+            r'WVPlayer\(', webpage, 'player', video_id,
+            transform_source=lambda x: js_to_json(x.replace('||', '}')))
+
+        return {
+            'id': video_id,
+            'title': clean_html(get_element_by_class('video_title', webpage)),
+            'description': clean_html(get_element_by_class('descr_long', webpage)),
+            'uploader': clean_html(get_element_by_class('username', webpage)),
+            'formats': list(self._extract_formats(wvplayer_props)),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': float_or_none(wvplayer_props.get('duration')),
+        }

From 80b732b7a9585b2a61e456dc0d2d014a439cbaee Mon Sep 17 00:00:00 2001
From: JC-Chung <52159296+JC-Chung@users.noreply.github.com>
Date: Sun, 23 Apr 2023 07:25:04 +0800
Subject: [PATCH 226/871] [extractor/twitch] Extract original size thumbnail
 (#6629)

Authored by: JC-Chung
---
 yt_dlp/extractor/twitch.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index 6321297bb..9b333f6f6 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -179,6 +179,14 @@ def _download_access_token(self, video_id, token_kind, param_name):
             video_id, ops,
             'Downloading %s access token GraphQL' % token_kind)['data'][method]
 
+    def _get_thumbnails(self, thumbnail):
+        return [{
+            'url': re.sub(r'\d+x\d+(\.\w+)($|(?=[?#]))', r'0x0\g<1>', thumbnail),
+            'preference': 1,
+        }, {
+            'url': thumbnail,
+        }] if thumbnail else None
+
 
 class TwitchVodIE(TwitchBaseIE):
     IE_NAME = 'twitch:vod'
@@ -460,15 +468,13 @@ def _extract_info_gql(self, info, item_id):
                 is_live, thumbnail = True, None
             else:
                 is_live = False
-                for p in ('width', 'height'):
-                    thumbnail = thumbnail.replace('{%s}' % p, '0')
 
         return {
             'id': vod_id,
             'title': info.get('title') or 'Untitled Broadcast',
             'description': info.get('description'),
             'duration': int_or_none(info.get('lengthSeconds')),
-            'thumbnail': thumbnail,
+            'thumbnails': self._get_thumbnails(thumbnail),
             'uploader': try_get(info, lambda x: x['owner']['displayName'], compat_str),
             'uploader_id': try_get(info, lambda x: x['owner']['login'], compat_str),
             'timestamp': unified_timestamp(info.get('publishedAt')),
@@ -1053,7 +1059,7 @@ def _real_extract(self, url):
             'display_id': channel_name,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
+            'thumbnails': self._get_thumbnails(thumbnail),
             'uploader': uploader,
             'uploader_id': channel_name,
             'timestamp': timestamp,

From 78fde6e3398ff11e5d383a66b28664badeab5180 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 24 Apr 2023 17:21:20 +0530
Subject: [PATCH 227/871] [outtmpl] Allow `\n` in replacements and default.

Fixes: https://github.com/yt-dlp/yt-dlp/issues/6808#issuecomment-1510055357
Fixes: https://github.com/yt-dlp/yt-dlp/issues/6808#issuecomment-1510363645
---
 test/test_YoutubeDL.py | 1 +
 yt_dlp/YoutubeDL.py    | 2 +-
 yt_dlp/options.py      | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 8da1e5e4b..49ae9e2b1 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -822,6 +822,7 @@ def expect_same_infodict(out):
         test('%(title&foo|baz)s.bar', 'baz.bar')
         test('%(x,id&foo|baz)s.bar', 'foo.bar')
         test('%(x,title&foo|baz)s.bar', 'baz.bar')
+        test('%(title&\n|)s', '\n')
 
         # Laziness
         def gen():
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 31f7645dc..61c149e47 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1156,7 +1156,7 @@ def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False):
         }
         MATH_FIELD_RE = rf'(?:{FIELD_RE}|-?{NUMBER_RE})'
         MATH_OPERATORS_RE = r'(?:%s)' % '|'.join(map(re.escape, MATH_FUNCTIONS.keys()))
-        INTERNAL_FORMAT_RE = re.compile(rf'''(?x)
+        INTERNAL_FORMAT_RE = re.compile(rf'''(?xs)
             (?P<negate>-)?
             (?P<fields>{FIELD_RE})
             (?P<maths>(?:{MATH_OPERATORS_RE}{MATH_FIELD_RE})*)
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 84aeda7f1..d334a9caa 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -243,7 +243,7 @@ def _dict_from_options_callback(
         if multiple_keys:
             allowed_keys = fr'({allowed_keys})(,({allowed_keys}))*'
         mobj = re.match(
-            fr'(?i)(?P<keys>{allowed_keys}){delimiter}(?P<val>.*)$',
+            fr'(?is)(?P<keys>{allowed_keys}){delimiter}(?P<val>.*)$',
             value[0] if multiple_args else value)
         if mobj is not None:
             keys, val = mobj.group('keys').split(','), mobj.group('val')

From ec9311c41b111110bc52cfbd6ea682c6fb23f77a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 24 Apr 2023 18:31:36 +0530
Subject: [PATCH 228/871] [outtmpl] Support `str.format` syntax inside
 replacements

Closes #6843
---
 README.md              |  2 +-
 test/test_YoutubeDL.py |  5 ++++-
 yt_dlp/YoutubeDL.py    | 18 ++++++++++++++++--
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 35229f728..efb490ab1 100644
--- a/README.md
+++ b/README.md
@@ -1246,7 +1246,7 @@ # OUTPUT TEMPLATE
 
 1. **Alternatives**: Alternate fields can be specified separated with a `,`. E.g. `%(release_date>%Y,upload_date>%Y|Unknown)s`
 
-1. **Replacement**: A replacement value can be specified using a `&` separator. If the field is *not* empty, this replacement value will be used instead of the actual field content. This is done after alternate fields are considered; thus the replacement is used if *any* of the alternative fields is *not* empty.
+1. **Replacement**: A replacement value can be specified using a `&` separator according to the [`str.format` mini-language](https://docs.python.org/3/library/string.html#format-specification-mini-language). If the field is *not* empty, this replacement value will be used instead of the actual field content. This is done after alternate fields are considered; thus the replacement is used if *any* of the alternative fields is *not* empty. E.g. `%(chapters&has chapters|no chapters)s`, `%(title&TITLE={:>20}|NO TITLE)s`
 
 1. **Default**: A literal default value can be specified for when the field is empty using a `|` separator. This overrides `--output-na-placeholder`. E.g. `%(uploader|Unknown)s`
 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 49ae9e2b1..3c26bd7c6 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -822,7 +822,10 @@ def expect_same_infodict(out):
         test('%(title&foo|baz)s.bar', 'baz.bar')
         test('%(x,id&foo|baz)s.bar', 'foo.bar')
         test('%(x,title&foo|baz)s.bar', 'baz.bar')
-        test('%(title&\n|)s', '\n')
+        test('%(id&a\nb|)s', ('a\nb', 'a b'))
+        test('%(id&hi {:>10} {}|)s', 'hi       1234 1234')
+        test(R'%(id&{0} {}|)s', 'NA')
+        test(R'%(id&{0.1}|)s', 'NA')
 
         # Laziness
         def gen():
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 61c149e47..dce6cf928 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -21,7 +21,7 @@
 import traceback
 import unicodedata
 import urllib.request
-from string import ascii_letters
+from string import Formatter, ascii_letters
 
 from .cache import Cache
 from .compat import compat_os_name, compat_shlex_quote
@@ -1237,6 +1237,14 @@ def _dumpjson_default(obj):
                 return list(obj)
             return repr(obj)
 
+        class _ReplacementFormatter(Formatter):
+            def get_field(self, field_name, args, kwargs):
+                if field_name.isdigit():
+                    return args[0], -1
+                raise ValueError('Unsupported field')
+
+        replacement_formatter = _ReplacementFormatter()
+
         def create_key(outer_mobj):
             if not outer_mobj.group('has_key'):
                 return outer_mobj.group(0)
@@ -1258,7 +1266,13 @@ def create_key(outer_mobj):
             if fmt == 's' and value is not None and key in field_size_compat_map.keys():
                 fmt = f'0{field_size_compat_map[key]:d}d'
 
-            value = default if value is None else value if replacement is None else replacement
+            if value is None:
+                value = default
+            elif replacement is not None:
+                try:
+                    value = replacement_formatter.format(replacement, value)
+                except ValueError:
+                    value = na
 
             flags = outer_mobj.group('conversion') or ''
             str_fmt = f'{fmt[:-1]}s'

From d669772c65e8630162fd6555d0a578b246591921 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 24 Apr 2023 18:52:09 +0530
Subject: [PATCH 229/871] Add `--no-quiet`

Closes #6796
---
 README.md          | 1 +
 yt_dlp/__init__.py | 3 ++-
 yt_dlp/options.py  | 6 +++++-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index efb490ab1..ef0c236b5 100644
--- a/README.md
+++ b/README.md
@@ -752,6 +752,7 @@ ## Internet Shortcut Options:
 ## Verbosity and Simulation Options:
     -q, --quiet                     Activate quiet mode. If used with --verbose,
                                     print the log to stderr
+    --no-quiet                      Deactivate quiet mode. (Default)
     --no-warnings                   Ignore warnings
     -s, --simulate                  Do not download the video and do not write
                                     anything to disk
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index bdac1212c..79b9a7679 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -704,7 +704,8 @@ def parse_options(argv=None):
         'dumpjson', 'dump_single_json', 'getdescription', 'getduration', 'getfilename',
         'getformat', 'getid', 'getthumbnail', 'gettitle', 'geturl'
     ))
-    opts.quiet = opts.quiet or any_getting or opts.print_json or bool(opts.forceprint)
+    if opts.quiet is None:
+        opts.quiet = any_getting or opts.print_json or bool(opts.forceprint)
 
     playlist_pps = [pp for pp in postprocessors if pp.get('when') == 'playlist']
     write_playlist_infojson = (opts.writeinfojson and not opts.clean_infojson
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index d334a9caa..a2f508552 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -1079,8 +1079,12 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
     verbosity = optparse.OptionGroup(parser, 'Verbosity and Simulation Options')
     verbosity.add_option(
         '-q', '--quiet',
-        action='store_true', dest='quiet', default=False,
+        action='store_true', dest='quiet', default=None,
         help='Activate quiet mode. If used with --verbose, print the log to stderr')
+    verbosity.add_option(
+        '--no-quiet',
+        action='store_false', dest='quiet',
+        help='Deactivate quiet mode. (Default)')
     verbosity.add_option(
         '--no-warnings',
         dest='no_warnings', action='store_true', default=False,

From 04f8018a0544736a18494bc3899d06b05b78fae6 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 24 Apr 2023 18:59:07 +0530
Subject: [PATCH 230/871] [extractor/hentaistigma] Remove extractor

Piracy site

Closes #6870
---
 yt_dlp/extractor/_extractors.py  |  1 -
 yt_dlp/extractor/hentaistigma.py | 37 --------------------------------
 2 files changed, 38 deletions(-)
 delete mode 100644 yt_dlp/extractor/hentaistigma.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index a81682e43..750708d77 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -721,7 +721,6 @@
 from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
-from .hentaistigma import HentaiStigmaIE
 from .hgtv import HGTVComShowIE
 from .hketv import HKETVIE
 from .hidive import HiDiveIE
diff --git a/yt_dlp/extractor/hentaistigma.py b/yt_dlp/extractor/hentaistigma.py
deleted file mode 100644
index ca5ffc2ae..000000000
--- a/yt_dlp/extractor/hentaistigma.py
+++ /dev/null
@@ -1,37 +0,0 @@
-from .common import InfoExtractor
-
-
-class HentaiStigmaIE(InfoExtractor):
-    _VALID_URL = r'^https?://hentai\.animestigma\.com/(?P<id>[^/]+)'
-    _TEST = {
-        'url': 'http://hentai.animestigma.com/inyouchuu-etsu-bonus/',
-        'md5': '4e3d07422a68a4cc363d8f57c8bf0d23',
-        'info_dict': {
-            'id': 'inyouchuu-etsu-bonus',
-            'ext': 'mp4',
-            'title': 'Inyouchuu Etsu Bonus',
-            'age_limit': 18,
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(
-            r'<h2[^>]+class="posttitle"[^>]*><a[^>]*>([^<]+)</a>',
-            webpage, 'title')
-        wrap_url = self._html_search_regex(
-            r'<iframe[^>]+src="([^"]+mp4)"', webpage, 'wrapper url')
-        wrap_webpage = self._download_webpage(wrap_url, video_id)
-
-        video_url = self._html_search_regex(
-            r'file\s*:\s*"([^"]+)"', wrap_webpage, 'video url')
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'age_limit': 18,
-        }

From c16644642b08e2bf4130a6c5fa01395d8718c990 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 24 Apr 2023 19:38:58 +0530
Subject: [PATCH 231/871] Add option `--xff`

Deprecates `--geo-bypass`, `--no-geo-bypass, `--geo-bypass-country`, `--geo-bypass-ip-block`
---
 README.md          | 18 +++++++++---------
 yt_dlp/__init__.py | 13 +++++++++----
 yt_dlp/options.py  | 25 +++++++++++++++----------
 3 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index ef0c236b5..47da19011 100644
--- a/README.md
+++ b/README.md
@@ -463,15 +463,11 @@ ## Geo-restriction:
                                     specified by --proxy (or none, if the option
                                     is not present) is used for the actual
                                     downloading
-    --geo-bypass                    Bypass geographic restriction via faking
-                                    X-Forwarded-For HTTP header (default)
-    --no-geo-bypass                 Do not bypass geographic restriction via
-                                    faking X-Forwarded-For HTTP header
-    --geo-bypass-country CODE       Force bypass geographic restriction with
-                                    explicitly provided two-letter ISO 3166-2
-                                    country code
-    --geo-bypass-ip-block IP_BLOCK  Force bypass geographic restriction with
-                                    explicitly provided IP block in CIDR notation
+    --xff VALUE                     How to fake X-Forwarded-For HTTP header to
+                                    try bypassing geographic restriction. One of
+                                    "default" (Only when known to be useful),
+                                    "never", a two-letter ISO 3166-2 country
+                                    code, or an IP block in CIDR notation
 
 ## Video Selection:
     -I, --playlist-items ITEM_SPEC  Comma separated playlist_index of the items
@@ -2168,6 +2164,10 @@ #### Not recommended
     --youtube-skip-hls-manifest      --extractor-args "youtube:skip=hls" (Alias: --no-youtube-include-hls-manifest)
     --youtube-include-dash-manifest  Default (Alias: --no-youtube-skip-dash-manifest)
     --youtube-include-hls-manifest   Default (Alias: --no-youtube-skip-hls-manifest)
+    --geo-bypass                     --xff "default"
+    --no-geo-bypass                  --xff "never"
+    --geo-bypass-country CODE        --xff CODE
+    --geo-bypass-ip-block IP_BLOCK   --xff IP_BLOCK
 
 
 #### Developer options
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 79b9a7679..47ee3cc02 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -396,12 +396,17 @@ def metadataparser_actions(f):
         except Exception as err:
             raise ValueError(f'Invalid playlist-items {opts.playlist_items!r}: {err}')
 
-    geo_bypass_code = opts.geo_bypass_ip_block or opts.geo_bypass_country
-    if geo_bypass_code is not None:
+    opts.geo_bypass_country, opts.geo_bypass_ip_block = None, None
+    if opts.geo_bypass.lower() not in ('default', 'never'):
         try:
-            GeoUtils.random_ipv4(geo_bypass_code)
+            GeoUtils.random_ipv4(opts.geo_bypass)
         except Exception:
-            raise ValueError('unsupported geo-bypass country or ip-block')
+            raise ValueError(f'Unsupported --xff "{opts.geo_bypass}"')
+        if len(opts.geo_bypass) == 2:
+            opts.geo_bypass_country = opts.geo_bypass
+        else:
+            opts.geo_bypass_ip_block = opts.geo_bypass
+    opts.geo_bypass = opts.geo_bypass.lower() != 'never'
 
     opts.match_filter = match_filter_func(opts.match_filter, opts.breaking_match_filter)
 
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index a2f508552..362a648cd 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -519,22 +519,27 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         '--cn-verification-proxy',
         dest='cn_verification_proxy', default=None, metavar='URL',
         help=optparse.SUPPRESS_HELP)
+    geo.add_option(
+        '--xff', metavar='VALUE',
+        dest='geo_bypass', default="default",
+        help=(
+            'How to fake X-Forwarded-For HTTP header to try bypassing geographic restriction. '
+            'One of "default" (Only when known to be useful), "never", '
+            'a two-letter ISO 3166-2 country code, or an IP block in CIDR notation'))
     geo.add_option(
         '--geo-bypass',
-        action='store_true', dest='geo_bypass', default=True,
-        help='Bypass geographic restriction via faking X-Forwarded-For HTTP header (default)')
+        action='store_const', dest='geo_bypass', const='default',
+        help=optparse.SUPPRESS_HELP)
     geo.add_option(
         '--no-geo-bypass',
-        action='store_false', dest='geo_bypass',
-        help='Do not bypass geographic restriction via faking X-Forwarded-For HTTP header')
+        action='store_const', dest='geo_bypass', const='never',
+        help=optparse.SUPPRESS_HELP)
     geo.add_option(
-        '--geo-bypass-country', metavar='CODE',
-        dest='geo_bypass_country', default=None,
-        help='Force bypass geographic restriction with explicitly provided two-letter ISO 3166-2 country code')
+        '--geo-bypass-country', metavar='CODE', dest='geo_bypass',
+        help=optparse.SUPPRESS_HELP)
     geo.add_option(
-        '--geo-bypass-ip-block', metavar='IP_BLOCK',
-        dest='geo_bypass_ip_block', default=None,
-        help='Force bypass geographic restriction with explicitly provided IP block in CIDR notation')
+        '--geo-bypass-ip-block', metavar='IP_BLOCK', dest='geo_bypass',
+        help=optparse.SUPPRESS_HELP)
 
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(

From 21b5ec86c2c37d10c5bb97edd7051d3aac16bb3e Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Mon, 24 Apr 2023 19:56:35 +0200
Subject: [PATCH 232/871] [utils] `traverse_obj`: Allow iterables in traversal
 (#6902)

Authored by: Grub4K
---
 test/test_utils.py | 4 ++++
 yt_dlp/utils.py    | 7 +++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index d4a301583..f2f3b8170 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2016,6 +2016,8 @@ def test_traverse_obj(self):
                          msg='nested `...` queries should work')
         self.assertCountEqual(traverse_obj(_TEST_DATA, (..., ..., 'index')), range(4),
                               msg='`...` query result should be flattened')
+        self.assertEqual(traverse_obj(range(4), ...), list(range(4)),
+                         msg='`...` should accept iterables')
 
         # Test function as key
         self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)),
@@ -2023,6 +2025,8 @@ def test_traverse_obj(self):
                          msg='function as query key should perform a filter based on (key, value)')
         self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), {'str'},
                               msg='exceptions in the query function should be catched')
+        self.assertEqual(traverse_obj(range(4), lambda _, x: x % 2 == 0), [0, 2],
+                         msg='function key should accept iterables')
         if __debug__:
             with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
                 traverse_obj(_TEST_DATA, lambda a: ...)
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 746a2885d..f69311462 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5528,7 +5528,6 @@ def traverse_obj(
                             If no `default` is given and the last path branches, a `list` of results
                             is always returned. If a path ends on a `dict` that result will always be a `dict`.
     """
-    is_sequence = lambda x: isinstance(x, collections.abc.Sequence) and not isinstance(x, (str, bytes))
     casefold = lambda k: k.casefold() if isinstance(k, str) else k
 
     if isinstance(expected_type, type):
@@ -5564,7 +5563,7 @@ def apply_key(key, obj, is_last):
             branching = True
             if isinstance(obj, collections.abc.Mapping):
                 result = obj.values()
-            elif is_sequence(obj):
+            elif isinstance(obj, collections.abc.Iterable) and not isinstance(obj, (str, bytes)):
                 result = obj
             elif isinstance(obj, re.Match):
                 result = obj.groups()
@@ -5578,7 +5577,7 @@ def apply_key(key, obj, is_last):
             branching = True
             if isinstance(obj, collections.abc.Mapping):
                 iter_obj = obj.items()
-            elif is_sequence(obj):
+            elif isinstance(obj, collections.abc.Iterable) and not isinstance(obj, (str, bytes)):
                 iter_obj = enumerate(obj)
             elif isinstance(obj, re.Match):
                 iter_obj = itertools.chain(
@@ -5614,7 +5613,7 @@ def apply_key(key, obj, is_last):
                 result = next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
 
         elif isinstance(key, (int, slice)):
-            if is_sequence(obj):
+            if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, (str, bytes)):
                 branching = isinstance(key, slice)
                 with contextlib.suppress(IndexError):
                     result = obj[key]

From 9b30cd3dfce83c2f0201b28a7a3ef44ab9722664 Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Mon, 24 Apr 2023 13:16:22 -0600
Subject: [PATCH 233/871] [extractors/rtvc] Add extractors (#6578)

* Add `RTVCPlay` extractor
* Add `RTVCPlayEmbed` extractor
* Add `RTVCKaltura` extractor
* Add `SenalColombiaLive` extractor

Closes #6457
Authored by: elyse0
---
 yt_dlp/extractor/_extractors.py   |   6 +
 yt_dlp/extractor/rtvcplay.py      | 285 ++++++++++++++++++++++++++++++
 yt_dlp/extractor/senalcolombia.py |  31 ++++
 3 files changed, 322 insertions(+)
 create mode 100644 yt_dlp/extractor/rtvcplay.py
 create mode 100644 yt_dlp/extractor/senalcolombia.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 750708d77..b82f52bca 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1621,6 +1621,11 @@
 from .rtp import RTPIE
 from .rtrfm import RTRFMIE
 from .rts import RTSIE
+from .rtvcplay import (
+    RTVCPlayIE,
+    RTVCPlayEmbedIE,
+    RTVCKalturaIE,
+)
 from .rtve import (
     RTVEALaCartaIE,
     RTVEAudioIE,
@@ -1690,6 +1695,7 @@
 )
 from .scrolller import ScrolllerIE
 from .seeker import SeekerIE
+from .senalcolombia import SenalColombiaLiveIE
 from .senategov import SenateISVPIE, SenateGovIE
 from .sendtonews import SendtoNewsIE
 from .servus import ServusIE
diff --git a/yt_dlp/extractor/rtvcplay.py b/yt_dlp/extractor/rtvcplay.py
new file mode 100644
index 000000000..741c47262
--- /dev/null
+++ b/yt_dlp/extractor/rtvcplay.py
@@ -0,0 +1,285 @@
+import re
+
+from .common import InfoExtractor, ExtractorError
+from ..utils import (
+    clean_html,
+    determine_ext,
+    int_or_none,
+    float_or_none,
+    js_to_json,
+    mimetype2ext,
+    traverse_obj,
+    urljoin,
+    url_or_none,
+)
+
+
+class RTVCPlayBaseIE(InfoExtractor):
+    _BASE_VALID_URL = r'https?://(?:www\.)?rtvcplay\.co'
+
+    def _extract_player_config(self, webpage, video_id):
+        return self._search_json(
+            r'<script\b[^>]*>[^<]*(?:var|let|const)\s+config\s*=', re.sub(r'"\s*\+\s*"', '', webpage),
+            'player_config', video_id, transform_source=js_to_json)
+
+    def _extract_formats_and_subtitles_player_config(self, player_config, video_id):
+        formats, subtitles = [], {}
+        for source in traverse_obj(player_config, ('sources', ..., lambda _, v: url_or_none(v['url']))):
+            ext = mimetype2ext(source.get('mimetype'), default=determine_ext(source['url']))
+            if ext == 'm3u8':
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    source['url'], video_id, 'mp4', fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            else:
+                formats.append({
+                    'url': source['url'],
+                    'ext': ext,
+                })
+
+        return formats, subtitles
+
+
+class RTVCPlayIE(RTVCPlayBaseIE):
+    _VALID_URL = RTVCPlayBaseIE._BASE_VALID_URL + r'/(?P<category>(?!embed)[^/]+)/(?:[^?#]+/)?(?P<id>[\w-]+)'
+
+    _TESTS = [{
+        'url': 'https://www.rtvcplay.co/en-vivo/canal-institucional',
+        'info_dict': {
+            'id': 'canal-institucional',
+            'title': r're:^Canal Institucional',
+            'description': 'md5:eff9e548394175928059320c006031ea',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'live_status': 'is_live',
+            'ext': 'mp4',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }, {
+        'url': 'https://www.rtvcplay.co/en-vivo/senal-colombia',
+        'info_dict': {
+            'id': 'senal-colombia',
+            'title': r're:^Señal Colombia',
+            'description': 'md5:799f16a401d97f40c33a2c6a3e2a507b',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'live_status': 'is_live',
+            'ext': 'mp4',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }, {
+        'url': 'https://www.rtvcplay.co/en-vivo/radio-nacional',
+        'info_dict': {
+            'id': 'radio-nacional',
+            'title': r're:^Radio Nacional',
+            'description': 'md5:5de009bc6a9fa79d2a6cf0b73f977d53',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'live_status': 'is_live',
+            'ext': 'mp4',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }, {
+        'url': 'https://www.rtvcplay.co/peliculas-ficcion/senoritas',
+        'md5': '1288ee6f6d1330d880f98bff2ed710a3',
+        'info_dict': {
+            'id': 'senoritas',
+            'title': 'Señoritas',
+            'description': 'md5:f095a2bb52cb6cf279daf6302f86fb32',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'ext': 'mp4',
+        },
+    }, {
+        'url': 'https://www.rtvcplay.co/competencias-basicas-ciudadanas-y-socioemocionales/profe-en-tu-casa/james-regresa-clases-28022022',
+        'md5': 'f040a7380a269ad633cf837384d5e9fc',
+        'info_dict': {
+            'id': 'james-regresa-clases-28022022',
+            'title': 'James regresa a clases - 28/02/2022',
+            'description': 'md5:c5dcdf757c7ab29305e8763c6007e675',
+            'ext': 'mp4',
+        },
+    }, {
+        'url': 'https://www.rtvcplay.co/peliculas-documentales/llinas-el-cerebro-y-el-universo',
+        'info_dict': {
+            'id': 'llinas-el-cerebro-y-el-universo',
+            'title': 'Llinás, el cerebro y el universo',
+            'description': 'md5:add875bf2309bb52b3e8b9b06116d9b0',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'playlist_mincount': 3,
+    }, {
+        'url': 'https://www.rtvcplay.co/competencias-basicas-ciudadanas-y-socioemocionales/profe-en-tu-casa',
+        'info_dict': {
+            'id': 'profe-en-tu-casa',
+            'title': 'Profe en tu casa',
+            'description': 'md5:47dbe20e263194413b1db2a2805a4f2e',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'playlist_mincount': 537,
+    }, {
+        'url': 'https://www.rtvcplay.co/series-al-oido/relato-de-un-naufrago-una-travesia-del-periodismo-a-la-literatura',
+        'info_dict': {
+            'id': 'relato-de-un-naufrago-una-travesia-del-periodismo-a-la-literatura',
+            'title': 'Relato de un náufrago: una travesía del periodismo a la literatura',
+            'description': 'md5:6da28fdca4a5a568ea47ef65ef775603',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'playlist_mincount': 5,
+    }, {
+        'url': 'https://www.rtvcplay.co/series-al-oido/diez-versiones',
+        'info_dict': {
+            'id': 'diez-versiones',
+            'title': 'Diez versiones',
+            'description': 'md5:997471ed971cb3fd8e41969457675306',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'playlist_mincount': 20,
+    }]
+
+    def _real_extract(self, url):
+        video_id, category = self._match_valid_url(url).group('id', 'category')
+        webpage = self._download_webpage(url, video_id)
+
+        hydration = self._search_json(
+            r'window\.__RTVCPLAY_STATE__\s*=', webpage, 'hydration',
+            video_id, transform_source=js_to_json)['content']['currentContent']
+
+        asset_id = traverse_obj(hydration, ('video', 'assetid'))
+        if asset_id:
+            hls_url = hydration['base_url_hls'].replace('[node:field_asset_id]', asset_id)
+        else:
+            hls_url = traverse_obj(hydration, ('channel', 'hls'))
+
+        metadata = traverse_obj(hydration, {
+            'title': 'title',
+            'description': 'description',
+            'thumbnail': ((('channel', 'image', 'logo'), ('resource', 'image', 'cover_desktop')), 'path'),
+        }, get_all=False)
+
+        # Probably it's a program's page
+        if not hls_url:
+            seasons = traverse_obj(
+                hydration, ('widgets', lambda _, y: y['type'] == 'seasonList', 'contents'),
+                get_all=False)
+            if not seasons:
+                podcast_episodes = hydration.get('audios')
+                if not podcast_episodes:
+                    raise ExtractorError('Could not find asset_id nor program playlist nor podcast episodes')
+
+                return self.playlist_result([
+                    self.url_result(episode['file'], url_transparent=True, **traverse_obj(episode, {
+                        'title': 'title',
+                        'description': ('description', {clean_html}),
+                        'episode_number': ('chapter_number', {float_or_none}, {int_or_none}),
+                        'season_number': ('season', {int_or_none}),
+                    })) for episode in podcast_episodes], video_id, **metadata)
+
+            entries = [self.url_result(
+                urljoin(url, episode['slug']), url_transparent=True,
+                **traverse_obj(season, {
+                    'season': 'title',
+                    'season_number': ('season', {int_or_none}),
+                }), **traverse_obj(episode, {
+                    'title': 'title',
+                    'thumbnail': ('image', 'cover', 'path'),
+                    'episode_number': ('chapter_number', {int_or_none}),
+                })) for season in seasons for episode in traverse_obj(season, ('contents', ...))]
+
+            return self.playlist_result(entries, video_id, **metadata)
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(hls_url, video_id, 'mp4')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': category == 'en-vivo',
+            **metadata,
+        }
+
+
+class RTVCPlayEmbedIE(RTVCPlayBaseIE):
+    _VALID_URL = RTVCPlayBaseIE._BASE_VALID_URL + r'/embed/(?P<id>[\w-]+)'
+
+    _TESTS = [{
+        'url': 'https://www.rtvcplay.co/embed/72b0e699-248b-4929-a4a8-3782702fa7f9',
+        'md5': 'ed529aeaee7aa2a72afe91ac7d1177a8',
+        'info_dict': {
+            'id': '72b0e699-248b-4929-a4a8-3782702fa7f9',
+            'title': 'Tráiler: Señoritas',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'ext': 'mp4',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        player_config = self._extract_player_config(webpage, video_id)
+        formats, subtitles = self._extract_formats_and_subtitles_player_config(player_config, video_id)
+
+        asset_id = traverse_obj(player_config, ('rtvcplay', 'assetid'))
+        metadata = {} if not asset_id else self._download_json(
+            f'https://cms.rtvcplay.co/api/v1/video/asset-id/{asset_id}', video_id, fatal=False)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(metadata, {
+                'title': 'title',
+                'description': 'description',
+                'thumbnail': ('image', ..., 'thumbnail', 'path'),
+            }, get_all=False)
+        }
+
+
+class RTVCKalturaIE(RTVCPlayBaseIE):
+    _VALID_URL = r'https?://media\.rtvc\.gov\.co/kalturartvc/(?P<id>[\w-]+)'
+
+    _TESTS = [{
+        'url': 'https://media.rtvc.gov.co/kalturartvc/indexSC.html',
+        'info_dict': {
+            'id': 'indexSC',
+            'title': r're:^Señal Colombia',
+            'description': 'md5:799f16a401d97f40c33a2c6a3e2a507b',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'live_status': 'is_live',
+            'ext': 'mp4',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        player_config = self._extract_player_config(webpage, video_id)
+        formats, subtitles = self._extract_formats_and_subtitles_player_config(player_config, video_id)
+
+        channel_id = traverse_obj(player_config, ('rtvcplay', 'channelId'))
+        metadata = {} if not channel_id else self._download_json(
+            f'https://cms.rtvcplay.co/api/v1/taxonomy_term/streaming/{channel_id}', video_id, fatal=False)
+
+        fmts, subs = self._extract_m3u8_formats_and_subtitles(
+            traverse_obj(metadata, ('channel', 'hls')), video_id, 'mp4', fatal=False)
+        formats.extend(fmts)
+        self._merge_subtitles(subs, target=subtitles)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': True,
+            **traverse_obj(metadata, {
+                'title': 'title',
+                'description': 'description',
+                'thumbnail': ('channel', 'image', 'logo', 'path'),
+            })
+        }
diff --git a/yt_dlp/extractor/senalcolombia.py b/yt_dlp/extractor/senalcolombia.py
new file mode 100644
index 000000000..f3c066da7
--- /dev/null
+++ b/yt_dlp/extractor/senalcolombia.py
@@ -0,0 +1,31 @@
+from .common import InfoExtractor
+from .rtvcplay import RTVCKalturaIE
+
+
+class SenalColombiaLiveIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?senalcolombia\.tv/(?P<id>senal-en-vivo)'
+
+    _TESTS = [{
+        'url': 'https://www.senalcolombia.tv/senal-en-vivo',
+        'info_dict': {
+            'id': 'indexSC',
+            'title': 're:^Señal Colombia',
+            'description': 'md5:799f16a401d97f40c33a2c6a3e2a507b',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'live_status': 'is_live',
+            'ext': 'mp4',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        hydration = self._search_json(
+            r'<script\b[^>]*data-drupal-selector\s*=\s*"[^"]*drupal-settings-json[^"]*"[^>]*>',
+            webpage, 'hydration', display_id)
+
+        return self.url_result(hydration['envivosrc'], RTVCKalturaIE, display_id)

From c86e433c35fe5da6cb29f3539eef97497f84ed38 Mon Sep 17 00:00:00 2001
From: sqrtNOT <77981959+sqrtNOT@users.noreply.github.com>
Date: Tue, 25 Apr 2023 10:21:06 +0000
Subject: [PATCH 234/871] [extractor/NiconicoSeries] Fix extraction (#6898)

Authored by: sqrtNOT
---
 yt_dlp/extractor/niconico.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py
index cacefeb42..30b4d7216 100644
--- a/yt_dlp/extractor/niconico.py
+++ b/yt_dlp/extractor/niconico.py
@@ -660,10 +660,10 @@ def _real_extract(self, url):
 
 class NiconicoSeriesIE(InfoExtractor):
     IE_NAME = 'niconico:series'
-    _VALID_URL = r'https?://(?:(?:www\.|sp\.)?nicovideo\.jp|nico\.ms)/series/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.|sp\.)?nicovideo\.jp(?:/user/\d+)?|nico\.ms)/series/(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'https://www.nicovideo.jp/series/110226',
+        'url': 'https://www.nicovideo.jp/user/44113208/series/110226',
         'info_dict': {
             'id': '110226',
             'title': 'ご立派ァ！のシリーズ',
@@ -683,7 +683,7 @@ class NiconicoSeriesIE(InfoExtractor):
 
     def _real_extract(self, url):
         list_id = self._match_id(url)
-        webpage = self._download_webpage(f'https://www.nicovideo.jp/series/{list_id}', list_id)
+        webpage = self._download_webpage(url, list_id)
 
         title = self._search_regex(
             (r'<title>「(.+)（全',
@@ -691,10 +691,9 @@ def _real_extract(self, url):
             webpage, 'title', fatal=False)
         if title:
             title = unescapeHTML(title)
-        playlist = [
-            self.url_result(f'https://www.nicovideo.jp/watch/{v_id}', video_id=v_id)
-            for v_id in re.findall(r'data-href=[\'"](?:https://www\.nicovideo\.jp)?/watch/([a-z0-9]+)', webpage)]
-        return self.playlist_result(playlist, list_id, title)
+        json_data = next(self._yield_json_ld(webpage, None, fatal=False))
+        return self.playlist_from_matches(
+            traverse_obj(json_data, ('itemListElement', ..., 'url')), list_id, title, ie=NiconicoIE)
 
 
 class NiconicoHistoryIE(NiconicoPlaylistBaseIE):

From 0c4e0fbcade0fc92d14c2a6d63e360fe067f6192 Mon Sep 17 00:00:00 2001
From: Neurognostic <donovan@tremura.email>
Date: Tue, 25 Apr 2023 12:13:54 -0400
Subject: [PATCH 235/871] [extractor/bitchute] Add more fallback subdomains
 (#6907)

Authored by: Neurognostic
---
 yt_dlp/extractor/bitchute.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/bitchute.py b/yt_dlp/extractor/bitchute.py
index 10e7b0b2b..a6779505e 100644
--- a/yt_dlp/extractor/bitchute.py
+++ b/yt_dlp/extractor/bitchute.py
@@ -77,7 +77,10 @@ class BitChuteIE(InfoExtractor):
     def _check_format(self, video_url, video_id):
         urls = orderedSet(
             re.sub(r'(^https?://)(seed\d+)(?=\.bitchute\.com)', fr'\g<1>{host}', video_url)
-            for host in (r'\g<2>', 'seed150', 'seed151', 'seed152', 'seed153'))
+            for host in (r'\g<2>', 'seed122', 'seed125', 'seed126', 'seed128',
+                         'seed132', 'seed150', 'seed151', 'seed152', 'seed153',
+                         'seed167', 'seed171', 'seed177', 'seed305', 'seed307',
+                         'seedp29xb', 'zb10-7gsop1v78'))
         for url in urls:
             try:
                 response = self._request_webpage(

From 62beefa818c75c20b6941389bb197051554a5d41 Mon Sep 17 00:00:00 2001
From: Noah <nkempers@outlook.de>
Date: Tue, 25 Apr 2023 22:46:14 +0200
Subject: [PATCH 236/871] [extractor/pornhub] Set access cookies to fix
 extraction (#6685)

Closes #4299
Authored by: Schmoaaaaah, arobase-che

Co-authored-by: Noah <nkempers@outlook.de>
Co-authored-by: ache <ache@ache.one>
---
 yt_dlp/extractor/pornhub.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/pornhub.py b/yt_dlp/extractor/pornhub.py
index 5d8d7c100..2f5a572a5 100644
--- a/yt_dlp/extractor/pornhub.py
+++ b/yt_dlp/extractor/pornhub.py
@@ -58,6 +58,11 @@ def dl(*args, **kwargs):
     def _real_initialize(self):
         self._logged_in = False
 
+    def _set_age_cookies(self, host):
+        self._set_cookie(host, 'age_verified', '1')
+        self._set_cookie(host, 'accessAgeDisclaimerPH', '1')
+        self._set_cookie(host, 'accessPH', '1')
+
     def _login(self, host):
         if self._logged_in:
             return
@@ -267,8 +272,7 @@ def _real_extract(self, url):
         video_id = mobj.group('id')
 
         self._login(host)
-
-        self._set_cookie(host, 'age_verified', '1')
+        self._set_age_cookies(host)
 
         def dl_webpage(platform):
             self._set_cookie(host, 'platform', platform)
@@ -569,6 +573,7 @@ def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         user_id = mobj.group('id')
         videos_url = '%s/videos' % mobj.group('url')
+        self._set_age_cookies(mobj.group('host'))
         page = self._extract_page(url)
         if page:
             videos_url = update_url_query(videos_url, {'page': page})
@@ -633,6 +638,7 @@ def _real_extract(self, url):
         item_id = mobj.group('id')
 
         self._login(host)
+        self._set_age_cookies(host)
 
         return self.playlist_result(self._entries(url, host, item_id), item_id)
 
@@ -812,5 +818,6 @@ def _real_extract(self, url):
         item_id = mobj.group('id')
 
         self._login(host)
+        self._set_age_cookies(host)
 
         return self.playlist_result(self._entries(mobj.group('url'), host, item_id), item_id)

From ed81b74802b4247ee8d9dc0ef87eb52baefede1c Mon Sep 17 00:00:00 2001
From: Alex Klapheke <alexklapheke@gmail.com>
Date: Wed, 26 Apr 2023 02:53:07 -0400
Subject: [PATCH 237/871] [extractor/aeonco] Support Youtube embeds (#6591)

Authored by: alexklapheke
---
 yt_dlp/extractor/aeonco.py | 52 +++++++++++++++++++++++++++++++-------
 1 file changed, 43 insertions(+), 9 deletions(-)

diff --git a/yt_dlp/extractor/aeonco.py b/yt_dlp/extractor/aeonco.py
index 4655862e3..390eae32b 100644
--- a/yt_dlp/extractor/aeonco.py
+++ b/yt_dlp/extractor/aeonco.py
@@ -1,5 +1,6 @@
 from .common import InfoExtractor
 from .vimeo import VimeoIE
+from ..utils import ExtractorError, traverse_obj, url_or_none
 
 
 class AeonCoIE(InfoExtractor):
@@ -19,22 +20,55 @@ class AeonCoIE(InfoExtractor):
         }
     }, {
         'url': 'https://aeon.co/videos/dazzling-timelapse-shows-how-microbes-spoil-our-food-and-sometimes-enrich-it',
-        'md5': '4e5f3dad9dbda0dbfa2da41a851e631e',
+        'md5': '03582d795382e49f2fd0b427b55de409',
         'info_dict': {
-            'id': '728595228',
+            'id': '759576926',
             'ext': 'mp4',
             'title': 'Wrought',
-            'thumbnail': 'https://i.vimeocdn.com/video/1484618528-c91452611f9a4e4497735a533da60d45b2fe472deb0c880f0afaab0cd2efb22a-d_1280',
-            'uploader': 'Biofilm Productions',
-            'uploader_id': 'user140352216',
-            'uploader_url': 'https://vimeo.com/user140352216',
+            'thumbnail': 'https://i.vimeocdn.com/video/1525599692-84614af88e446612f49ca966cf8f80eab2c73376bedd80555741c521c26f9a3e-d_1280',
+            'uploader': 'Aeon Video',
+            'uploader_id': 'aeonvideo',
+            'uploader_url': 'https://vimeo.com/aeonvideo',
             'duration': 1344
         }
+    }, {
+        'url': 'https://aeon.co/videos/chew-over-the-prisoners-dilemma-and-see-if-you-can-find-the-rational-path-out',
+        'md5': '1cfda0bf3ae24df17d00f2c0cb6cc21b',
+        'info_dict': {
+            'id': 'emyi4z-O0ls',
+            'ext': 'mp4',
+            'title': 'How to outsmart the Prisoner’s Dilemma - Lucas Husted',
+            'thumbnail': 'https://i.ytimg.com/vi_webp/emyi4z-O0ls/maxresdefault.webp',
+            'uploader': 'TED-Ed',
+            'uploader_id': '@TEDEd',
+            'uploader_url': 'https://www.youtube.com/@TEDEd',
+            'duration': 344,
+            'upload_date': '20200827',
+            'channel_id': 'UCsooa4yRKGN_zEE8iknghZA',
+            'playable_in_embed': True,
+            'description': 'md5:c0959524f08cb60f96fd010f3dfb17f3',
+            'categories': ['Education'],
+            'like_count': int,
+            'channel': 'TED-Ed',
+            'chapters': 'count:7',
+            'channel_url': 'https://www.youtube.com/channel/UCsooa4yRKGN_zEE8iknghZA',
+            'tags': 'count:26',
+            'availability': 'public',
+            'channel_follower_count': int,
+            'view_count': int,
+            'age_limit': 0,
+            'live_status': 'not_live',
+            'comment_count': int,
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        vimeo_id = self._search_regex(r'hosterId":\s*"(?P<id>[0-9]+)', webpage, 'vimeo id')
-        vimeo_url = VimeoIE._smuggle_referrer(f'https://player.vimeo.com/video/{vimeo_id}', 'https://aeon.co')
-        return self.url_result(vimeo_url, VimeoIE)
+        embed_url = traverse_obj(self._yield_json_ld(webpage, video_id), (
+            lambda _, v: v['@type'] == 'VideoObject', 'embedUrl', {url_or_none}), get_all=False)
+        if not embed_url:
+            raise ExtractorError('No embed URL found in webpage')
+        if 'player.vimeo.com' in embed_url:
+            embed_url = VimeoIE._smuggle_referrer(embed_url, 'https://aeon.co/')
+        return self.url_result(embed_url)

From 30647668a92a0ca5cd108776804baac0996bd9f7 Mon Sep 17 00:00:00 2001
From: garret <76261416+garret1317@users.noreply.github.com>
Date: Thu, 27 Apr 2023 00:42:07 +0100
Subject: [PATCH 238/871] [extractor/globalplayer] Add extractors (#6903)

Authored by: garret1317
---
 yt_dlp/extractor/_extractors.py  |   7 +
 yt_dlp/extractor/globalplayer.py | 254 +++++++++++++++++++++++++++++++
 2 files changed, 261 insertions(+)
 create mode 100755 yt_dlp/extractor/globalplayer.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b82f52bca..3b5ae63b1 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -685,6 +685,13 @@
 from .giantbomb import GiantBombIE
 from .giga import GigaIE
 from .glide import GlideIE
+from .globalplayer import (
+    GlobalPlayerLiveIE,
+    GlobalPlayerLivePlaylistIE,
+    GlobalPlayerAudioIE,
+    GlobalPlayerAudioEpisodeIE,
+    GlobalPlayerVideoIE
+)
 from .globo import (
     GloboIE,
     GloboArticleIE,
diff --git a/yt_dlp/extractor/globalplayer.py b/yt_dlp/extractor/globalplayer.py
new file mode 100755
index 000000000..e0c0d58fd
--- /dev/null
+++ b/yt_dlp/extractor/globalplayer.py
@@ -0,0 +1,254 @@
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    join_nonempty,
+    parse_duration,
+    str_or_none,
+    traverse_obj,
+    unified_strdate,
+    unified_timestamp,
+    urlhandle_detect_ext,
+)
+
+
+class GlobalPlayerBaseIE(InfoExtractor):
+    def _get_page_props(self, url, video_id):
+        webpage = self._download_webpage(url, video_id)
+        return self._search_nextjs_data(webpage, video_id)['props']['pageProps']
+
+    def _request_ext(self, url, video_id):
+        return urlhandle_detect_ext(self._request_webpage(  # Server rejects HEAD requests
+            url, video_id, note='Determining source extension'))
+
+    def _extract_audio(self, episode, series):
+        return {
+            'vcodec': 'none',
+            **traverse_obj(series, {
+                'series': 'title',
+                'series_id': 'id',
+                'thumbnail': 'imageUrl',
+                'uploader': 'itunesAuthor',  # podcasts only
+            }),
+            **traverse_obj(episode, {
+                'id': 'id',
+                'description': ('description', {clean_html}),
+                'duration': ('duration', {parse_duration}),
+                'thumbnail': 'imageUrl',
+                'url': 'streamUrl',
+                'timestamp': (('pubDate', 'startDate'), {unified_timestamp}),
+                'title': 'title',
+            }, get_all=False)
+        }
+
+
+class GlobalPlayerLiveIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/live/(?P<id>\w+)/\w+'
+    _TESTS = [{
+        'url': 'https://www.globalplayer.com/live/smoothchill/uk/',
+        'info_dict': {
+            'id': '2mx1E',
+            'ext': 'aac',
+            'display_id': 'smoothchill-uk',
+            'title': 're:^Smooth Chill.+$',
+            'thumbnail': 'https://herald.musicradio.com/media/f296ade8-50c9-4f60-911f-924e96873620.png',
+            'description': 'Music To Chill To',
+            'live_status': 'is_live',
+        },
+    }, {
+        # national station
+        'url': 'https://www.globalplayer.com/live/heart/uk/',
+        'info_dict': {
+            'id': '2mwx4',
+            'ext': 'aac',
+            'description': 'turn up the feel good!',
+            'thumbnail': 'https://herald.musicradio.com/media/49b9e8cb-15bf-4bf2-8c28-a4850cc6b0f3.png',
+            'live_status': 'is_live',
+            'title': 're:^Heart UK.+$',
+            'display_id': 'heart-uk',
+        },
+    }, {
+        # regional variation
+        'url': 'https://www.globalplayer.com/live/heart/london/',
+        'info_dict': {
+            'id': 'AMqg',
+            'ext': 'aac',
+            'thumbnail': 'https://herald.musicradio.com/media/49b9e8cb-15bf-4bf2-8c28-a4850cc6b0f3.png',
+            'title': 're:^Heart London.+$',
+            'live_status': 'is_live',
+            'display_id': 'heart-london',
+            'description': 'turn up the feel good!',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        station = self._get_page_props(url, video_id)['station']
+        stream_url = station['streamUrl']
+
+        return {
+            'id': station['id'],
+            'display_id': join_nonempty('brandSlug', 'slug', from_dict=station) or station.get('legacyStationPrefix'),
+            'url': stream_url,
+            'ext': self._request_ext(stream_url, video_id),
+            'vcodec': 'none',
+            'is_live': True,
+            **traverse_obj(station, {
+                'title': (('name', 'brandName'), {str_or_none}),
+                'description': 'tagline',
+                'thumbnail': 'brandLogo',
+            }, get_all=False),
+        }
+
+
+class GlobalPlayerLivePlaylistIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/playlists/(?P<id>\w+)'
+    _TESTS = [{
+        # "live playlist"
+        'url': 'https://www.globalplayer.com/playlists/8bLk/',
+        'info_dict': {
+            'id': '8bLk',
+            'ext': 'aac',
+            'live_status': 'is_live',
+            'description': 'md5:e10f5e10b01a7f2c14ba815509fbb38d',
+            'thumbnail': 'https://images.globalplayer.com/images/551379?width=450&signature=oMLPZIoi5_dBSHnTMREW0Xg76mA=',
+            'title': 're:^Classic FM Hall of Fame.+$'
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        station = self._get_page_props(url, video_id)['playlistData']
+        stream_url = station['streamUrl']
+
+        return {
+            'id': video_id,
+            'url': stream_url,
+            'ext': self._request_ext(stream_url, video_id),
+            'vcodec': 'none',
+            'is_live': True,
+            **traverse_obj(station, {
+                'title': 'title',
+                'description': 'description',
+                'thumbnail': 'image',
+            }),
+        }
+
+
+class GlobalPlayerAudioIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/(?:(?P<podcast>podcasts)/|catchup/\w+/\w+/)(?P<id>\w+)/?(?:$|[?#])'
+    _TESTS = [{
+        # podcast
+        'url': 'https://www.globalplayer.com/podcasts/42KuaM/',
+        'playlist_mincount': 5,
+        'info_dict': {
+            'id': '42KuaM',
+            'title': 'Filthy Ritual',
+            'thumbnail': 'md5:60286e7d12d795bd1bbc9efc6cee643e',
+            'categories': ['Society & Culture', 'True Crime'],
+            'uploader': 'Global',
+            'description': 'md5:da5b918eac9ae319454a10a563afacf9',
+        },
+    }, {
+        # radio catchup
+        'url': 'https://www.globalplayer.com/catchup/lbc/uk/46vyD7z/',
+        'playlist_mincount': 3,
+        'info_dict': {
+            'id': '46vyD7z',
+            'description': 'Nick Ferrari At Breakfast is Leading Britain\'s Conversation.',
+            'title': 'Nick Ferrari',
+            'thumbnail': 'md5:4df24d8a226f5b2508efbcc6ae874ebf',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id, podcast = self._match_valid_url(url).group('id', 'podcast')
+        props = self._get_page_props(url, video_id)
+        series = props['podcastInfo'] if podcast else props['catchupInfo']
+
+        return {
+            '_type': 'playlist',
+            'id': video_id,
+            'entries': [self._extract_audio(ep, series) for ep in traverse_obj(
+                        series, ('episodes', lambda _, v: v['id'] and v['streamUrl']))],
+            'categories': traverse_obj(series, ('categories', ..., 'name')) or None,
+            **traverse_obj(series, {
+                'description': 'description',
+                'thumbnail': 'imageUrl',
+                'title': 'title',
+                'uploader': 'itunesAuthor',  # podcasts only
+            }),
+        }
+
+
+class GlobalPlayerAudioEpisodeIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/(?:(?P<podcast>podcasts)|catchup/\w+/\w+)/episodes/(?P<id>\w+)/?(?:$|[?#])'
+    _TESTS = [{
+        # podcast
+        'url': 'https://www.globalplayer.com/podcasts/episodes/7DrfNnE/',
+        'info_dict': {
+            'id': '7DrfNnE',
+            'ext': 'mp3',
+            'title': 'Filthy Ritual - Trailer',
+            'description': 'md5:1f1562fd0f01b4773b590984f94223e0',
+            'thumbnail': 'md5:60286e7d12d795bd1bbc9efc6cee643e',
+            'duration': 225.0,
+            'timestamp': 1681254900,
+            'series': 'Filthy Ritual',
+            'series_id': '42KuaM',
+            'upload_date': '20230411',
+            'uploader': 'Global',
+        },
+    }, {
+        # radio catchup
+        'url': 'https://www.globalplayer.com/catchup/lbc/uk/episodes/2zGq26Vcv1fCWhddC4JAwETXWe/',
+        'info_dict': {
+            'id': '2zGq26Vcv1fCWhddC4JAwETXWe',
+            'ext': 'm4a',
+            'timestamp': 1682056800,
+            'series': 'Nick Ferrari',
+            'thumbnail': 'md5:4df24d8a226f5b2508efbcc6ae874ebf',
+            'upload_date': '20230421',
+            'series_id': '46vyD7z',
+            'description': 'Nick Ferrari At Breakfast is Leading Britain\'s Conversation.',
+            'title': 'Nick Ferrari',
+            'duration': 10800.0,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id, podcast = self._match_valid_url(url).group('id', 'podcast')
+        props = self._get_page_props(url, video_id)
+        episode = props['podcastEpisode'] if podcast else props['catchupEpisode']
+
+        return self._extract_audio(
+            episode, traverse_obj(episode, 'podcast', 'show', expected_type=dict) or {})
+
+
+class GlobalPlayerVideoIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/videos/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://www.globalplayer.com/videos/2JsSZ7Gm2uP/',
+        'info_dict': {
+            'id': '2JsSZ7Gm2uP',
+            'ext': 'mp4',
+            'description': 'md5:6a9f063c67c42f218e42eee7d0298bfd',
+            'thumbnail': 'md5:d4498af48e15aae4839ce77b97d39550',
+            'upload_date': '20230420',
+            'title': 'Treble Malakai Bayoh sings a sublime Handel aria at Classic FM Live',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        meta = self._get_page_props(url, video_id)['videoData']
+
+        return {
+            'id': video_id,
+            **traverse_obj(meta, {
+                'url': 'url',
+                'thumbnail': ('image', 'url'),
+                'title': 'title',
+                'upload_date': ('publish_date', {unified_strdate}),
+                'description': 'description',
+            }),
+        }

From 170605840ea9d5ad75da6576485ea7d125b428ee Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 27 Apr 2023 05:52:22 +0530
Subject: [PATCH 239/871] Populate `filename` and `urls` fields at all stages
 of `--print`

Closes https://github.com/yt-dlp/yt-dlp/issues/6920
---
 yt_dlp/YoutubeDL.py | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index dce6cf928..482b1a49e 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1677,7 +1677,7 @@ def process_ie_result(self, ie_result, download=True, extra_info=None):
                 self.add_extra_info(info_copy, extra_info)
                 info_copy, _ = self.pre_process(info_copy)
                 self._fill_common_fields(info_copy, False)
-                self.__forced_printings(info_copy, self.prepare_filename(info_copy), incomplete=True)
+                self.__forced_printings(info_copy)
                 self._raise_pending_errors(info_copy)
                 if self.params.get('force_write_download_archive', False):
                     self.record_download_archive(info_copy)
@@ -2719,7 +2719,7 @@ def is_wellformed(f):
             self.list_formats(info_dict)
         if list_only:
             # Without this printing, -F --print-json will not work
-            self.__forced_printings(info_dict, self.prepare_filename(info_dict), incomplete=True)
+            self.__forced_printings(info_dict)
             return info_dict
 
         format_selector = self.format_selector
@@ -2879,6 +2879,12 @@ def _forceprint(self, key, info_dict):
         if info_dict is None:
             return
         info_copy = info_dict.copy()
+        info_copy.setdefault('filename', self.prepare_filename(info_dict))
+        if info_dict.get('requested_formats') is not None:
+            # For RTMP URLs, also include the playpath
+            info_copy['urls'] = '\n'.join(f['url'] + f.get('play_path', '') for f in info_dict['requested_formats'])
+        elif info_dict.get('url'):
+            info_copy['urls'] = info_dict['url'] + info_dict.get('play_path', '')
         info_copy['formats_table'] = self.render_formats_table(info_dict)
         info_copy['thumbnails_table'] = self.render_thumbnails_table(info_dict)
         info_copy['subtitles_table'] = self.render_subtitles_table(info_dict.get('id'), info_dict.get('subtitles'))
@@ -2907,7 +2913,9 @@ def format_tmpl(tmpl):
                 with open(filename, 'a', encoding='utf-8', newline='') as f:
                     f.write(self.evaluate_outtmpl(tmpl, info_copy) + os.linesep)
 
-    def __forced_printings(self, info_dict, filename, incomplete):
+        return info_copy
+
+    def __forced_printings(self, info_dict, filename=None, incomplete=True):
         def print_mandatory(field, actual_field=None):
             if actual_field is None:
                 actual_field = field
@@ -2920,20 +2928,14 @@ def print_optional(field):
                     and info_dict.get(field) is not None):
                 self.to_stdout(info_dict[field])
 
-        info_dict = info_dict.copy()
-        if filename is not None:
-            info_dict['filename'] = filename
-        if info_dict.get('requested_formats') is not None:
-            # For RTMP URLs, also include the playpath
-            info_dict['urls'] = '\n'.join(f['url'] + f.get('play_path', '') for f in info_dict['requested_formats'])
-        elif info_dict.get('url'):
-            info_dict['urls'] = info_dict['url'] + info_dict.get('play_path', '')
-
         if (self.params.get('forcejson')
                 or self.params['forceprint'].get('video')
                 or self.params['print_to_file'].get('video')):
             self.post_extract(info_dict)
-        self._forceprint('video', info_dict)
+
+        if filename:
+            info_dict['filename'] = filename
+        info_dict = self._forceprint('video', info_dict)
 
         print_mandatory('title')
         print_mandatory('id')
@@ -3493,10 +3495,10 @@ def run_pp(self, pp, infodict):
         return infodict
 
     def run_all_pps(self, key, info, *, additional_pps=None):
-        if key != 'video':
-            self._forceprint(key, info)
         for pp in (additional_pps or []) + self._pps[key]:
             info = self.run_pp(pp, info)
+        if key != 'video':
+            self._forceprint(key, info)
         return info
 
     def pre_process(self, ie_info, key='pre_process', files_to_move=None):

From 7cf51f21916292cd80bdeceb37489f5322f166dd Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 27 Apr 2023 07:42:17 +0530
Subject: [PATCH 240/871] [jsinterp] Handle negative numbers better

Closes #6131
---
 test/test_jsinterp.py          | 16 ++++++++++++++++
 test/test_youtube_signature.py |  4 ++++
 yt_dlp/jsinterp.py             |  8 +++++---
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index e090dc791..3283657d7 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -445,6 +445,22 @@ def test_bitwise_operators_overflow(self):
         jsi = JSInterpreter('function x(){return 1236566549 << 5}')
         self.assertEqual(jsi.call_function('x'), 915423904)
 
+    def test_negative(self):
+        jsi = JSInterpreter("function f(){return 2    *    -2.0;}")
+        self.assertEqual(jsi.call_function('f'), -4)
+
+        jsi = JSInterpreter('function f(){return 2    -    - -2;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+
+        jsi = JSInterpreter('function f(){return 2    -    - - -2;}')
+        self.assertEqual(jsi.call_function('f'), 4)
+
+        jsi = JSInterpreter('function f(){return 2    -    + + - -2;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+
+        jsi = JSInterpreter('function f(){return 2    +    - + - -2;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 336e80291..e2b3f0870 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -142,6 +142,10 @@
         'https://www.youtube.com/s/player/dac945fd/player_ias.vflset/en_US/base.js',
         'o8BkRxXhuYsBCWi6RplPdP', '3Lx32v_hmzTm6A',
     ),
+    (
+        'https://www.youtube.com/s/player/6f20102c/player_ias.vflset/en_US/base.js',
+        'lE8DhoDmKqnmJJ', 'pJTTX6XyJP2BYw',
+    ),
 ]
 
 
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index db6526009..5571ecfeb 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -243,7 +243,7 @@ def _separate(expr, delim=',', max_split=None):
             return
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
-        in_quote, escaping, after_op, in_regex_char_group = None, False, True, False
+        in_quote, escaping, after_op, in_regex_char_group, in_unary_op = None, False, True, False, False
         for idx, char in enumerate(expr):
             if not in_quote and char in _MATCHING_PARENS:
                 counters[_MATCHING_PARENS[char]] += 1
@@ -258,9 +258,11 @@ def _separate(expr, delim=',', max_split=None):
                 elif in_quote == '/' and char in '[]':
                     in_regex_char_group = char == '['
             escaping = not escaping and in_quote and char == '\\'
-            after_op = not in_quote and char in OP_CHARS or (char.isspace() and after_op)
+            in_unary_op = (not in_quote and not in_regex_char_group
+                           and after_op not in (True, False) and char in '-+')
+            after_op = char if (not in_quote and char in OP_CHARS) else (char.isspace() and after_op)
 
-            if char != delim[pos] or any(counters.values()) or in_quote:
+            if char != delim[pos] or any(counters.values()) or in_quote or in_unary_op:
                 pos = 0
                 continue
             elif pos != delim_len:

From b5f61b69d4561b81fc98c226b176f0c15493e688 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 27 Apr 2023 19:35:28 +0530
Subject: [PATCH 241/871] Fix bug in 170605840ea9d5ad75da6576485ea7d125b428ee

and related refactor
---
 yt_dlp/YoutubeDL.py | 40 +++++++++++++++++-----------------------
 1 file changed, 17 insertions(+), 23 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 482b1a49e..a8b4a650e 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2916,36 +2916,30 @@ def format_tmpl(tmpl):
         return info_copy
 
     def __forced_printings(self, info_dict, filename=None, incomplete=True):
-        def print_mandatory(field, actual_field=None):
-            if actual_field is None:
-                actual_field = field
-            if (self.params.get('force%s' % field, False)
-                    and (not incomplete or info_dict.get(actual_field) is not None)):
-                self.to_stdout(info_dict[actual_field])
-
-        def print_optional(field):
-            if (self.params.get('force%s' % field, False)
-                    and info_dict.get(field) is not None):
-                self.to_stdout(info_dict[field])
-
         if (self.params.get('forcejson')
                 or self.params['forceprint'].get('video')
                 or self.params['print_to_file'].get('video')):
             self.post_extract(info_dict)
-
         if filename:
             info_dict['filename'] = filename
-        info_dict = self._forceprint('video', info_dict)
+        info_copy = self._forceprint('video', info_dict)
 
-        print_mandatory('title')
-        print_mandatory('id')
-        print_mandatory('url', 'urls')
-        print_optional('thumbnail')
-        print_optional('description')
-        print_optional('filename')
-        if self.params.get('forceduration') and info_dict.get('duration') is not None:
-            self.to_stdout(formatSeconds(info_dict['duration']))
-        print_mandatory('format')
+        def print_field(field, actual_field=None, optional=False):
+            if actual_field is None:
+                actual_field = field
+            if self.params.get(f'force{field}') and (
+                    info_copy.get(field) is not None or (not optional and not incomplete)):
+                self.to_stdout(info_copy[actual_field])
+
+        print_field('title')
+        print_field('id')
+        print_field('url', 'urls')
+        print_field('thumbnail', optional=True)
+        print_field('description', optional=True)
+        print_field('filename', optional=True)
+        if self.params.get('forceduration') and info_copy.get('duration') is not None:
+            self.to_stdout(formatSeconds(info_copy['duration']))
+        print_field('format')
 
         if self.params.get('forcejson'):
             self.to_stdout(json.dumps(self.sanitize_info(info_dict)))

From 7a7b1376fbce0067cf37566bb47131bc0022638d Mon Sep 17 00:00:00 2001
From: makeworld <25111343+makew0rld@users.noreply.github.com>
Date: Thu, 27 Apr 2023 22:42:25 -0400
Subject: [PATCH 242/871] [extractor/cbc] Fix live extractor, playlist
 `_VALID_URL` (#6625)

Authored by: makew0rld
---
 yt_dlp/extractor/cbc.py | 120 +++++++++++++++++++++++++++-------------
 1 file changed, 83 insertions(+), 37 deletions(-)

diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index eadb3f8c0..e42f06246 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -8,14 +8,16 @@
     compat_str,
 )
 from ..utils import (
+    ExtractorError,
     int_or_none,
     join_nonempty,
     js_to_json,
     orderedSet,
+    parse_iso8601,
     smuggle_url,
     strip_or_none,
+    traverse_obj,
     try_get,
-    ExtractorError,
 )
 
 
@@ -404,7 +406,7 @@ def _real_extract(self, url):
 
 class CBCGemPlaylistIE(InfoExtractor):
     IE_NAME = 'gem.cbc.ca:playlist'
-    _VALID_URL = r'https?://gem\.cbc\.ca/media/(?P<id>(?P<show>[0-9a-z-]+)/s(?P<season>[0-9]+))/?(?:[?#]|$)'
+    _VALID_URL = r'https?://gem\.cbc\.ca/(?:media/)?(?P<id>(?P<show>[0-9a-z-]+)/s(?P<season>[0-9]+))/?(?:[?#]|$)'
     _TESTS = [{
         # TV show playlist, all public videos
         'url': 'https://gem.cbc.ca/media/schitts-creek/s06',
@@ -414,6 +416,9 @@ class CBCGemPlaylistIE(InfoExtractor):
             'title': 'Season 6',
             'description': 'md5:6a92104a56cbeb5818cc47884d4326a2',
         },
+    }, {
+        'url': 'https://gem.cbc.ca/schitts-creek/s06',
+        'only_matching': True,
     }]
     _API_BASE = 'https://services.radio-canada.ca/ott/cbc-api/v2/shows/'
 
@@ -473,49 +478,90 @@ def _real_extract(self, url):
 
 class CBCGemLiveIE(InfoExtractor):
     IE_NAME = 'gem.cbc.ca:live'
-    _VALID_URL = r'https?://gem\.cbc\.ca/live/(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://gem.cbc.ca/live/920604739687',
-        'info_dict': {
-            'title': 'Ottawa',
-            'description': 'The live TV channel and local programming from Ottawa',
-            'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/CBC_OTT_VMS/Live_Channel_Static_Images/Ottawa_2880x1620.jpg',
-            'is_live': True,
-            'id': 'AyqZwxRqh8EH',
-            'ext': 'mp4',
-            'timestamp': 1492106160,
-            'upload_date': '20170413',
-            'uploader': 'CBCC-NEW',
+    _VALID_URL = r'https?://gem\.cbc\.ca/live(?:-event)?/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'https://gem.cbc.ca/live/920604739687',
+            'info_dict': {
+                'title': 'Ottawa',
+                'description': 'The live TV channel and local programming from Ottawa',
+                'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/CBC_OTT_VMS/Live_Channel_Static_Images/Ottawa_2880x1620.jpg',
+                'is_live': True,
+                'id': 'AyqZwxRqh8EH',
+                'ext': 'mp4',
+                'timestamp': 1492106160,
+                'upload_date': '20170413',
+                'uploader': 'CBCC-NEW',
+            },
+            'skip': 'Live might have ended',
         },
-        'skip': 'Live might have ended',
-    }
-
-    # It's unclear where the chars at the end come from, but they appear to be
-    # constant. Might need updating in the future.
-    # There are two URLs, some livestreams are in one, and some
-    # in the other. The JSON schema is the same for both.
-    _API_URLS = ['https://tpfeed.cbc.ca/f/ExhSPC/t_t3UKJR6MAT', 'https://tpfeed.cbc.ca/f/ExhSPC/FNiv9xQx_BnT']
+        {
+            'url': 'https://gem.cbc.ca/live/44',
+            'info_dict': {
+                'id': '44',
+                'ext': 'mp4',
+                'is_live': True,
+                'title': r're:^Ottawa [0-9\-: ]+',
+                'description': 'The live TV channel and local programming from Ottawa',
+                'live_status': 'is_live',
+                'thumbnail': r're:https://images.gem.cbc.ca/v1/cbc-gem/live/.*'
+            },
+            'params': {'skip_download': True},
+            'skip': 'Live might have ended',
+        },
+        {
+            'url': 'https://gem.cbc.ca/live-event/10835',
+            'info_dict': {
+                'id': '10835',
+                'ext': 'mp4',
+                'is_live': True,
+                'title': r're:^The National \| Biden’s trip wraps up, Paltrow testifies, Bird flu [0-9\-: ]+',
+                'description': 'March 24, 2023 | President Biden’s Ottawa visit ends with big pledges from both countries. Plus, Gwyneth Paltrow testifies in her ski collision trial.',
+                'live_status': 'is_live',
+                'thumbnail': r're:https://images.gem.cbc.ca/v1/cbc-gem/live/.*',
+                'timestamp': 1679706000,
+                'upload_date': '20230325',
+            },
+            'params': {'skip_download': True},
+            'skip': 'Live might have ended',
+        }
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_info = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['data']
 
-        for api_url in self._API_URLS:
-            video_info = next((
-                stream for stream in self._download_json(api_url, video_id)['entries']
-                if stream.get('guid') == video_id), None)
-            if video_info:
-                break
-        else:
+        # Two types of metadata JSON
+        if not video_info.get('formattedIdMedia'):
+            video_info = traverse_obj(
+                video_info, (('freeTv', ('streams', ...)), 'items', lambda _, v: v['key'] == video_id, {dict}),
+                get_all=False, default={})
+
+        video_stream_id = video_info.get('formattedIdMedia')
+        if not video_stream_id:
             raise ExtractorError('Couldn\'t find video metadata, maybe this livestream is now offline', expected=True)
 
+        stream_data = self._download_json(
+            'https://services.radio-canada.ca/media/validation/v2/', video_id, query={
+                'appCode': 'mpx',
+                'connectionType': 'hd',
+                'deviceType': 'ipad',
+                'idMedia': video_stream_id,
+                'multibitrate': 'true',
+                'output': 'json',
+                'tech': 'hls',
+                'manifestType': 'desktop',
+            })
+
         return {
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
-            'url': video_info['content'][0]['url'],
             'id': video_id,
-            'title': video_info.get('title'),
-            'description': video_info.get('description'),
-            'tags': try_get(video_info, lambda x: x['keywords'].split(', ')),
-            'thumbnail': video_info.get('cbc$staticImage'),
+            'formats': self._extract_m3u8_formats(stream_data['url'], video_id, 'mp4', live=True),
             'is_live': True,
+            **traverse_obj(video_info, {
+                'title': 'title',
+                'description': 'description',
+                'thumbnail': ('images', 'card', 'url'),
+                'timestamp': ('airDate', {parse_iso8601}),
+            })
         }

From f005a35aa7e4f67a0c603a946c0dd714c151b2d6 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 29 Apr 2023 00:58:48 +0530
Subject: [PATCH 243/871] Ensure pre-processor errors do not block `--print`

Closes #6937
---
 yt_dlp/YoutubeDL.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index a8b4a650e..857b7ea37 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3489,10 +3489,12 @@ def run_pp(self, pp, infodict):
         return infodict
 
     def run_all_pps(self, key, info, *, additional_pps=None):
-        for pp in (additional_pps or []) + self._pps[key]:
-            info = self.run_pp(pp, info)
-        if key != 'video':
-            self._forceprint(key, info)
+        try:
+            for pp in (additional_pps or []) + self._pps[key]:
+                info = self.run_pp(pp, info)
+        finally:
+            if key != 'video':
+                self._forceprint(key, info)
         return info
 
     def pre_process(self, ie_info, key='pre_process', files_to_move=None):

From 17ba4343cf99701692a7f4798fd42b50f644faba Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 29 Apr 2023 02:57:50 +0530
Subject: [PATCH 244/871] Fix f005a35aa7e4f67a0c603a946c0dd714c151b2d6

Printing inside `finally` causes the order of logging to change
when there is an error, which is undesirable. So this is reverted.

The issue of `--print` being blocked by pre-processors was an
unintentional side-effect of changing the operation orders in
170605840ea9d5ad75da6576485ea7d125b428ee, and this is also partially
reverted.
---
 yt_dlp/YoutubeDL.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 857b7ea37..8ee42b86a 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3488,13 +3488,11 @@ def run_pp(self, pp, infodict):
                 *files_to_delete, info=infodict, msg='Deleting original file %s (pass -k to keep)')
         return infodict
 
-    def run_all_pps(self, key, info, *, additional_pps=None):
-        try:
-            for pp in (additional_pps or []) + self._pps[key]:
-                info = self.run_pp(pp, info)
-        finally:
-            if key != 'video':
-                self._forceprint(key, info)
+    def run_all_pps(self, key, info, *, additional_pps=None, fatal=True):
+        if key != 'video':
+            self._forceprint(key, info)
+        for pp in (additional_pps or []) + self._pps[key]:
+            info = self.run_pp(pp, info)
         return info
 
     def pre_process(self, ie_info, key='pre_process', files_to_move=None):

From 4d9280c9c853733534dda60486fa949bcca36c9e Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 29 Apr 2023 13:19:35 -0500
Subject: [PATCH 245/871] [extractor/reddit] Add login support (#6950)

Closes #6949
Authored by: bashonly
---
 yt_dlp/extractor/reddit.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/yt_dlp/extractor/reddit.py b/yt_dlp/extractor/reddit.py
index 3e458456c..13615e82f 100644
--- a/yt_dlp/extractor/reddit.py
+++ b/yt_dlp/extractor/reddit.py
@@ -8,11 +8,13 @@
     traverse_obj,
     try_get,
     unescapeHTML,
+    urlencode_postdata,
     url_or_none,
 )
 
 
 class RedditIE(InfoExtractor):
+    _NETRC_MACHINE = 'reddit'
     _VALID_URL = r'https?://(?P<host>(?:\w+\.)?reddit(?:media)?\.com)/(?P<slug>(?:(?:r|user)/[^/]+/)?comments/(?P<id>[^/?#&]+))'
     _TESTS = [{
         'url': 'https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/',
@@ -176,6 +178,25 @@ class RedditIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _perform_login(self, username, password):
+        captcha = self._download_json(
+            'https://www.reddit.com/api/requires_captcha/login.json', None,
+            'Checking login requirement')['required']
+        if captcha:
+            raise ExtractorError('Reddit is requiring captcha before login', expected=True)
+        login = self._download_json(
+            f'https://www.reddit.com/api/login/{username}', None, data=urlencode_postdata({
+                'op': 'login-main',
+                'user': username,
+                'passwd': password,
+                'api_type': 'json',
+            }), note='Logging in', errnote='Login request failed')
+        errors = '; '.join(traverse_obj(login, ('json', 'errors', ..., 1)))
+        if errors:
+            raise ExtractorError(f'Unable to login, Reddit API says {errors}', expected=True)
+        elif not traverse_obj(login, ('json', 'data', 'cookie', {str})):
+            raise ExtractorError('Unable to login, no cookie was returned')
+
     def _real_extract(self, url):
         host, slug, video_id = self._match_valid_url(url).group('host', 'slug', 'id')
 

From b079c26f0af8085bccdadc72c61c8164ca5ab0f8 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 30 Apr 2023 19:50:22 +0200
Subject: [PATCH 246/871] [utils] `traverse_obj`: More fixes (#6959)

- Fix result when branching with `traverse_string`
- Fix `slice` path on `dict`s
- Fix tests and docstrings from 21b5ec86c2c37d10c5bb97edd7051d3aac16bb3e
- Add `is_iterable_like` helper function

Authored by: Grub4K
---
 test/test_utils.py | 21 +++++++++++++++++++--
 yt_dlp/utils.py    | 28 ++++++++++++++++++----------
 2 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index f2f3b8170..e1bf6ac20 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2016,7 +2016,7 @@ def test_traverse_obj(self):
                          msg='nested `...` queries should work')
         self.assertCountEqual(traverse_obj(_TEST_DATA, (..., ..., 'index')), range(4),
                               msg='`...` query result should be flattened')
-        self.assertEqual(traverse_obj(range(4), ...), list(range(4)),
+        self.assertEqual(traverse_obj(iter(range(4)), ...), list(range(4)),
                          msg='`...` should accept iterables')
 
         # Test function as key
@@ -2025,7 +2025,7 @@ def test_traverse_obj(self):
                          msg='function as query key should perform a filter based on (key, value)')
         self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), {'str'},
                               msg='exceptions in the query function should be catched')
-        self.assertEqual(traverse_obj(range(4), lambda _, x: x % 2 == 0), [0, 2],
+        self.assertEqual(traverse_obj(iter(range(4)), lambda _, x: x % 2 == 0), [0, 2],
                          msg='function key should accept iterables')
         if __debug__:
             with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
@@ -2051,6 +2051,17 @@ def test_traverse_obj(self):
             with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
                 traverse_obj(_TEST_DATA, {str.upper, str})
 
+        # Test `slice` as a key
+        _SLICE_DATA = [0, 1, 2, 3, 4]
+        self.assertEqual(traverse_obj(_TEST_DATA, ('dict', slice(1))), None,
+                         msg='slice on a dictionary should not throw')
+        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1)), _SLICE_DATA[:1],
+                         msg='slice key should apply slice to sequence')
+        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 2)), _SLICE_DATA[1:2],
+                         msg='slice key should apply slice to sequence')
+        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 4, 2)), _SLICE_DATA[1:4:2],
+                         msg='slice key should apply slice to sequence')
+
         # Test alternative paths
         self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str',
                          msg='multiple `paths` should be treated as alternative paths')
@@ -2234,6 +2245,12 @@ def test_traverse_obj(self):
         self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', (0, 2)),
                                       traverse_string=True), ['s', 'r'],
                          msg='branching should result in list if `traverse_string`')
+        self.assertEqual(traverse_obj({}, (0, ...), traverse_string=True), [],
+                         msg='branching should result in list if `traverse_string`')
+        self.assertEqual(traverse_obj({}, (0, lambda x, y: True), traverse_string=True), [],
+                         msg='branching should result in list if `traverse_string`')
+        self.assertEqual(traverse_obj({}, (0, slice(1)), traverse_string=True), [],
+                         msg='branching should result in list if `traverse_string`')
 
         # Test is_user_input behavior
         _IS_USER_INPUT_DATA = {'range8': list(range(8))}
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index f69311462..2f5e66720 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3273,8 +3273,14 @@ def multipart_encode(data, boundary=None):
     return out, content_type
 
 
-def variadic(x, allowed_types=(str, bytes, dict)):
-    return x if isinstance(x, collections.abc.Iterable) and not isinstance(x, allowed_types) else (x,)
+def is_iterable_like(x, allowed_types=collections.abc.Iterable, blocked_types=NO_DEFAULT):
+    if blocked_types is NO_DEFAULT:
+        blocked_types = (str, bytes, collections.abc.Mapping)
+    return isinstance(x, allowed_types) and not isinstance(x, blocked_types)
+
+
+def variadic(x, allowed_types=NO_DEFAULT):
+    return x if is_iterable_like(x, blocked_types=allowed_types) else (x,)
 
 
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
@@ -5467,7 +5473,7 @@ def traverse_obj(
         obj, *paths, default=NO_DEFAULT, expected_type=None, get_all=True,
         casesense=True, is_user_input=False, traverse_string=False):
     """
-    Safely traverse nested `dict`s and `Sequence`s
+    Safely traverse nested `dict`s and `Iterable`s
 
     >>> obj = [{}, {"key": "value"}]
     >>> traverse_obj(obj, (1, "key"))
@@ -5475,7 +5481,7 @@ def traverse_obj(
 
     Each of the provided `paths` is tested and the first producing a valid result will be returned.
     The next path will also be tested if the path branched but no results could be found.
-    Supported values for traversal are `Mapping`, `Sequence` and `re.Match`.
+    Supported values for traversal are `Mapping`, `Iterable` and `re.Match`.
     Unhelpful values (`{}`, `None`) are treated as the absence of a value and discarded.
 
     The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
@@ -5492,7 +5498,7 @@ def traverse_obj(
                             Read as: `[traverse_obj(obj, branch) for branch in branches]`.
         - `function`:       Branch out and return values filtered by the function.
                             Read as: `[value for key, value in obj if function(key, value)]`.
-                            For `Sequence`s, `key` is the index of the value.
+                            For `Iterable`s, `key` is the index of the value.
                             For `re.Match`es, `key` is the group number (0 = full match)
                             as well as additionally any group names, if given.
         - `dict`            Transform the current object and return a matching dict.
@@ -5540,7 +5546,9 @@ def apply_key(key, obj, is_last):
         result = None
 
         if obj is None and traverse_string:
-            pass
+            if key is ... or callable(key) or isinstance(key, slice):
+                branching = True
+                result = ()
 
         elif key is None:
             result = obj
@@ -5563,7 +5571,7 @@ def apply_key(key, obj, is_last):
             branching = True
             if isinstance(obj, collections.abc.Mapping):
                 result = obj.values()
-            elif isinstance(obj, collections.abc.Iterable) and not isinstance(obj, (str, bytes)):
+            elif is_iterable_like(obj):
                 result = obj
             elif isinstance(obj, re.Match):
                 result = obj.groups()
@@ -5577,7 +5585,7 @@ def apply_key(key, obj, is_last):
             branching = True
             if isinstance(obj, collections.abc.Mapping):
                 iter_obj = obj.items()
-            elif isinstance(obj, collections.abc.Iterable) and not isinstance(obj, (str, bytes)):
+            elif is_iterable_like(obj):
                 iter_obj = enumerate(obj)
             elif isinstance(obj, re.Match):
                 iter_obj = itertools.chain(
@@ -5601,7 +5609,7 @@ def apply_key(key, obj, is_last):
             } or None
 
         elif isinstance(obj, collections.abc.Mapping):
-            result = (obj.get(key) if casesense or (key in obj) else
+            result = (try_call(obj.get, args=(key,)) if casesense or try_call(obj.__contains__, args=(key,)) else
                       next((v for k, v in obj.items() if casefold(k) == key), None))
 
         elif isinstance(obj, re.Match):
@@ -5613,7 +5621,7 @@ def apply_key(key, obj, is_last):
                 result = next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
 
         elif isinstance(key, (int, slice)):
-            if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, (str, bytes)):
+            if is_iterable_like(obj, collections.abc.Sequence):
                 branching = isinstance(key, slice)
                 with contextlib.suppress(IndexError):
                     result = obj[key]

From 147e62fc584c3ea6fdb09bb7a47905df68553a22 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Mon, 1 May 2023 18:55:28 -0500
Subject: [PATCH 247/871] [extractor/twitter] Default to GraphQL, handle auth
 errors (#6957)

Closes #6763
Authored by: bashonly
---
 README.md                   |   2 +-
 yt_dlp/extractor/twitter.py | 132 ++++++++++++++++--------------------
 2 files changed, 60 insertions(+), 74 deletions(-)

diff --git a/README.md b/README.md
index 47da19011..c1f34235d 100644
--- a/README.md
+++ b/README.md
@@ -1833,7 +1833,7 @@ #### rokfinchannel
 * `tab`: Which tab to download - one of `new`, `top`, `videos`, `podcasts`, `streams`, `stacks`
 
 #### twitter
-* `force_graphql`: Force usage of the GraphQL API. By default it will only be used if login cookies are provided
+* `legacy_api`: Force usage of the legacy Twitter API instead of the GraphQL API for tweet extraction. Has no effect if login cookies are passed
 
 **Note**: These options may be changed/removed in the future without concern for backward compatibility
 
diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 3f1899e96..d9a89c44b 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1,6 +1,5 @@
 import json
 import re
-import urllib.error
 
 from .common import InfoExtractor
 from .periscope import PeriscopeBaseIE, PeriscopeIE
@@ -17,6 +16,7 @@
     format_field,
     int_or_none,
     make_archive_id,
+    remove_end,
     str_or_none,
     strip_or_none,
     traverse_obj,
@@ -32,11 +32,9 @@
 class TwitterBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitter.com/1.1/'
     _GRAPHQL_API_BASE = 'https://twitter.com/i/api/graphql/'
-    _TOKENS = {
-        'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA': None,
-        'AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw': None,
-    }
     _BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?(?:twitter\.com|twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid\.onion)/'
+    _AUTH = {'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'}
+    _guest_token = None
 
     def _extract_variant_formats(self, variant, video_id):
         variant_url = variant.get('url')
@@ -94,7 +92,7 @@ def is_logged_in(self):
 
     def _call_api(self, path, video_id, query={}, graphql=False):
         cookies = self._get_cookies(self._API_BASE)
-        headers = {}
+        headers = self._AUTH.copy()
 
         csrf_cookie = cookies.get('ct0')
         if csrf_cookie:
@@ -107,54 +105,34 @@ def _call_api(self, path, video_id, query={}, graphql=False):
                 'x-twitter-active-user': 'yes',
             })
 
-        last_error = None
-        for bearer_token in self._TOKENS:
-            for first_attempt in (True, False):
-                headers['Authorization'] = f'Bearer {bearer_token}'
+        for first_attempt in (True, False):
+            if not self.is_logged_in and not self._guest_token:
+                headers.pop('x-guest-token', None)
+                self._guest_token = traverse_obj(self._download_json(
+                    f'{self._API_BASE}guest/activate.json', video_id,
+                    'Downloading guest token', data=b'', headers=headers), 'guest_token')
+            if self._guest_token:
+                headers['x-guest-token'] = self._guest_token
+            elif not self.is_logged_in:
+                raise ExtractorError('Could not retrieve guest token')
 
-                if not self.is_logged_in:
-                    if not self._TOKENS[bearer_token]:
-                        headers.pop('x-guest-token', None)
-                        guest_token_response = self._download_json(
-                            self._API_BASE + 'guest/activate.json', video_id,
-                            'Downloading guest token', data=b'', headers=headers)
+            allowed_status = {400, 401, 403, 404} if graphql else {403}
+            result = self._download_json(
+                (self._GRAPHQL_API_BASE if graphql else self._API_BASE) + path,
+                video_id, headers=headers, query=query, expected_status=allowed_status,
+                note=f'Downloading {"GraphQL" if graphql else "legacy API"} JSON')
 
-                        self._TOKENS[bearer_token] = guest_token_response.get('guest_token')
-                        if not self._TOKENS[bearer_token]:
-                            raise ExtractorError('Could not retrieve guest token')
+            if result.get('errors'):
+                errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
+                if not self.is_logged_in and first_attempt and 'bad guest token' in errors.lower():
+                    self.to_screen('Guest token has expired. Refreshing guest token')
+                    self._guest_token = None
+                    continue
 
-                    headers['x-guest-token'] = self._TOKENS[bearer_token]
+                raise ExtractorError(
+                    f'Error(s) while querying API: {errors or "Unknown error"}', expected=True)
 
-                try:
-                    allowed_status = {400, 403, 404} if graphql else {403}
-                    result = self._download_json(
-                        (self._GRAPHQL_API_BASE if graphql else self._API_BASE) + path,
-                        video_id, headers=headers, query=query, expected_status=allowed_status)
-
-                except ExtractorError as e:
-                    if last_error:
-                        raise last_error
-
-                    if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code != 404:
-                        raise
-
-                    last_error = e
-                    self.report_warning(
-                        'Twitter API gave 404 response, retrying with deprecated auth token. '
-                        'Only one media item can be extracted')
-                    break  # continue outer loop with next bearer_token
-
-                if result.get('errors'):
-                    errors = traverse_obj(result, ('errors', ..., 'message'), expected_type=str)
-                    if first_attempt and any('bad guest token' in error.lower() for error in errors):
-                        self.to_screen('Guest token has expired. Refreshing guest token')
-                        self._TOKENS[bearer_token] = None
-                        continue
-
-                    error_message = ', '.join(set(errors)) or 'Unknown error'
-                    raise ExtractorError(f'Error(s) while querying API: {error_message}', expected=True)
-
-                return result
+            return result
 
     def _build_graphql_query(self, media_id):
         raise NotImplementedError('Method must be implemented to support GraphQL')
@@ -313,6 +291,7 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 18,
         },
@@ -391,6 +370,7 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': ['Damndaniel'],
             'age_limit': 0,
         },
@@ -431,6 +411,7 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -480,6 +461,7 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': ['Maria'],
             'age_limit': 0,
         },
@@ -505,6 +487,7 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -529,6 +512,7 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -589,6 +573,7 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -630,12 +615,12 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': ['HurricaneIan'],
             'age_limit': 0,
         },
     }, {
-        # Adult content, uses old token
-        # Fails if not logged in (GraphQL)
+        # Adult content, fails if not logged in (GraphQL)
         'url': 'https://twitter.com/Rizdraws/status/1575199173472927762',
         'info_dict': {
             'id': '1575199163847000068',
@@ -655,9 +640,8 @@ class TwitterIE(TwitterBaseIE):
             'age_limit': 18,
             'tags': []
         },
-        'expected_warnings': ['404'],
+        'skip': 'Requires authentication',
     }, {
-        # Description is missing one https://t.co url (GraphQL)
         'url': 'https://twitter.com/Srirachachau/status/1395079556562706435',
         'playlist_mincount': 2,
         'info_dict': {
@@ -669,14 +653,13 @@ class TwitterIE(TwitterBaseIE):
             'upload_date': '20210519',
             'age_limit': 0,
             'repost_count': int,
-            'description': 'Here it is! Finished my gothic western cartoon. Pretty proud of it. It\'s got some goofs and lots of splashy over the top violence, something for everyone, hope you like it https://t.co/fOsG5glUnw https://t.co/kbXZrozlY7',
+            'description': 'Here it is! Finished my gothic western cartoon. Pretty proud of it. It\'s got some goofs and lots of splashy over the top violence, something for everyone, hope you like it https://t.co/fOsG5glUnw',
             'uploader_id': 'Srirachachau',
             'comment_count': int,
             'uploader_url': 'https://twitter.com/Srirachachau',
             'timestamp': 1621447860,
         },
     }, {
-        # Description is missing one https://t.co url (GraphQL)
         'url': 'https://twitter.com/DavidToons_/status/1578353380363501568',
         'playlist_mincount': 2,
         'info_dict': {
@@ -688,7 +671,7 @@ class TwitterIE(TwitterBaseIE):
             'uploader': str,
             'timestamp': 1665143744,
             'uploader_url': 'https://twitter.com/DavidToons_',
-            'description': 'Chris sounds like Linda from Bob\'s Burgers, so as an animator: this had to be done. https://t.co/glfQdgfFXH https://t.co/WgJauwIW1w',
+            'description': 'Chris sounds like Linda from Bob\'s Burgers, so as an animator: this had to be done. https://t.co/WgJauwIW1w',
             'tags': [],
             'comment_count': int,
             'upload_date': '20221007',
@@ -752,7 +735,7 @@ class TwitterIE(TwitterBaseIE):
         'info_dict': {
             'id': '1600649511827013632',
             'ext': 'mp4',
-            'title': 'md5:dac4f4d4c591fcc4e88a253eba472dc3',
+            'title': 'md5:7662a0a27ce6faa3e5b160340f3cfab1',
             'thumbnail': r're:^https?://.+\.jpg',
             'timestamp': 1670459604.0,
             'uploader_id': 'CTVJLaidlaw',
@@ -764,6 +747,7 @@ class TwitterIE(TwitterBaseIE):
             'uploader_url': 'https://twitter.com/CTVJLaidlaw',
             'display_id': '1600649710662213632',
             'like_count': int,
+            'view_count': int,
             'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
             'upload_date': '20221208',
             'age_limit': 0,
@@ -791,6 +775,7 @@ class TwitterIE(TwitterBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'view_count': int,
         },
     }, {
         'url': 'https://twitter.com/hlo_again/status/1599108751385972737/video/2',
@@ -806,6 +791,7 @@ class TwitterIE(TwitterBaseIE):
             'repost_count': int,
             'duration': 9.531,
             'comment_count': int,
+            'view_count': int,
             'upload_date': '20221203',
             'age_limit': 0,
             'timestamp': 1670092210.0,
@@ -815,7 +801,6 @@ class TwitterIE(TwitterBaseIE):
         },
         'params': {'noplaylist': True},
     }, {
-        # Media view count is GraphQL only, force in test
         'url': 'https://twitter.com/MunTheShinobi/status/1600009574919962625',
         'info_dict': {
             'id': '1600009362759733248',
@@ -826,10 +811,10 @@ class TwitterIE(TwitterBaseIE):
             'view_count': int,
             'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1600009362759733248/pu/img/XVhFQivj75H_YxxV.jpg?name=orig',
             'age_limit': 0,
-            'uploader': 'Mün The Shinobi | BlaqBoi\'s Therapist',
+            'uploader': 'Mün The Shinobi',
             'repost_count': int,
             'upload_date': '20221206',
-            'title': 'Mün The Shinobi | BlaqBoi\'s Therapist - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
+            'title': 'Mün The Shinobi - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
             'comment_count': int,
             'like_count': int,
             'tags': [],
@@ -837,9 +822,8 @@ class TwitterIE(TwitterBaseIE):
             'duration': 139.987,
             'timestamp': 1670306984.0,
         },
-        'params': {'extractor_args': {'twitter': {'force_graphql': ['']}}},
     }, {
-        # url to retweet id
+        # url to retweet id, legacy API
         'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
         'info_dict': {
             'id': '1623274794488659969',
@@ -860,6 +844,7 @@ class TwitterIE(TwitterBaseIE):
             'repost_count': int,
             'comment_count': int,
         },
+        'params': {'extractor_args': {'twitter': {'legacy_api': ['']}}},
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -905,11 +890,13 @@ def _graphql_to_legacy(self, data, twid):
             'tweet_results', 'result', ('tweet', None),
         ), expected_type=dict, default={}, get_all=False)
 
-        if result.get('__typename') not in ('Tweet', None):
+        if result.get('__typename') not in ('Tweet', 'TweetTombstone', None):
             self.report_warning(f'Unknown typename: {result.get("__typename")}', twid, only_once=True)
 
         if 'tombstone' in result:
-            cause = traverse_obj(result, ('tombstone', 'text', 'text'), expected_type=str)
+            cause = remove_end(traverse_obj(result, ('tombstone', 'text', 'text', {str})), '. Learn more')
+            if cause and 'adult content' in cause:
+                self.raise_login_required(cause)
             raise ExtractorError(f'Twitter API says: {cause or "Unknown error"}', expected=True)
 
         status = result.get('legacy', {})
@@ -922,7 +909,7 @@ def _graphql_to_legacy(self, data, twid):
         # extra transformation is needed since result does not match legacy format
         binding_values = {
             binding_value.get('key'): binding_value.get('value')
-            for binding_value in traverse_obj(status, ('card', 'binding_values', ...), expected_type=dict)
+            for binding_value in traverse_obj(status, ('card', 'binding_values', ..., {dict}))
         }
         if binding_values:
             status['card']['binding_values'] = binding_values
@@ -965,12 +952,7 @@ def _build_graphql_query(self, media_id):
 
     def _real_extract(self, url):
         twid, selected_index = self._match_valid_url(url).group('id', 'index')
-        if self.is_logged_in or self._configuration_arg('force_graphql'):
-            self.write_debug(f'Using GraphQL API (Auth = {self.is_logged_in})')
-            result = self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid)
-            status = self._graphql_to_legacy(result, twid)
-
-        else:
+        if self._configuration_arg('legacy_api') and not self.is_logged_in:
             status = traverse_obj(self._call_api(f'statuses/show/{twid}.json', twid, {
                 'cards_platform': 'Web-12',
                 'include_cards': 1,
@@ -978,6 +960,9 @@ def _real_extract(self, url):
                 'include_user_entities': 0,
                 'tweet_mode': 'extended',
             }), 'retweeted_status', None)
+        else:
+            result = self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid)
+            status = self._graphql_to_legacy(result, twid)
 
         title = description = status['full_text'].replace('\n', ' ')
         # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
@@ -1142,7 +1127,8 @@ def get_binding_value(k):
         if not entries:
             expanded_url = traverse_obj(status, ('entities', 'urls', 0, 'expanded_url'), expected_type=url_or_none)
             if not expanded_url or expanded_url == url:
-                raise ExtractorError('No video could be found in this tweet', expected=True)
+                self.raise_no_formats('No video could be found in this tweet', expected=True)
+                return info
 
             return self.url_result(expanded_url, display_id=twid, **info)
 

From b423b6a48e0b19260bc95ab7d72d2138d7f124dc Mon Sep 17 00:00:00 2001
From: Nicholas Defranco <39540565+nick-cd@users.noreply.github.com>
Date: Mon, 1 May 2023 20:03:27 -0400
Subject: [PATCH 248/871] [extractor/dlf] Add extractors (#6697)

Closes #6430
Authored by: nick-cd
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/dlf.py         | 192 ++++++++++++++++++++++++++++++++
 2 files changed, 196 insertions(+)
 create mode 100644 yt_dlp/extractor/dlf.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 3b5ae63b1..2d582f67f 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -452,6 +452,10 @@
 )
 from .democracynow import DemocracynowIE
 from .detik import DetikEmbedIE
+from .dlf import (
+    DLFIE,
+    DLFCorpusIE,
+)
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .digg import DiggIE
diff --git a/yt_dlp/extractor/dlf.py b/yt_dlp/extractor/dlf.py
new file mode 100644
index 000000000..88a4149b5
--- /dev/null
+++ b/yt_dlp/extractor/dlf.py
@@ -0,0 +1,192 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    extract_attributes,
+    int_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class DLFBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?deutschlandfunk\.de/'
+    _BUTTON_REGEX = r'(<button[^>]+alt="Anhören"[^>]+data-audio-diraid[^>]*>)'
+
+    def _parse_button_attrs(self, button, audio_id=None):
+        attrs = extract_attributes(button)
+        audio_id = audio_id or attrs['data-audio-diraid']
+
+        url = traverse_obj(
+            attrs, 'data-audio-download-src', 'data-audio', 'data-audioreference',
+            'data-audio-src', expected_type=url_or_none)
+        ext = determine_ext(url)
+
+        return {
+            'id': audio_id,
+            'extractor_key': DLFIE.ie_key(),
+            'extractor': DLFIE.IE_NAME,
+            **traverse_obj(attrs, {
+                'title': (('data-audiotitle', 'data-audio-title', 'data-audio-download-tracking-title'), {str}),
+                'duration': (('data-audioduration', 'data-audio-duration'), {int_or_none}),
+                'thumbnail': ('data-audioimage', {url_or_none}),
+                'uploader': 'data-audio-producer',
+                'series': 'data-audio-series',
+                'channel': 'data-audio-origin-site-name',
+                'webpage_url': ('data-audio-download-tracking-path', {url_or_none}),
+            }, get_all=False),
+            'formats': (self._extract_m3u8_formats(url, audio_id, fatal=False)
+                        if ext == 'm3u8' else [{'url': url, 'ext': ext, 'vcodec': 'none'}])
+        }
+
+
+class DLFIE(DLFBaseIE):
+    IE_NAME = 'dlf'
+    _VALID_URL = DLFBaseIE._VALID_URL_BASE + r'[\w-]+-dlf-(?P<id>[\da-f]{8})-100\.html'
+    _TESTS = [
+        # Audio as an HLS stream
+        {
+            'url': 'https://www.deutschlandfunk.de/tanz-der-saiteninstrumente-das-wild-strings-trio-aus-slowenien-dlf-03a3eb19-100.html',
+            'info_dict': {
+                'id': '03a3eb19',
+                'title': r're:Tanz der Saiteninstrumente [-/] Das Wild Strings Trio aus Slowenien',
+                'ext': 'm4a',
+                'duration': 3298,
+                'thumbnail': 'https://assets.deutschlandfunk.de/FALLBACK-IMAGE-AUDIO/512x512.png?t=1603714364673',
+                'uploader': 'Deutschlandfunk',
+                'series': 'On Stage',
+                'channel': 'deutschlandfunk'
+            },
+            'params': {
+                'skip_download': 'm3u8'
+            },
+            'skip': 'This webpage no longer exists'
+        }, {
+            'url': 'https://www.deutschlandfunk.de/russische-athleten-kehren-zurueck-auf-die-sportbuehne-ein-gefaehrlicher-tueroeffner-dlf-d9cc1856-100.html',
+            'info_dict': {
+                'id': 'd9cc1856',
+                'title': 'Russische Athleten kehren zurück auf die Sportbühne: Ein gefährlicher Türöffner',
+                'ext': 'mp3',
+                'duration': 291,
+                'thumbnail': 'https://assets.deutschlandfunk.de/FALLBACK-IMAGE-AUDIO/512x512.png?t=1603714364673',
+                'uploader': 'Deutschlandfunk',
+                'series': 'Kommentare und Themen der Woche',
+                'channel': 'deutschlandfunk'
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        webpage = self._download_webpage(url, audio_id)
+
+        return self._parse_button_attrs(
+            self._search_regex(self._BUTTON_REGEX, webpage, 'button'), audio_id)
+
+
+class DLFCorpusIE(DLFBaseIE):
+    IE_NAME = 'dlf:corpus'
+    IE_DESC = 'DLF Multi-feed Archives'
+    _VALID_URL = DLFBaseIE._VALID_URL_BASE + r'(?P<id>(?![\w-]+-dlf-[\da-f]{8})[\w-]+-\d+)\.html'
+    _TESTS = [
+        # Recorded news broadcast with referrals to related broadcasts
+        {
+            'url': 'https://www.deutschlandfunk.de/fechten-russland-belarus-ukraine-protest-100.html',
+            'info_dict': {
+                'id': 'fechten-russland-belarus-ukraine-protest-100',
+                'title': r're:Wiederzulassung als neutrale Athleten [-/] Was die Rückkehr russischer und belarussischer Sportler beim Fechten bedeutet',
+                'description': 'md5:91340aab29c71aa7518ad5be13d1e8ad'
+            },
+            'playlist_mincount': 5,
+            'playlist': [{
+                'info_dict': {
+                    'id': '1fc5d64a',
+                    'title': r're:Wiederzulassung als neutrale Athleten [-/] Was die Rückkehr russischer und belarussischer Sportler beim Fechten bedeutet',
+                    'ext': 'mp3',
+                    'duration': 252,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/aad16241-6b76-4a09-958b-96d0ee1d6f57/512x512.jpg?t=1679480020313',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Sport',
+                    'channel': 'deutschlandfunk'
+                }
+            }, {
+                'info_dict': {
+                    'id': '2ada145f',
+                    'title': r're:(?:Sportpolitik / )?Fechtverband votiert für Rückkehr russischer Athleten',
+                    'ext': 'mp3',
+                    'duration': 336,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/FILE_93982766f7317df30409b8a184ac044a/512x512.jpg?t=1678547581005',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Deutschlandfunk Nova',
+                    'channel': 'deutschlandfunk-nova'
+                }
+            }, {
+                'info_dict': {
+                    'id': '5e55e8c9',
+                    'title': r're:Wiederzulassung von Russland und Belarus [-/] "Herumlavieren" des Fechter-Bundes sorgt für Unverständnis',
+                    'ext': 'mp3',
+                    'duration': 187,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/a595989d-1ed1-4a2e-8370-b64d7f11d757/512x512.jpg?t=1679173825412',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Sport am Samstag',
+                    'channel': 'deutschlandfunk'
+                }
+            }, {
+                'info_dict': {
+                    'id': '47e1a096',
+                    'title': r're:Rückkehr Russlands im Fechten [-/] "Fassungslos, dass es einfach so passiert ist"',
+                    'ext': 'mp3',
+                    'duration': 602,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/da4c494a-21cc-48b4-9cc7-40e09fd442c2/512x512.jpg?t=1678562155770',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Sport am Samstag',
+                    'channel': 'deutschlandfunk'
+                }
+            }, {
+                'info_dict': {
+                    'id': '5e55e8c9',
+                    'title': r're:Wiederzulassung von Russland und Belarus [-/] "Herumlavieren" des Fechter-Bundes sorgt für Unverständnis',
+                    'ext': 'mp3',
+                    'duration': 187,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/a595989d-1ed1-4a2e-8370-b64d7f11d757/512x512.jpg?t=1679173825412',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Sport am Samstag',
+                    'channel': 'deutschlandfunk'
+                }
+            }]
+        },
+        # Podcast feed with tag buttons, playlist count fluctuates
+        {
+            'url': 'https://www.deutschlandfunk.de/kommentare-und-themen-der-woche-100.html',
+            'info_dict': {
+                'id': 'kommentare-und-themen-der-woche-100',
+                'title': 'Meinung - Kommentare und Themen der Woche',
+                'description': 'md5:2901bbd65cd2d45e116d399a099ce5d5',
+            },
+            'playlist_mincount': 10,
+        },
+        # Podcast feed with no description
+        {
+            'url': 'https://www.deutschlandfunk.de/podcast-tolle-idee-100.html',
+            'info_dict': {
+                'id': 'podcast-tolle-idee-100',
+                'title': 'Wissenschaftspodcast - Tolle Idee! - Was wurde daraus?',
+            },
+            'playlist_mincount': 11,
+        },
+    ]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'description': self._html_search_meta(
+                ['description', 'og:description', 'twitter:description'], webpage, default=None),
+            'title': self._html_search_meta(
+                ['og:title', 'twitter:title'], webpage, default=None),
+            'entries': map(self._parse_button_attrs, re.findall(self._BUTTON_REGEX, webpage)),
+        }

From 2f07c4c1da4361af213e5791279b9d152d2e4ce3 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Wed, 3 May 2023 15:46:37 -0500
Subject: [PATCH 249/871] [extractor/clipchamp] Add extractor (#6978)

Closes #6973
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/clipchamp.py   | 61 +++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 yt_dlp/extractor/clipchamp.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 2d582f67f..974c8a254 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -356,6 +356,7 @@
 )
 from .ciscowebex import CiscoWebexIE
 from .cjsw import CJSWIE
+from .clipchamp import ClipchampIE
 from .cliphunter import CliphunterIE
 from .clippit import ClippitIE
 from .cliprs import ClipRsIE
diff --git a/yt_dlp/extractor/clipchamp.py b/yt_dlp/extractor/clipchamp.py
new file mode 100644
index 000000000..a8bdf7e50
--- /dev/null
+++ b/yt_dlp/extractor/clipchamp.py
@@ -0,0 +1,61 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    traverse_obj,
+    unified_timestamp,
+    url_or_none,
+)
+
+
+class ClipchampIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?clipchamp\.com/watch/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://clipchamp.com/watch/gRXZ4ZhdDaU',
+        'info_dict': {
+            'id': 'gRXZ4ZhdDaU',
+            'ext': 'mp4',
+            'title': 'Untitled video',
+            'uploader': 'Alexander Schwartz',
+            'timestamp': 1680805580,
+            'upload_date': '20230406',
+            'thumbnail': r're:^https?://.+\.jpg',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    _STREAM_URL_TMPL = 'https://%s.cloudflarestream.com/%s/manifest/video.%s'
+    _STREAM_URL_QUERY = {'parentOrigin': 'https://clipchamp.com'}
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['video']
+
+        storage_location = data.get('storage_location')
+        if storage_location != 'cf_stream':
+            raise ExtractorError(f'Unsupported clip storage location "{storage_location}"')
+
+        path = data['download_url']
+        iframe = self._download_webpage(
+            f'https://iframe.cloudflarestream.com/{path}', video_id, 'Downloading player iframe')
+        subdomain = self._search_regex(
+            r'\bcustomer-domain-prefix=["\']([\w-]+)["\']', iframe,
+            'subdomain', fatal=False) or 'customer-2ut9yn3y6fta1yxe'
+
+        formats = self._extract_mpd_formats(
+            self._STREAM_URL_TMPL % (subdomain, path, 'mpd'), video_id,
+            query=self._STREAM_URL_QUERY, fatal=False, mpd_id='dash')
+        formats.extend(self._extract_m3u8_formats(
+            self._STREAM_URL_TMPL % (subdomain, path, 'm3u8'), video_id, 'mp4',
+            query=self._STREAM_URL_QUERY, fatal=False, m3u8_id='hls'))
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'uploader': ' '.join(traverse_obj(data, ('creator', ('first_name', 'last_name'), {str}))) or None,
+            **traverse_obj(data, {
+                'title': ('project', 'project_name', {str}),
+                'timestamp': ('created_at', {unified_timestamp}),
+                'thumbnail': ('thumbnail_url', {url_or_none}),
+            }),
+        }

From 45998b3e371b819ce0dbe50da703809a048cc2fe Mon Sep 17 00:00:00 2001
From: Eveldee <eveldee0680@live.fr>
Date: Fri, 5 May 2023 07:31:41 +0200
Subject: [PATCH 250/871] [utils] `locked_file`: Fix for virtiofs (#6840)

Authored by: brandon-dacrib
Closes #6823
---
 yt_dlp/utils.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 2f5e66720..47aa75c47 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -2187,10 +2187,11 @@ def _lock_file(f, exclusive, block):
                 fcntl.lockf(f, flags)
 
         def _unlock_file(f):
-            try:
-                fcntl.flock(f, fcntl.LOCK_UN)
-            except OSError:
-                fcntl.lockf(f, fcntl.LOCK_UN)
+            with contextlib.suppress(OSError):
+                return fcntl.flock(f, fcntl.LOCK_UN)
+            with contextlib.suppress(OSError):
+                return fcntl.lockf(f, fcntl.LOCK_UN)  # AOSP does not have flock()
+            return fcntl.flock(f, fcntl.LOCK_UN | fcntl.LOCK_NB)  # virtiofs needs LOCK_NB on unlocking
 
     except ImportError:
 

From ddae33754ae1f32dd9c64cf895c47d20f6b5f336 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 5 May 2023 09:41:56 +0530
Subject: [PATCH 251/871] [extractor/youporn] Extract m3u8 formats

Closes #6977
---
 yt_dlp/extractor/youporn.py | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/yt_dlp/extractor/youporn.py b/yt_dlp/extractor/youporn.py
index 8f1b9911b..6ee0abcae 100644
--- a/yt_dlp/extractor/youporn.py
+++ b/yt_dlp/extractor/youporn.py
@@ -6,6 +6,7 @@
     int_or_none,
     merge_dicts,
     str_to_int,
+    traverse_obj,
     unified_strdate,
     url_or_none,
 )
@@ -86,32 +87,31 @@ class YouPornIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
-
+        video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
         definitions = self._download_json(
-            'https://www.youporn.com/api/video/media_definitions/%s/' % video_id,
-            display_id)
+            f'https://www.youporn.com/api/video/media_definitions/{video_id}/', display_id or video_id)
+
+        def get_format_data(data, f):
+            return traverse_obj(data, lambda _, v: v['format'] == f and url_or_none(v['videoUrl']))
 
         formats = []
-        for definition in definitions:
-            if not isinstance(definition, dict):
-                continue
-            video_url = url_or_none(definition.get('videoUrl'))
-            if not video_url:
-                continue
-            f = {
-                'url': video_url,
-                'filesize': int_or_none(definition.get('videoSize')),
-            }
+        # Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
+        for hls_url in traverse_obj(get_format_data(definitions, 'hls'), (
+                lambda _, v: not isinstance(v['defaultQuality'], bool), 'videoUrl'), (..., 'videoUrl')):
+            formats.extend(self._extract_m3u8_formats(hls_url, video_id, 'mp4', fatal=False, m3u8_id='hls'))
+
+        for definition in get_format_data(definitions, 'mp4'):
+            f = traverse_obj(definition, {
+                'url': 'videoUrl',
+                'filesize': ('videoSize', {int_or_none})
+            })
             height = int_or_none(definition.get('quality'))
             # Video URL's path looks like this:
             #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
             #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
             #  /videos/201703/11/109285532/1080P_4000K_109285532.mp4
             # We will benefit from it by extracting some metadata
-            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
+            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', definition['videoUrl'])
             if mobj:
                 if not height:
                     height = int(mobj.group('height'))
@@ -179,6 +179,7 @@ def extract_tag_box(regex, title):
             'tags')
 
         data = self._search_json_ld(webpage, video_id, expected_type='VideoObject', fatal=False)
+        data.pop('url', None)
         return merge_dicts(data, {
             'id': video_id,
             'display_id': display_id,

From 0c7ce146e4d2a84e656d78f6857952bfd25ab389 Mon Sep 17 00:00:00 2001
From: "lauren n. liberda" <lauren@selfisekai.rocks>
Date: Sat, 6 May 2023 02:09:49 +0200
Subject: [PATCH 252/871] [extractor/tvp] Use new API (#6989)

Authored by: selfisekai
Closes #6987
---
 yt_dlp/extractor/tvp.py | 94 ++++++++++++++++++++++++++++++-----------
 1 file changed, 70 insertions(+), 24 deletions(-)

diff --git a/yt_dlp/extractor/tvp.py b/yt_dlp/extractor/tvp.py
index f8ded2646..2aa0dd870 100644
--- a/yt_dlp/extractor/tvp.py
+++ b/yt_dlp/extractor/tvp.py
@@ -482,21 +482,34 @@ def _real_extract(self, url):
 class TVPVODBaseIE(InfoExtractor):
     _API_BASE_URL = 'https://vod.tvp.pl/api/products'
 
-    def _call_api(self, resource, video_id, **kwargs):
-        return self._download_json(
+    def _call_api(self, resource, video_id, query={}, **kwargs):
+        is_valid = lambda x: 200 <= x < 300
+        document, urlh = self._download_json_handle(
             f'{self._API_BASE_URL}/{resource}', video_id,
-            query={'lang': 'pl', 'platform': 'BROWSER'}, **kwargs)
+            query={'lang': 'pl', 'platform': 'BROWSER', **query},
+            expected_status=lambda x: is_valid(x) or 400 <= x < 500, **kwargs)
+        if is_valid(urlh.status):
+            return document
+        raise ExtractorError(f'Woronicza said: {document.get("code")} (HTTP {urlh.status})')
 
-    def _parse_video(self, video):
-        return {
-            '_type': 'url',
-            'url': 'tvp:' + video['externalUid'],
-            'ie_key': TVPEmbedIE.ie_key(),
-            'title': video.get('title'),
-            'description': traverse_obj(video, ('lead', 'description')),
-            'age_limit': int_or_none(video.get('rating')),
-            'duration': int_or_none(video.get('duration')),
-        }
+    def _parse_video(self, video, with_url=True):
+        info_dict = traverse_obj(video, {
+            'id': ('id', {str_or_none}),
+            'title': 'title',
+            'age_limit': ('rating', {int_or_none}),
+            'duration': ('duration', {int_or_none}),
+            'episode_number': ('number', {int_or_none}),
+            'series': ('season', 'serial', 'title', {str_or_none}),
+            'thumbnails': ('images', ..., ..., {'url': ('url', {url_or_none})}),
+        })
+        info_dict['description'] = clean_html(dict_get(video, ('lead', 'description')))
+        if with_url:
+            info_dict.update({
+                '_type': 'url',
+                'url': video['webUrl'],
+                'ie_key': TVPVODVideoIE.ie_key(),
+            })
+        return info_dict
 
 
 class TVPVODVideoIE(TVPVODBaseIE):
@@ -506,37 +519,70 @@ class TVPVODVideoIE(TVPVODBaseIE):
     _TESTS = [{
         'url': 'https://vod.tvp.pl/dla-dzieci,24/laboratorium-alchemika-odcinki,309338/odcinek-24,S01E24,311357',
         'info_dict': {
-            'id': '60468609',
+            'id': '311357',
             'ext': 'mp4',
-            'title': 'Laboratorium alchemika, Tusze termiczne. Jak zobaczyć niewidoczne. Odcinek 24',
+            'title': 'Tusze termiczne. Jak zobaczyć niewidoczne. Odcinek 24',
             'description': 'md5:1d4098d3e537092ccbac1abf49b7cd4c',
             'duration': 300,
             'episode_number': 24,
             'episode': 'Episode 24',
             'age_limit': 0,
             'series': 'Laboratorium alchemika',
-            'thumbnail': 're:https://.+',
+            'thumbnail': 're:https?://.+',
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://vod.tvp.pl/filmy-dokumentalne,163/ukrainski-sluga-narodu,339667',
         'info_dict': {
-            'id': '51640077',
+            'id': '339667',
             'ext': 'mp4',
-            'title': 'Ukraiński sługa narodu, Ukraiński sługa narodu',
-            'series': 'Ukraiński sługa narodu',
+            'title': 'Ukraiński sługa narodu',
             'description': 'md5:b7940c0a8e439b0c81653a986f544ef3',
             'age_limit': 12,
-            'episode': 'Episode 0',
-            'episode_number': 0,
             'duration': 3051,
-            'thumbnail': 're:https://.+',
+            'thumbnail': 're:https?://.+',
+            'subtitles': 'count:2',
         },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'embed fails with "payment required"',
+        'url': 'https://vod.tvp.pl/seriale,18/polowanie-na-cmy-odcinki,390116/odcinek-7,S01E07,398869',
+        'info_dict': {
+            'id': '398869',
+            'ext': 'mp4',
+            'title': 'odc. 7',
+            'description': 'md5:dd2bb33f023dc5c2fbaddfbe4cb5dba0',
+            'duration': 2750,
+            'age_limit': 16,
+            'series': 'Polowanie na ćmy',
+            'episode_number': 7,
+            'episode': 'Episode 7',
+            'thumbnail': 're:https?://.+',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        return self._parse_video(self._call_api(f'vods/{video_id}', video_id))
+        info_dict = self._parse_video(self._call_api(f'vods/{video_id}', video_id), with_url=False)
+
+        playlist = self._call_api(f'{video_id}/videos/playlist', video_id, query={'videoType': 'MOVIE'})
+
+        info_dict['formats'] = []
+        for manifest_url in traverse_obj(playlist, ('sources', 'HLS', ..., 'src')):
+            info_dict['formats'].extend(self._extract_m3u8_formats(manifest_url, video_id, fatal=False))
+        for manifest_url in traverse_obj(playlist, ('sources', 'DASH', ..., 'src')):
+            info_dict['formats'].extend(self._extract_mpd_formats(manifest_url, video_id, fatal=False))
+
+        info_dict['subtitles'] = {}
+        for sub in playlist.get('subtitles') or []:
+            info_dict['subtitles'].setdefault(sub.get('language') or 'und', []).append({
+                'url': sub['url'],
+                'ext': 'ttml',
+            })
+
+        return info_dict
 
 
 class TVPVODSeriesIE(TVPVODBaseIE):
@@ -551,7 +597,7 @@ class TVPVODSeriesIE(TVPVODBaseIE):
             'age_limit': 12,
             'categories': ['seriale'],
         },
-        'playlist_count': 129,
+        'playlist_count': 130,
     }, {
         'url': 'https://vod.tvp.pl/programy,88/rolnik-szuka-zony-odcinki,284514',
         'only_matching': True,

From c449c0655d7c8549e6e1389c26b628053b253d39 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Sat, 6 May 2023 18:14:40 +0900
Subject: [PATCH 253/871] [extractor/abematv] Add fallback for title and
 description extraction and extract more metadata (#6994)

Authored by: Lesmiscore
---
 yt_dlp/extractor/abematv.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index f611c1f2c..c9166b6b8 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -436,6 +436,16 @@ def _real_extract(self, url):
             if 3 not in ondemand_types:
                 # cannot acquire decryption key for these streams
                 self.report_warning('This is a premium-only stream')
+            info.update(traverse_obj(api_response, {
+                'series': ('series', 'title'),
+                'season': ('season', 'title'),
+                'season_number': ('season', 'sequence'),
+                'episode_number': ('episode', 'number'),
+            }))
+            if not title:
+                title = traverse_obj(api_response, ('episode', 'title'))
+            if not description:
+                description = traverse_obj(api_response, ('episode', 'content'))
 
             m3u8_url = f'https://vod-abematv.akamaized.net/program/{video_id}/playlist.m3u8'
         elif video_type == 'slots':

From 3b52a606881e6adadc33444abdeacce562b79330 Mon Sep 17 00:00:00 2001
From: ringus1 <ringus1@users.noreply.github.com>
Date: Tue, 9 May 2023 01:19:42 +0200
Subject: [PATCH 254/871] [extractor/facebook] Fix metadata extraction (#6856)

Closes #3432
Authored by: ringus1
---
 yt_dlp/extractor/facebook.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 1404be612..9d871eb28 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -390,7 +390,10 @@ def extract_metadata(webpage):
                 k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')), expected_type=dict)
             title = get_first(media, ('title', 'text'))
             description = get_first(media, ('creation_story', 'comet_sections', 'message', 'story', 'message', 'text'))
-            uploader_data = get_first(media, 'owner') or get_first(post, ('node', 'actors', ...)) or {}
+            uploader_data = (
+                get_first(media, ('owner', {dict}))
+                or get_first(post, (..., 'video', lambda k, v: k == 'owner' and v['name']))
+                or get_first(post, ('node', 'actors', ..., {dict})) or {})
 
             page_title = title or self._html_search_regex((
                 r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>(?P<content>[^<]*)</h2>',
@@ -415,16 +418,17 @@ def extract_metadata(webpage):
             # in https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/
             if thumbnail and not re.search(r'\.(?:jpg|png)', thumbnail):
                 thumbnail = None
-            view_count = parse_count(self._search_regex(
-                r'\bviewCount\s*:\s*["\']([\d,.]+)', webpage, 'view count',
-                default=None))
             info_dict = {
                 'description': description,
                 'uploader': uploader,
                 'uploader_id': uploader_data.get('id'),
                 'timestamp': timestamp,
                 'thumbnail': thumbnail,
-                'view_count': view_count,
+                'view_count': parse_count(self._search_regex(
+                    (r'\bviewCount\s*:\s*["\']([\d,.]+)', r'video_view_count["\']\s*:\s*(\d+)',),
+                    webpage, 'view count', default=None)),
+                'concurrent_view_count': get_first(post, (
+                    ('video', (..., ..., 'attachments', ..., 'media')), 'liveViewerCount', {int_or_none})),
             }
 
             info_json_ld = self._search_json_ld(webpage, video_id, default={})

From ef8fb7f029b816dfc95600727d84400591a3b5c5 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Mon, 8 May 2023 18:45:31 -0500
Subject: [PATCH 255/871] [extractor/wrestleuniverse] Fix extraction, add login
 (#6982)

Closes #6975
Authored by: bashonly, Grub4K

Co-authored-by: Simon Sawicki <contact@grub4k.xyz>
---
 README.md                           |   3 +
 yt_dlp/extractor/wrestleuniverse.py | 137 +++++++++++++++++++++-------
 2 files changed, 105 insertions(+), 35 deletions(-)

diff --git a/README.md b/README.md
index c1f34235d..993ac5a5f 100644
--- a/README.md
+++ b/README.md
@@ -1835,6 +1835,9 @@ #### rokfinchannel
 #### twitter
 * `legacy_api`: Force usage of the legacy Twitter API instead of the GraphQL API for tweet extraction. Has no effect if login cookies are passed
 
+### wrestleuniverse
+* `device_id`: UUID value assigned by the website and used to enforce device limits for paid livestream content. Can be found in browser local storage
+
 **Note**: These options may be changed/removed in the future without concern for backward compatibility
 
 <!-- MANPAGE: MOVE "INSTALLATION" SECTION HERE -->
diff --git a/yt_dlp/extractor/wrestleuniverse.py b/yt_dlp/extractor/wrestleuniverse.py
index 5c6dec2c4..946edf20a 100644
--- a/yt_dlp/extractor/wrestleuniverse.py
+++ b/yt_dlp/extractor/wrestleuniverse.py
@@ -2,6 +2,7 @@
 import binascii
 import json
 import time
+import uuid
 
 from .common import InfoExtractor
 from ..dependencies import Cryptodome
@@ -12,30 +13,95 @@
     traverse_obj,
     try_call,
     url_or_none,
+    urlencode_postdata,
 )
 
 
 class WrestleUniverseBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'wrestleuniverse'
     _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
     _API_PATH = None
-    _TOKEN = None
+    _REAL_TOKEN = None
     _TOKEN_EXPIRY = None
+    _REFRESH_TOKEN = None
+    _DEVICE_ID = None
+    _LOGIN_QUERY = {'key': 'AIzaSyCaRPBsDQYVDUWWBXjsTrHESi2r_F3RAdA'}
+    _LOGIN_HEADERS = {
+        'Accept': '*/*',
+        'Content-Type': 'application/json',
+        'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web',
+        'X-Firebase-gmpid': '1:307308870738:web:820f38fe5150c8976e338b',
+        'Referer': 'https://www.wrestle-universe.com/',
+        'Origin': 'https://www.wrestle-universe.com',
+    }
 
-    def _get_token_cookie(self):
-        if not self._TOKEN or not self._TOKEN_EXPIRY:
-            self._TOKEN = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
-            if not self._TOKEN:
+    @property
+    def _TOKEN(self):
+        if not self._REAL_TOKEN or not self._TOKEN_EXPIRY:
+            token = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
+            if not token and not self._REFRESH_TOKEN:
                 self.raise_login_required()
-            expiry = traverse_obj(jwt_decode_hs256(self._TOKEN), ('exp', {int_or_none}))
-            if not expiry:
-                raise ExtractorError('There was a problem with the token cookie')
-            self._TOKEN_EXPIRY = expiry
+            self._REAL_TOKEN = token
 
-        if self._TOKEN_EXPIRY <= int(time.time()):
-            raise ExtractorError(
-                'Expired token. Refresh your cookies in browser and try again', expected=True)
+        if not self._REAL_TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
+            if not self._REFRESH_TOKEN:
+                raise ExtractorError(
+                    'Expired token. Refresh your cookies in browser and try again', expected=True)
+            self._refresh_token()
 
-        return self._TOKEN
+        return self._REAL_TOKEN
+
+    @_TOKEN.setter
+    def _TOKEN(self, value):
+        self._REAL_TOKEN = value
+
+        expiry = traverse_obj(value, ({jwt_decode_hs256}, 'exp', {int_or_none}))
+        if not expiry:
+            raise ExtractorError('There was a problem with the auth token')
+        self._TOKEN_EXPIRY = expiry
+
+    def _perform_login(self, username, password):
+        login = self._download_json(
+            'https://identitytoolkit.googleapis.com/v1/accounts:signInWithPassword', None,
+            'Logging in', query=self._LOGIN_QUERY, headers=self._LOGIN_HEADERS, data=json.dumps({
+                'returnSecureToken': True,
+                'email': username,
+                'password': password,
+            }, separators=(',', ':')).encode())
+        self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str}))
+        if not self._REFRESH_TOKEN:
+            self.report_warning('No refresh token was granted')
+        self._TOKEN = traverse_obj(login, ('idToken', {str}))
+
+    def _real_initialize(self):
+        if WrestleUniverseBaseIE._DEVICE_ID:
+            return
+
+        WrestleUniverseBaseIE._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key='WrestleUniverse')[0]
+        if not WrestleUniverseBaseIE._DEVICE_ID:
+            WrestleUniverseBaseIE._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
+            if WrestleUniverseBaseIE._DEVICE_ID:
+                return
+            WrestleUniverseBaseIE._DEVICE_ID = str(uuid.uuid4())
+
+        self.cache.store(self._NETRC_MACHINE, 'device_id', WrestleUniverseBaseIE._DEVICE_ID)
+
+    def _refresh_token(self):
+        refresh = self._download_json(
+            'https://securetoken.googleapis.com/v1/token', None, 'Refreshing token',
+            query=self._LOGIN_QUERY, data=urlencode_postdata({
+                'grant_type': 'refresh_token',
+                'refresh_token': self._REFRESH_TOKEN,
+            }), headers={
+                **self._LOGIN_HEADERS,
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+        if traverse_obj(refresh, ('refresh_token', {str})):
+            self._REFRESH_TOKEN = refresh['refresh_token']
+        token = traverse_obj(refresh, 'access_token', 'id_token', expected_type=str)
+        if not token:
+            raise ExtractorError('No auth token returned from refresh request')
+        self._TOKEN = token
 
     def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True):
         headers = {'CA-CID': ''}
@@ -43,7 +109,7 @@ def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={
             headers['Content-Type'] = 'application/json;charset=utf-8'
             data = json.dumps(data, separators=(',', ':')).encode()
         if auth:
-            headers['Authorization'] = f'Bearer {self._get_token_cookie()}'
+            headers['Authorization'] = f'Bearer {self._TOKEN}'
         return self._download_json(
             f'https://api.wrestle-universe.com/v1/{self._API_PATH}/{video_id}{param}', video_id,
             note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
@@ -65,7 +131,7 @@ def decrypt(data):
 
         token = base64.b64encode(private_key.public_key().export_key('DER')).decode()
         api_json = self._call_api(video_id, param, msg, data={
-            # 'deviceId' (random uuid4 generated at login) is not required yet
+            'deviceId': self._DEVICE_ID,
             'token': token,
             **data,
         }, query=query, fatal=fatal)
@@ -105,7 +171,7 @@ class WrestleUniverseVODIE(WrestleUniverseBaseIE):
             'upload_date': '20230129',
             'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN',
             'chapters': 'count:7',
-            'cast': 'count:18',
+            'cast': 'count:21',
         },
         'params': {
             'skip_download': 'm3u8',
@@ -169,6 +235,7 @@ class WrestleUniversePPVIE(WrestleUniverseBaseIE):
         'params': {
             'skip_download': 'm3u8',
         },
+        'skip': 'No longer available',
     }, {
         'note': 'unencrypted HLS',
         'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx',
@@ -196,14 +263,17 @@ def _real_extract(self, url):
         lang, video_id = self._match_valid_url(url).group('lang', 'id')
         metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData')
 
-        info = traverse_obj(metadata, {
-            'title': ('displayName', {str}),
-            'description': ('description', {str}),
-            'channel': ('labels', 'group', {str}),
-            'location': ('labels', 'venue', {str}),
-            'timestamp': ('startTime', {int_or_none}),
-            'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
-        })
+        info = {
+            'id': video_id,
+            **traverse_obj(metadata, {
+                'title': ('displayName', {str}),
+                'description': ('description', {str}),
+                'channel': ('labels', 'group', {str}),
+                'location': ('labels', 'venue', {str}),
+                'timestamp': ('startTime', {int_or_none}),
+                'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
+            }),
+        }
 
         ended_time = traverse_obj(metadata, ('endedTime', {int_or_none}))
         if info.get('timestamp') and ended_time:
@@ -211,23 +281,20 @@ def _real_extract(self, url):
 
         video_data, decrypt = self._call_encrypted_api(
             video_id, ':watchArchive', 'watch archive', data={'method': 1})
-        formats = self._get_formats(video_data, (
+        info['formats'] = self._get_formats(video_data, (
             ('hls', None), ('urls', 'chromecastUrls'), ..., {url_or_none}), video_id)
-        for f in formats:
+        for f in info['formats']:
             # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
             if f.get('tbr'):
                 f['tbr'] = int(f['tbr'] / 2.5)
 
         hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt}))
-        if not hls_aes_key and traverse_obj(video_data, ('hls', 'encryptType', {int}), default=0) > 0:
-            self.report_warning('HLS AES-128 key was not found in API response')
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'hls_aes': {
+        if hls_aes_key:
+            info['hls_aes'] = {
                 'key': hls_aes_key,
                 'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
             },
-            **info,
-        }
+        elif traverse_obj(video_data, ('hls', 'encryptType', {int})):
+            self.report_warning('HLS AES-128 key was not found in API response')
+
+        return info

From 21b9413cf7dd4830b2ece57af21589dd4538fc52 Mon Sep 17 00:00:00 2001
From: toomyzoom <52140413+toomyzoom@users.noreply.github.com>
Date: Thu, 11 May 2023 02:48:35 -0700
Subject: [PATCH 256/871] [extractor/iwara] Implement login (#6721)

Authored by: toomyzoom
---
 yt_dlp/extractor/iwara.py | 88 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 83 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/iwara.py b/yt_dlp/extractor/iwara.py
index a5aad26ee..bdc39a7dd 100644
--- a/yt_dlp/extractor/iwara.py
+++ b/yt_dlp/extractor/iwara.py
@@ -1,6 +1,7 @@
 import functools
 import urllib.parse
 import hashlib
+import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -14,7 +15,49 @@
 )
 
 
-class IwaraIE(InfoExtractor):
+# https://github.com/yt-dlp/yt-dlp/issues/6671
+class IwaraBaseIE(InfoExtractor):
+    _USERTOKEN = None
+    _MEDIATOKEN = None
+    _NETRC_MACHINE = 'iwara'
+
+    def _get_user_token(self, invalidate=False):
+        if not invalidate and self._USERTOKEN:
+            return self._USERTOKEN
+
+        username, password = self._get_login_info()
+        IwaraBaseIE._USERTOKEN = username and self.cache.load(self._NETRC_MACHINE, username)
+        if not IwaraBaseIE._USERTOKEN or invalidate:
+            IwaraBaseIE._USERTOKEN = self._download_json(
+                'https://api.iwara.tv/user/login', None, note='Logging in',
+                data=json.dumps({
+                    'email': username,
+                    'password': password
+                }).encode('utf-8'),
+                headers={
+                    'Content-Type': 'application/json'
+                })['token']
+
+            self.cache.store(self._NETRC_MACHINE, username, IwaraBaseIE._USERTOKEN)
+
+        return self._USERTOKEN
+
+    def _get_media_token(self, invalidate=False):
+        if not invalidate and self._MEDIATOKEN:
+            return self._MEDIATOKEN
+
+        IwaraBaseIE._MEDIATOKEN = self._download_json(
+            'https://api.iwara.tv/user/token', None, note='Fetching media token',
+            data=b'',  # Need to have some data here, even if it's empty
+            headers={
+                'Authorization': f'Bearer {self._get_user_token()}',
+                'Content-Type': 'application/json'
+            })['accessToken']
+
+        return self._MEDIATOKEN
+
+
+class IwaraIE(IwaraBaseIE):
     IE_NAME = 'iwara'
     _VALID_URL = r'https?://(?:www\.|ecchi\.)?iwara\.tv/videos?/(?P<id>[a-zA-Z0-9]+)'
     _TESTS = [{
@@ -56,6 +99,26 @@ class IwaraIE(InfoExtractor):
             'timestamp': 1678732213,
             'modified_timestamp': 1679110271,
         },
+    }, {
+        'url': 'https://iwara.tv/video/blggmfno8ghl725bg',
+        'info_dict': {
+            'id': 'blggmfno8ghl725bg',
+            'ext': 'mp4',
+            'age_limit': 18,
+            'title': 'お外でおしっこしちゃう猫耳ロリメイド',
+            'description': 'md5:0342ba9bf6db09edbbb28729657c3611',
+            'uploader': 'Fe_Kurosabi',
+            'uploader_id': 'fekurosabi',
+            'tags': [
+                'pee'
+            ],
+            'like_count': 192,
+            'view_count': 12119,
+            'comment_count': 0,
+            'timestamp': 1598880567,
+            'modified_timestamp': 1598908995,
+            'availability': 'needs_auth',
+        },
     }]
 
     def _extract_formats(self, video_id, fileurl):
@@ -79,12 +142,18 @@ def _extract_formats(self, video_id, fileurl):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(f'https://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True)
+        username, password = self._get_login_info()
+        headers = {
+            'Authorization': f'Bearer {self._get_media_token()}',
+        } if username and password else None
+        video_data = self._download_json(f'https://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True, headers=headers)
         errmsg = video_data.get('message')
         # at this point we can actually get uploaded user info, but do we need it?
         if errmsg == 'errors.privateVideo':
             self.raise_login_required('Private video. Login if you have permissions to watch')
-        elif errmsg:
+        elif errmsg == 'errors.notFound' and not username:
+            self.raise_login_required('Video may need login to view')
+        elif errmsg:  # None if success
             raise ExtractorError(f'Iwara says: {errmsg}')
 
         if not video_data.get('fileUrl'):
@@ -112,8 +181,17 @@ def _real_extract(self, url):
             'formats': list(self._extract_formats(video_id, video_data.get('fileUrl'))),
         }
 
+    def _perform_login(self, username, password):
+        if self.cache.load(self._NETRC_MACHINE, username) and self._get_media_token():
+            self.write_debug('Skipping logging in')
+            return
 
-class IwaraUserIE(InfoExtractor):
+        IwaraBaseIE._USERTOKEN = self._get_user_token(True)
+        self._get_media_token(True)
+        self.cache.store(self._NETRC_MACHINE, username, IwaraBaseIE._USERTOKEN)
+
+
+class IwaraUserIE(IwaraBaseIE):
     _VALID_URL = r'https?://(?:www\.)?iwara\.tv/profile/(?P<id>[^/?#&]+)'
     IE_NAME = 'iwara:user'
     _PER_PAGE = 32
@@ -165,7 +243,7 @@ def _real_extract(self, url):
             playlist_id, traverse_obj(user_info, ('user', 'name')))
 
 
-class IwaraPlaylistIE(InfoExtractor):
+class IwaraPlaylistIE(IwaraBaseIE):
     # the ID is an UUID but I don't think it's necessary to write concrete regex
     _VALID_URL = r'https?://(?:www\.)?iwara\.tv/playlist/(?P<id>[0-9a-f-]+)'
     IE_NAME = 'iwara:playlist'

From c8bc203fbf3bb09914e53f0833eed622ab7edbb9 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 20 May 2023 02:35:08 +0530
Subject: [PATCH 257/871] [docs] Misc improvements

Closes #6814, closes #6940, closes #6733, closes #6923, closes #6566, closes #6726, closes #6728
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |  6 ++---
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |  2 +-
 .github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml |  6 ++---
 .github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml  |  2 +-
 .github/PULL_REQUEST_TEMPLATE.md              |  6 +++++
 CONTRIBUTING.md                               |  4 ++--
 Collaborators.md                              |  8 +++----
 README.md                                     | 23 ++++++++++---------
 yt_dlp/YoutubeDL.py                           |  5 ++--
 yt_dlp/extractor/unsupported.py               |  5 ++--
 yt_dlp/options.py                             | 10 ++++----
 11 files changed, 43 insertions(+), 34 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index cdbb86760..77b777d5a 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -1,5 +1,5 @@
-name: Broken site
-description: Report error in a supported site
+name: Broken site support
+description: Report issue with yt-dlp on a supported site
 labels: [triage, site-bug]
 body:
   - type: checkboxes
@@ -16,7 +16,7 @@ body:
       description: |
         Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
       options:
-        - label: I'm reporting that a **supported** site is broken
+        - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
         - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index bf1d97bba..122dda4f2 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -1,4 +1,4 @@
-name: Bug report
+name: Core bug report
 description: Report a bug unrelated to any particular site or extractor
 labels: [triage, bug]
 body:
diff --git a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
index 1f6f92634..a51db789f 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
@@ -1,5 +1,5 @@
-name: Broken site
-description: Report error in a supported site
+name: Broken site support
+description: Report issue with yt-dlp on a supported site
 labels: [triage, site-bug]
 body:
   %(no_skip)s
@@ -10,7 +10,7 @@ body:
       description: |
         Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
       options:
-        - label: I'm reporting that a **supported** site is broken
+        - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml
index 90f59e70b..9ab490267 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml
@@ -1,4 +1,4 @@
-name: Bug report
+name: Core bug report
 description: Report a bug unrelated to any particular site or extractor
 labels: [triage, bug]
 body:
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index c4d3e812e..cbed82173 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -40,4 +40,10 @@ ### What is the purpose of your *pull request*?
 - [ ] Core bug fix/improvement
 - [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes))
 
+
+<!-- Do NOT edit/remove anything below this! -->
+</details><details><summary>Copilot Summary</summary>  
+
+copilot:all
+
 </details>
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ae2c45423..a8587fe92 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -79,7 +79,7 @@ ###  Are you using the latest version?
 
 ###  Is the issue already documented?
 
-Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/yt-dlp/yt-dlp/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2021.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/yt-dlp/yt-dlp/search?type=Issues) of this repository. If there is an issue, subcribe to it to be notified when there is any progress. Unless you have something useful to add to the converation, please refrain from commenting.
 
 Additionally, it is also helpful to see if the issue has already been documented in the [youtube-dl issue tracker](https://github.com/ytdl-org/youtube-dl/issues). If similar issues have already been reported in youtube-dl (but not in our issue tracker), links to them can be included in your issue report here.
 
@@ -246,7 +246,7 @@ ## yt-dlp coding conventions
 
 This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code.
 
-Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hosters out of your control and this layout tends to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize dependency on the source's layout and even to make the code foresee potential future changes and be ready for that. This is important because it will allow the extractor not to break on minor layout changes thus keeping old yt-dlp versions working. Even though this breakage issue may be easily fixed by a new version of yt-dlp, this could take some time, during which the the extractor will remain broken.
+Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hosters out of your control and this layout tends to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize dependency on the source's layout and even to make the code foresee potential future changes and be ready for that. This is important because it will allow the extractor not to break on minor layout changes thus keeping old yt-dlp versions working. Even though this breakage issue may be easily fixed by a new version of yt-dlp, this could take some time, during which the extractor will remain broken.
 
 
 ### Mandatory and optional metafields
diff --git a/Collaborators.md b/Collaborators.md
index 71baf5080..a0976dd8c 100644
--- a/Collaborators.md
+++ b/Collaborators.md
@@ -8,7 +8,7 @@ # Collaborators
 ## [pukkandan](https://github.com/pukkandan)
 
 [![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/pukkandan)
-[![gh-sponsor](https://img.shields.io/badge/_-Github-red.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/pukkandan)
+[![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/pukkandan)
 
 * Owner of the fork
 
@@ -26,7 +26,7 @@ ## [shirt](https://github.com/shirt-dev)
 
 ## [coletdjnz](https://github.com/coletdjnz)
 
-[![gh-sponsor](https://img.shields.io/badge/_-Github-red.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/coletdjnz)
+[![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/coletdjnz)
 
 * Improved plugin architecture
 * YouTube improvements including: age-gate bypass, private playlists, multiple-clients (to avoid throttling) and a lot of under-the-hood improvements
@@ -44,7 +44,7 @@ ## [Ashish0804](https://github.com/Ashish0804) <sub><sup>[Inactive]</sup></sub>
 * Improved/fixed support for HiDive, HotStar, Hungama, LBRY, LinkedInLearning, Mxplayer, SonyLiv, TV2, Vimeo, VLive etc
 
 
-## [Lesmiscore](https://github.com/Lesmiscore) <sub><sup>(nao20010128nao)</sup></sub>
+## [Lesmiscore](https://github.com/Lesmiscore)
 
 **Bitcoin**: bc1qfd02r007cutfdjwjmyy9w23rjvtls6ncve7r3s  
 **Monacoin**: mona1q3tf7dzvshrhfe3md379xtvt2n22duhglv5dskr
@@ -64,7 +64,7 @@ ## [bashonly](https://github.com/bashonly)
 
 ## [Grub4K](https://github.com/Grub4K)
 
-[![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/Grub4K) [![gh-sponsor](https://img.shields.io/badge/_-Github-red.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/Grub4K)
+[![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/Grub4K) [![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/Grub4K)
 
 * `--update-to`, automated release, nightly builds
 * Rework internals like `traverse_obj`, various core refactors and bugs fixes
diff --git a/README.md b/README.md
index 993ac5a5f..6dff57b4c 100644
--- a/README.md
+++ b/README.md
@@ -85,7 +85,7 @@ # NEW FEATURES
 * **Merged with animelover1984/youtube-dl**: You get most of the features and improvements from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl) including `--write-comments`, `BiliBiliSearch`, `BilibiliChannel`, Embedding thumbnail in mp4/ogg/opus, playlist infojson etc. Note that NicoNico livestreams are not available. See [#31](https://github.com/yt-dlp/yt-dlp/pull/31) for details.
 
 * **YouTube improvements**:
-    * Supports Clips, Stories (`ytstories:<channel UCID>`), Search (including filters)**\***, YouTube Music Search, Channel-specific search, Search prefixes (`ytsearch:`, `ytsearchdate:`)**\***, Mixes, YouTube Music Albums/Channels ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723)), and Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`, `:ytnotif`)
+    * Supports Clips, Stories (`ytstories:<channel UCID>`), Search (including filters)**\***, YouTube Music Search, Channel-specific search, Search prefixes (`ytsearch:`, `ytsearchdate:`)**\***, Mixes, and Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`, `:ytnotif`)
     * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326) **\***
     * Supports some (but not all) age-gated content without cookies
     * Download livestreams from the start using `--live-from-start` (*experimental*)
@@ -179,13 +179,13 @@ # INSTALLATION
 [![All versions](https://img.shields.io/badge/-All_Versions-lightgrey.svg?style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/releases)
 <!-- MANPAGE: END EXCLUDED SECTION -->
 
-You can install yt-dlp using [the binaries](#release-files), [PIP](https://pypi.org/project/yt-dlp) or one using a third-party package manager. See [the wiki](https://github.com/yt-dlp/yt-dlp/wiki/Installation) for detailed instructions
+You can install yt-dlp using [the binaries](#release-files), [pip](https://pypi.org/project/yt-dlp) or one using a third-party package manager. See [the wiki](https://github.com/yt-dlp/yt-dlp/wiki/Installation) for detailed instructions
 
 
 ## UPDATE
 You can use `yt-dlp -U` to update if you are using the [release binaries](#release-files)
 
-If you [installed with PIP](https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip), simply re-run the same command that was used to install the program
+If you [installed with pip](https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip), simply re-run the same command that was used to install the program
 
 For other third-party package managers, see [the wiki](https://github.com/yt-dlp/yt-dlp/wiki/Installation#third-party-package-managers) or refer their documentation
 
@@ -409,7 +409,8 @@ ## General Options:
                                     configuration files
     --flat-playlist                 Do not extract the videos of a playlist,
                                     only list them
-    --no-flat-playlist              Extract the videos of a playlist
+    --no-flat-playlist              Fully extract the videos of a playlist
+                                    (default)
     --live-from-start               Download livestreams from the start.
                                     Currently only supported for YouTube
                                     (Experimental)
@@ -465,9 +466,9 @@ ## Geo-restriction:
                                     downloading
     --xff VALUE                     How to fake X-Forwarded-For HTTP header to
                                     try bypassing geographic restriction. One of
-                                    "default" (Only when known to be useful),
-                                    "never", a two-letter ISO 3166-2 country
-                                    code, or an IP block in CIDR notation
+                                    "default" (only when known to be useful),
+                                    "never", an IP block in CIDR notation, or a
+                                    two-letter ISO 3166-2 country code
 
 ## Video Selection:
     -I, --playlist-items ITEM_SPEC  Comma separated playlist_index of the items
@@ -514,7 +515,7 @@ ## Video Selection:
                                     dogs" (caseless). Use "--match-filter -" to
                                     interactively ask whether to download each
                                     video
-    --no-match-filter               Do not use any --match-filter (default)
+    --no-match-filters              Do not use any --match-filter (default)
     --break-match-filters FILTER    Same as "--match-filters" but stops the
                                     download process when a video is rejected
     --no-break-match-filters        Do not use any --break-match-filters (default)
@@ -1709,7 +1710,7 @@ # MODIFYING METADATA
 
 This option also has a few special uses:
 
-* You can download an additional URL based on the metadata of the currently downloaded video. To do this, set the field `additional_urls` to the URL that you want to download. E.g. `--parse-metadata "description:(?P<additional_urls>https?://www\.vimeo\.com/\d+)` will download the first vimeo video found in the description
+* You can download an additional URL based on the metadata of the currently downloaded video. To do this, set the field `additional_urls` to the URL that you want to download. E.g. `--parse-metadata "description:(?P<additional_urls>https?://www\.vimeo\.com/\d+)"` will download the first vimeo video found in the description
 
 * You can use this to change the metadata that is embedded in the media file. To do this, set the value of the corresponding field with a `meta_` prefix. For example, any value you set to `meta_description` field will be added to the `description` field in the file - you can use this to set a different "description" and "synopsis". To modify the metadata of individual streams, use the `meta<n>_` prefix (e.g. `meta1_language`). Any value set to the `meta_` field will overwrite all default values.
 
@@ -1883,7 +1884,7 @@ ## Installing Plugins
     * **System Plugins**
       * `/etc/yt-dlp/plugins/<package name>/yt_dlp_plugins/`
       * `/etc/yt-dlp-plugins/<package name>/yt_dlp_plugins/`
-2. **Executable location**: Plugin packages can similarly be installed in a `yt-dlp-plugins` directory under the executable location:
+2. **Executable location**: Plugin packages can similarly be installed in a `yt-dlp-plugins` directory under the executable location (recommended for portable installations):
     * Binary: where `<root-dir>/yt-dlp.exe`, `<root-dir>/yt-dlp-plugins/<package name>/yt_dlp_plugins/`
     * Source: where `<root-dir>/yt_dlp/__main__.py`, `<root-dir>/yt-dlp-plugins/<package name>/yt_dlp_plugins/`
 
@@ -2071,7 +2072,7 @@ #### Use a custom format selector
 ```python
 import yt_dlp
 
-URL = ['https://www.youtube.com/watch?v=BaW_jenozKc']
+URLS = ['https://www.youtube.com/watch?v=BaW_jenozKc']
 
 def format_selector(ctx):
     """ Select the best video and the best audio that won't result in an mkv.
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 8ee42b86a..8f52a71a9 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -190,6 +190,7 @@ class YoutubeDL:
     ap_username:       Multiple-system operator account username.
     ap_password:       Multiple-system operator account password.
     usenetrc:          Use netrc for authentication instead.
+    netrc_location:    Location of the netrc file. Defaults to ~/.netrc.
     verbose:           Print additional info to stdout.
     quiet:             Do not print messages to stdout.
     no_warnings:       Do not print out anything for warnings.
@@ -3994,7 +3995,7 @@ def _write_subtitles(self, info_dict, filename):
             # that way it will silently go on when used with unsupporting IE
             return ret
         elif not subtitles:
-            self.to_screen('[info] There\'s no subtitles for the requested languages')
+            self.to_screen('[info] There are no subtitles for the requested languages')
             return ret
         sub_filename_base = self.prepare_filename(info_dict, 'subtitle')
         if not sub_filename_base:
@@ -4048,7 +4049,7 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
         if write_all or self.params.get('writethumbnail', False):
             thumbnails = info_dict.get('thumbnails') or []
             if not thumbnails:
-                self.to_screen(f'[info] There\'s no {label} thumbnails to download')
+                self.to_screen(f'[info] There are no {label} thumbnails to download')
                 return ret
         multiple = write_all and len(thumbnails) > 1
 
diff --git a/yt_dlp/extractor/unsupported.py b/yt_dlp/extractor/unsupported.py
index a56bd284f..1bc49786f 100644
--- a/yt_dlp/extractor/unsupported.py
+++ b/yt_dlp/extractor/unsupported.py
@@ -131,8 +131,9 @@ class KnownPiracyIE(UnsupportedInfoExtractor):
     URLS = (
         r'dood\.(?:to|watch|so|pm|wf|re)',
         # Sites youtube-dl supports, but we won't
-        r'https://viewsb\.com',
-        r'https://filemoon\.sx',
+        r'viewsb\.com',
+        r'filemoon\.sx',
+        r'hentai\.animestigma\.com',
     )
 
     _TESTS = [{
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 362a648cd..dc46ce998 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -411,7 +411,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
     general.add_option(
         '--no-flat-playlist',
         action='store_false', dest='extract_flat',
-        help='Extract the videos of a playlist')
+        help='Fully extract the videos of a playlist (default)')
     general.add_option(
         '--live-from-start',
         action='store_true', dest='live_from_start',
@@ -521,11 +521,11 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         help=optparse.SUPPRESS_HELP)
     geo.add_option(
         '--xff', metavar='VALUE',
-        dest='geo_bypass', default="default",
+        dest='geo_bypass', default='default',
         help=(
             'How to fake X-Forwarded-For HTTP header to try bypassing geographic restriction. '
-            'One of "default" (Only when known to be useful), "never", '
-            'a two-letter ISO 3166-2 country code, or an IP block in CIDR notation'))
+            'One of "default" (only when known to be useful), "never", '
+            'an IP block in CIDR notation, or a two-letter ISO 3166-2 country code'))
     geo.add_option(
         '--geo-bypass',
         action='store_const', dest='geo_bypass', const='default',
@@ -617,7 +617,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
             'that contains the phrase "cats & dogs" (caseless). '
             'Use "--match-filter -" to interactively ask whether to download each video'))
     selection.add_option(
-        '--no-match-filter',
+        '--no-match-filters',
         dest='match_filter', action='store_const', const=None,
         help='Do not use any --match-filter (default)')
     selection.add_option(

From f7f7a877bf8e87fd4eb0ad2494ad948ca7691114 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 20 May 2023 04:05:22 +0530
Subject: [PATCH 258/871] [extractor/booyah] Remove extractor

Site shut down. Closes #6425
---
 yt_dlp/extractor/_extractors.py |  1 -
 yt_dlp/extractor/booyah.py      | 86 ---------------------------------
 2 files changed, 87 deletions(-)
 delete mode 100644 yt_dlp/extractor/booyah.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 974c8a254..fd2bfa9a1 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -247,7 +247,6 @@
 from .bostonglobe import BostonGlobeIE
 from .box import BoxIE
 from .boxcast import BoxCastVideoIE
-from .booyah import BooyahClipsIE
 from .bpb import BpbIE
 from .br import (
     BRIE,
diff --git a/yt_dlp/extractor/booyah.py b/yt_dlp/extractor/booyah.py
deleted file mode 100644
index 5c55f2c76..000000000
--- a/yt_dlp/extractor/booyah.py
+++ /dev/null
@@ -1,86 +0,0 @@
-from .common import InfoExtractor
-from ..utils import int_or_none, str_or_none, traverse_obj
-
-
-class BooyahBaseIE(InfoExtractor):
-    _BOOYAH_SESSION_KEY = None
-
-    def _real_initialize(self):
-        BooyahBaseIE._BOOYAH_SESSION_KEY = self._request_webpage(
-            'https://booyah.live/api/v3/auths/sessions', None, data=b'').getheader('booyah-session-key')
-
-    def _get_comments(self, video_id):
-        comment_json = self._download_json(
-            f'https://booyah.live/api/v3/playbacks/{video_id}/comments/tops', video_id,
-            headers={'Booyah-Session-Key': self._BOOYAH_SESSION_KEY}, fatal=False) or {}
-
-        return [{
-            'id': comment.get('comment_id'),
-            'author': comment.get('from_nickname'),
-            'author_id': comment.get('from_uid'),
-            'author_thumbnail': comment.get('from_thumbnail'),
-            'text': comment.get('content'),
-            'timestamp': comment.get('create_time'),
-            'like_count': comment.get('like_cnt'),
-        } for comment in comment_json.get('comment_list') or ()]
-
-
-class BooyahClipsIE(BooyahBaseIE):
-    _VALID_URL = r'https?://booyah.live/clips/(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://booyah.live/clips/13887261322952306617',
-        'info_dict': {
-            'id': '13887261322952306617',
-            'ext': 'mp4',
-            'view_count': int,
-            'duration': 30,
-            'channel_id': 90565760,
-            'like_count': int,
-            'title': 'Cayendo con estilo 😎',
-            'uploader': '♡LɪꜱGΛ​MER​',
-            'comment_count': int,
-            'uploader_id': '90565760',
-            'thumbnail': 'https://resmambet-a.akamaihd.net/mambet-storage/Clip/90565760/90565760-27204374-fba0-409d-9d7b-63a48b5c0e75.jpg',
-            'upload_date': '20220617',
-            'timestamp': 1655490556,
-            'modified_timestamp': 1655490556,
-            'modified_date': '20220617',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        json_data = self._download_json(
-            f'https://booyah.live/api/v3/playbacks/{video_id}', video_id,
-            headers={'Booyah-Session-key': self._BOOYAH_SESSION_KEY})
-
-        formats = []
-        for video_data in json_data['playback']['endpoint_list']:
-            formats.extend(({
-                'url': video_data.get('stream_url'),
-                'ext': 'mp4',
-                'height': video_data.get('resolution'),
-            }, {
-                'url': video_data.get('download_url'),
-                'ext': 'mp4',
-                'format_note': 'Watermarked',
-                'height': video_data.get('resolution'),
-                'preference': -10,
-            }))
-
-        return {
-            'id': video_id,
-            'title': traverse_obj(json_data, ('playback', 'name')),
-            'thumbnail': traverse_obj(json_data, ('playback', 'thumbnail_url')),
-            'formats': formats,
-            'view_count': traverse_obj(json_data, ('playback', 'views')),
-            'like_count': traverse_obj(json_data, ('playback', 'likes')),
-            'duration': traverse_obj(json_data, ('playback', 'duration')),
-            'comment_count': traverse_obj(json_data, ('playback', 'comment_cnt')),
-            'channel_id': traverse_obj(json_data, ('playback', 'channel_id')),
-            'uploader': traverse_obj(json_data, ('user', 'nickname')),
-            'uploader_id': str_or_none(traverse_obj(json_data, ('user', 'uid'))),
-            'modified_timestamp': int_or_none(traverse_obj(json_data, ('playback', 'update_time_ms')), 1000),
-            'timestamp': int_or_none(traverse_obj(json_data, ('playback', 'create_time_ms')), 1000),
-            '__post_extractor': self.extract_comments(video_id, self._get_comments(video_id)),
-        }

From 1d7656184c6b8aa46b29149893894b3c24f1df00 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 20 May 2023 02:57:59 +0530
Subject: [PATCH 259/871] [jsinterp] Handle `NaN` in bitwise operators

Closes #6131
---
 test/test_jsinterp.py          | 10 ++++++++++
 test/test_youtube_signature.py |  4 ++++
 yt_dlp/jsinterp.py             |  7 ++++++-
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 3283657d7..26711502a 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -445,6 +445,16 @@ def test_bitwise_operators_overflow(self):
         jsi = JSInterpreter('function x(){return 1236566549 << 5}')
         self.assertEqual(jsi.call_function('x'), 915423904)
 
+    def test_bitwise_operators_typecast(self):
+        jsi = JSInterpreter('function x(){return null << 5}')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+        jsi = JSInterpreter('function x(){return undefined >> 5}')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+        jsi = JSInterpreter('function x(){return 42 << NaN}')
+        self.assertEqual(jsi.call_function('x'), 42)
+
     def test_negative(self):
         jsi = JSInterpreter("function f(){return 2    *    -2.0;}")
         self.assertEqual(jsi.call_function('f'), -4)
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index e2b3f0870..13120d97f 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -146,6 +146,10 @@
         'https://www.youtube.com/s/player/6f20102c/player_ias.vflset/en_US/base.js',
         'lE8DhoDmKqnmJJ', 'pJTTX6XyJP2BYw',
     ),
+    (
+        'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js',
+        'aCi3iElgd2kq0bxVbQ', 'QX1y8jGb2IbZ0w',
+    ),
 ]
 
 
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index 5571ecfeb..965b1c0f2 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -20,7 +20,12 @@
 
 def _js_bit_op(op):
     def zeroise(x):
-        return 0 if x in (None, JS_Undefined) else x
+        if x in (None, JS_Undefined):
+            return 0
+        with contextlib.suppress(TypeError):
+            if math.isnan(x):  # NB: NaN cannot be checked by membership
+                return 0
+        return x
 
     def wrapped(a, b):
         return op(zeroise(a), zeroise(b)) & 0xffffffff

From 6f2287cb18cbfb27518f068d868fa9390fee78ad Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 20 May 2023 03:06:23 +0530
Subject: [PATCH 260/871] [cleanup] Misc

Closes #7030, closes #6967
---
 test/helper.py         |   4 +-
 test/test_YoutubeDL.py |   8 +-
 test/test_jsinterp.py  | 558 +++++++++++++++--------------------------
 yt_dlp/YoutubeDL.py    |  65 ++---
 yt_dlp/jsinterp.py     |   2 +-
 yt_dlp/utils.py        |   8 +-
 6 files changed, 243 insertions(+), 402 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 0b90660ff..539b2f618 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -194,8 +194,8 @@ def sanitize_got_info_dict(got_dict):
         'formats', 'thumbnails', 'subtitles', 'automatic_captions', 'comments', 'entries',
 
         # Auto-generated
-        'autonumber', 'playlist', 'format_index', 'video_ext', 'audio_ext', 'duration_string', 'epoch',
-        'fulltitle', 'extractor', 'extractor_key', 'filepath', 'infojson_filename', 'original_url', 'n_entries',
+        'autonumber', 'playlist', 'format_index', 'video_ext', 'audio_ext', 'duration_string', 'epoch', 'n_entries',
+        'fulltitle', 'extractor', 'extractor_key', 'filename', 'filepath', 'infojson_filename', 'original_url',
 
         # Only live_status needs to be checked
         'is_live', 'was_live',
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 3c26bd7c6..477fd220e 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -757,7 +757,7 @@ def expect_same_infodict(out):
         test('%(id)r %(height)r', "'1234' 1080")
         test('%(ext)s-%(ext|def)d', 'mp4-def')
         test('%(width|0)04d', '0000')
-        test('a%(width|)d', 'a', outtmpl_na_placeholder='none')
+        test('a%(width|b)d', 'ab', outtmpl_na_placeholder='none')
 
         FORMATS = self.outtmpl_info['formats']
         sanitize = lambda x: x.replace(':', '：').replace('"', "＂").replace('\n', ' ')
@@ -871,12 +871,12 @@ def test_postprocessors(self):
 
         class SimplePP(PostProcessor):
             def run(self, info):
-                with open(audiofile, 'wt') as f:
+                with open(audiofile, 'w') as f:
                     f.write('EXAMPLE')
                 return [info['filepath']], info
 
         def run_pp(params, PP):
-            with open(filename, 'wt') as f:
+            with open(filename, 'w') as f:
                 f.write('EXAMPLE')
             ydl = YoutubeDL(params)
             ydl.add_post_processor(PP())
@@ -895,7 +895,7 @@ def run_pp(params, PP):
 
         class ModifierPP(PostProcessor):
             def run(self, info):
-                with open(info['filepath'], 'wt') as f:
+                with open(info['filepath'], 'w') as f:
                     f.write('MODIFIED')
                 return [], info
 
diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 26711502a..444909b84 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -14,462 +14,302 @@
 
 
 class TestJSInterpreter(unittest.TestCase):
+    def _test(self, code, ret, func='f', args=()):
+        self.assertEqual(JSInterpreter(code).call_function(func, *args), ret)
+
     def test_basic(self):
-        jsi = JSInterpreter('function x(){;}')
-        self.assertEqual(jsi.call_function('x'), None)
-
-        jsi = JSInterpreter('function x3(){return 42;}')
-        self.assertEqual(jsi.call_function('x3'), 42)
-
-        jsi = JSInterpreter('function x3(){42}')
-        self.assertEqual(jsi.call_function('x3'), None)
-
-        jsi = JSInterpreter('var x5 = function(){return 42;}')
-        self.assertEqual(jsi.call_function('x5'), 42)
-
-    def test_calc(self):
-        jsi = JSInterpreter('function x4(a){return 2*a+1;}')
-        self.assertEqual(jsi.call_function('x4', 3), 7)
-
-    def test_empty_return(self):
-        jsi = JSInterpreter('function f(){return; y()}')
+        jsi = JSInterpreter('function f(){;}')
+        self.assertEqual(repr(jsi.extract_function('f')), 'F<f>')
         self.assertEqual(jsi.call_function('f'), None)
 
-    def test_morespace(self):
-        jsi = JSInterpreter('function x (a) { return 2 * a + 1 ; }')
-        self.assertEqual(jsi.call_function('x', 3), 7)
+        self._test('function f(){return 42;}', 42)
+        self._test('function f(){42}', None)
+        self._test('var f = function(){return 42;}', 42)
 
-        jsi = JSInterpreter('function f () { x =  2  ; return x; }')
-        self.assertEqual(jsi.call_function('f'), 2)
+    def test_calc(self):
+        self._test('function f(a){return 2*a+1;}', 7, args=[3])
+
+    def test_empty_return(self):
+        self._test('function f(){return; y()}', None)
+
+    def test_morespace(self):
+        self._test('function f (a) { return 2 * a + 1 ; }', 7, args=[3])
+        self._test('function f () { x =  2  ; return x; }', 2)
 
     def test_strange_chars(self):
-        jsi = JSInterpreter('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }')
-        self.assertEqual(jsi.call_function('$_xY1', 20), 21)
+        self._test('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }',
+                   21, args=[20], func='$_xY1')
 
     def test_operators(self):
-        jsi = JSInterpreter('function f(){return 1 << 5;}')
-        self.assertEqual(jsi.call_function('f'), 32)
-
-        jsi = JSInterpreter('function f(){return 2 ** 5}')
-        self.assertEqual(jsi.call_function('f'), 32)
-
-        jsi = JSInterpreter('function f(){return 19 & 21;}')
-        self.assertEqual(jsi.call_function('f'), 17)
-
-        jsi = JSInterpreter('function f(){return 11 >> 2;}')
-        self.assertEqual(jsi.call_function('f'), 2)
-
-        jsi = JSInterpreter('function f(){return []? 2+3: 4;}')
-        self.assertEqual(jsi.call_function('f'), 5)
-
-        jsi = JSInterpreter('function f(){return 1 == 2}')
-        self.assertEqual(jsi.call_function('f'), False)
-
-        jsi = JSInterpreter('function f(){return 0 && 1 || 2;}')
-        self.assertEqual(jsi.call_function('f'), 2)
-
-        jsi = JSInterpreter('function f(){return 0 ?? 42;}')
-        self.assertEqual(jsi.call_function('f'), 0)
-
-        jsi = JSInterpreter('function f(){return "life, the universe and everything" < 42;}')
-        self.assertFalse(jsi.call_function('f'))
+        self._test('function f(){return 1 << 5;}', 32)
+        self._test('function f(){return 2 ** 5}', 32)
+        self._test('function f(){return 19 & 21;}', 17)
+        self._test('function f(){return 11 >> 2;}', 2)
+        self._test('function f(){return []? 2+3: 4;}', 5)
+        self._test('function f(){return 1 == 2}', False)
+        self._test('function f(){return 0 && 1 || 2;}', 2)
+        self._test('function f(){return 0 ?? 42;}', 0)
+        self._test('function f(){return "life, the universe and everything" < 42;}', False)
 
     def test_array_access(self):
-        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
-        self.assertEqual(jsi.call_function('f'), [5, 2, 7])
+        self._test('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}', [5, 2, 7])
 
     def test_parens(self):
-        jsi = JSInterpreter('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}')
-        self.assertEqual(jsi.call_function('f'), 7)
-
-        jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
-        self.assertEqual(jsi.call_function('f'), 9)
+        self._test('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}', 7)
+        self._test('function f(){return (1 + 2) * 3;}', 9)
 
     def test_quotes(self):
-        jsi = JSInterpreter(R'function f(){return "a\"\\("}')
-        self.assertEqual(jsi.call_function('f'), R'a"\(')
+        self._test(R'function f(){return "a\"\\("}', R'a"\(')
 
     def test_assignments(self):
-        jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
-        self.assertEqual(jsi.call_function('f'), 31)
-
-        jsi = JSInterpreter('function f(){var x = 20; x += 30 + 1; return x;}')
-        self.assertEqual(jsi.call_function('f'), 51)
-
-        jsi = JSInterpreter('function f(){var x = 20; x -= 30 + 1; return x;}')
-        self.assertEqual(jsi.call_function('f'), -11)
+        self._test('function f(){var x = 20; x = 30 + 1; return x;}', 31)
+        self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51)
+        self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11)
 
     def test_comments(self):
         'Skipping: Not yet fully implemented'
         return
-        jsi = JSInterpreter('''
-        function x() {
-            var x = /* 1 + */ 2;
-            var y = /* 30
-            * 40 */ 50;
-            return x + y;
-        }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 52)
+        self._test('''
+            function f() {
+                var x = /* 1 + */ 2;
+                var y = /* 30
+                * 40 */ 50;
+                return x + y;
+            }
+        ''', 52)
 
-        jsi = JSInterpreter('''
-        function f() {
-            var x = "/*";
-            var y = 1 /* comment */ + 2;
-            return y;
-        }
-        ''')
-        self.assertEqual(jsi.call_function('f'), 3)
+        self._test('''
+            function f() {
+                var x = "/*";
+                var y = 1 /* comment */ + 2;
+                return y;
+            }
+        ''', 3)
 
     def test_precedence(self):
-        jsi = JSInterpreter('''
-        function x() {
-            var a = [10, 20, 30, 40, 50];
-            var b = 6;
-            a[0]=a[b%a.length];
-            return a;
-        }''')
-        self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
+        self._test('''
+            function f() {
+                var a = [10, 20, 30, 40, 50];
+                var b = 6;
+                a[0]=a[b%a.length];
+                return a;
+            }
+        ''', [20, 20, 30, 40, 50])
 
     def test_builtins(self):
-        jsi = JSInterpreter('''
-        function x() { return NaN }
-        ''')
-        self.assertTrue(math.isnan(jsi.call_function('x')))
+        jsi = JSInterpreter('function f() { return NaN }')
+        self.assertTrue(math.isnan(jsi.call_function('f')))
 
-        jsi = JSInterpreter('''
-        function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 86000)
-        jsi = JSInterpreter('''
-        function x(dt) { return new Date(dt) - 0; }
-        ''')
-        self.assertEqual(jsi.call_function('x', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
+        self._test('function f() { return new Date("Wednesday 31 December 1969 18:01:26 MDT") - 0; }',
+                   86000)
+        self._test('function f(dt) { return new Date(dt) - 0; }',
+                   86000, args=['Wednesday 31 December 1969 18:01:26 MDT'])
 
     def test_call(self):
         jsi = JSInterpreter('''
-        function x() { return 2; }
-        function y(a) { return x() + (a?a:0); }
-        function z() { return y(3); }
+            function x() { return 2; }
+            function y(a) { return x() + (a?a:0); }
+            function z() { return y(3); }
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
         self.assertEqual(jsi.call_function('y'), 2)
 
     def test_if(self):
-        jsi = JSInterpreter('''
-        function x() {
-            let a = 9;
-            if (0==0) {a++}
-            return a
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
+        self._test('''
+            function f() {
+                let a = 9;
+                if (0==0) {a++}
+                return a
+            }
+        ''', 10)
 
-        jsi = JSInterpreter('''
-        function x() {
-            if (0==0) {return 10}
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
+        self._test('''
+            function f() {
+                if (0==0) {return 10}
+            }
+        ''', 10)
 
-        jsi = JSInterpreter('''
-        function x() {
-            if (0!=0) {return 1}
-            else {return 10}
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
+        self._test('''
+            function f() {
+                if (0!=0) {return 1}
+                else {return 10}
+            }
+        ''', 10)
 
         """  # Unsupported
-        jsi = JSInterpreter('''
-        function x() {
-            if (0!=0) {return 1}
-            else if (1==0) {return 2}
-            else {return 10}
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
+        self._test('''
+            function f() {
+                if (0!=0) {return 1}
+                else if (1==0) {return 2}
+                else {return 10}
+            }
+        ''', 10)
         """
 
     def test_for_loop(self):
-        jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) {a++} return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 10)
+        self._test('function f() { a=0; for (i=0; i-10; i++) {a++} return a }', 10)
 
     def test_switch(self):
         jsi = JSInterpreter('''
-        function x(f) { switch(f){
-            case 1:f+=1;
-            case 2:f+=2;
-            case 3:f+=3;break;
-            case 4:f+=4;
-            default:f=0;
-        } return f }
+            function f(x) { switch(x){
+                case 1:x+=1;
+                case 2:x+=2;
+                case 3:x+=3;break;
+                case 4:x+=4;
+                default:x=0;
+            } return x }
         ''')
-        self.assertEqual(jsi.call_function('x', 1), 7)
-        self.assertEqual(jsi.call_function('x', 3), 6)
-        self.assertEqual(jsi.call_function('x', 5), 0)
+        self.assertEqual(jsi.call_function('f', 1), 7)
+        self.assertEqual(jsi.call_function('f', 3), 6)
+        self.assertEqual(jsi.call_function('f', 5), 0)
 
     def test_switch_default(self):
         jsi = JSInterpreter('''
-        function x(f) { switch(f){
-            case 2: f+=2;
-            default: f-=1;
-            case 5:
-            case 6: f+=6;
-            case 0: break;
-            case 1: f+=1;
-        } return f }
+            function f(x) { switch(x){
+                case 2: x+=2;
+                default: x-=1;
+                case 5:
+                case 6: x+=6;
+                case 0: break;
+                case 1: x+=1;
+            } return x }
         ''')
-        self.assertEqual(jsi.call_function('x', 1), 2)
-        self.assertEqual(jsi.call_function('x', 5), 11)
-        self.assertEqual(jsi.call_function('x', 9), 14)
+        self.assertEqual(jsi.call_function('f', 1), 2)
+        self.assertEqual(jsi.call_function('f', 5), 11)
+        self.assertEqual(jsi.call_function('f', 9), 14)
 
     def test_try(self):
-        jsi = JSInterpreter('''
-        function x() { try{return 10} catch(e){return 5} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 10)
+        self._test('function f() { try{return 10} catch(e){return 5} }', 10)
 
     def test_catch(self):
-        jsi = JSInterpreter('''
-        function x() { try{throw 10} catch(e){return 5} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 5)
+        self._test('function f() { try{throw 10} catch(e){return 5} }', 5)
 
     def test_finally(self):
-        jsi = JSInterpreter('''
-        function x() { try{throw 10} finally {return 42} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 42)
-        jsi = JSInterpreter('''
-        function x() { try{throw 10} catch(e){return 5} finally {return 42} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 42)
+        self._test('function f() { try{throw 10} finally {return 42} }', 42)
+        self._test('function f() { try{throw 10} catch(e){return 5} finally {return 42} }', 42)
 
     def test_nested_try(self):
-        jsi = JSInterpreter('''
-        function x() {try {
-            try{throw 10} finally {throw 42}
-            } catch(e){return 5} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 5)
+        self._test('''
+            function f() {try {
+                try{throw 10} finally {throw 42}
+                } catch(e){return 5} }
+        ''', 5)
 
     def test_for_loop_continue(self):
-        jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 0)
+        self._test('function f() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }', 0)
 
     def test_for_loop_break(self):
-        jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { break; a++ } return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 0)
+        self._test('function f() { a=0; for (i=0; i-10; i++) { break; a++ } return a }', 0)
 
     def test_for_loop_try(self):
-        jsi = JSInterpreter('''
-        function x() {
-            for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} };
-            return 42 }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 42)
+        self._test('''
+            function f() {
+                for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} };
+                return 42 }
+        ''', 42)
 
     def test_literal_list(self):
-        jsi = JSInterpreter('''
-        function x() { return [1, 2, "asdf", [5, 6, 7]][3] }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+        self._test('function f() { return [1, 2, "asdf", [5, 6, 7]][3] }', [5, 6, 7])
 
     def test_comma(self):
-        jsi = JSInterpreter('''
-        function x() { a=5; a -= 1, a+=3; return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 7)
-
-        jsi = JSInterpreter('''
-        function x() { a=5; return (a -= 1, a+=3, a); }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 7)
-
-        jsi = JSInterpreter('''
-        function x() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 5)
+        self._test('function f() { a=5; a -= 1, a+=3; return a }', 7)
+        self._test('function f() { a=5; return (a -= 1, a+=3, a); }', 7)
+        self._test('function f() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }', 5)
 
     def test_void(self):
-        jsi = JSInterpreter('''
-        function x() { return void 42; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), None)
+        self._test('function f() { return void 42; }', None)
 
     def test_return_function(self):
         jsi = JSInterpreter('''
-        function x() { return [1, function(){return 1}][1] }
+            function f() { return [1, function(){return 1}][1] }
         ''')
-        self.assertEqual(jsi.call_function('x')([]), 1)
+        self.assertEqual(jsi.call_function('f')([]), 1)
 
     def test_null(self):
-        jsi = JSInterpreter('''
-        function x() { return null; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), None)
-
-        jsi = JSInterpreter('''
-        function x() { return [null > 0, null < 0, null == 0, null === 0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [null >= 0, null <= 0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [True, True])
+        self._test('function f() { return null; }', None)
+        self._test('function f() { return [null > 0, null < 0, null == 0, null === 0]; }',
+                   [False, False, False, False])
+        self._test('function f() { return [null >= 0, null <= 0]; }', [True, True])
 
     def test_undefined(self):
-        jsi = JSInterpreter('''
-        function x() { return undefined === undefined; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), True)
+        self._test('function f() { return undefined === undefined; }', True)
+        self._test('function f() { return undefined; }', JS_Undefined)
+        self._test('function f() {return undefined ?? 42; }', 42)
+        self._test('function f() { let v; return v; }', JS_Undefined)
+        self._test('function f() { let v; return v**0; }', 1)
+        self._test('function f() { let v; return [v>42, v<=42, v&&42, 42&&v]; }',
+                   [False, False, JS_Undefined, JS_Undefined])
+
+        self._test('''
+            function f() { return [
+                undefined === undefined,
+                undefined == undefined,
+                undefined == null,
+                undefined < undefined,
+                undefined > undefined,
+                undefined === 0,
+                undefined == 0,
+                undefined < 0,
+                undefined > 0,
+                undefined >= 0,
+                undefined <= 0,
+                undefined > null,
+                undefined < null,
+                undefined === null
+            ]; }
+        ''', list(map(bool, (1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))))
 
         jsi = JSInterpreter('''
-        function x() { return undefined; }
+            function f() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
         ''')
-        self.assertEqual(jsi.call_function('x'), JS_Undefined)
-
-        jsi = JSInterpreter('''
-        function x() { let v; return v; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), JS_Undefined)
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined === undefined, undefined == undefined, undefined < undefined, undefined > undefined]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [True, True, False, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined === 0, undefined == 0, undefined < 0, undefined > 0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined >= 0, undefined <= 0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined > null, undefined < null, undefined == null, undefined === null]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, True, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined === null, undefined == null, undefined < null, undefined > null]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, True, False, False])
-
-        jsi = JSInterpreter('''
-        function x() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
-        ''')
-        for y in jsi.call_function('x'):
+        for y in jsi.call_function('f'):
             self.assertTrue(math.isnan(y))
 
-        jsi = JSInterpreter('''
-        function x() { let v; return v**0; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 1)
-
-        jsi = JSInterpreter('''
-        function x() { let v; return [v>42, v<=42, v&&42, 42&&v]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, JS_Undefined, JS_Undefined])
-
-        jsi = JSInterpreter('function x(){return undefined ?? 42; }')
-        self.assertEqual(jsi.call_function('x'), 42)
-
     def test_object(self):
-        jsi = JSInterpreter('''
-        function x() { return {}; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), {})
-
-        jsi = JSInterpreter('''
-        function x() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [42, 0])
-
-        jsi = JSInterpreter('''
-        function x() { let a; return a?.qq; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), JS_Undefined)
-
-        jsi = JSInterpreter('''
-        function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), JS_Undefined)
+        self._test('function f() { return {}; }', {})
+        self._test('function f() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }', [42, 0])
+        self._test('function f() { let a; return a?.qq; }', JS_Undefined)
+        self._test('function f() { let a = {m1: 42, m2: 0 }; return a?.qq; }', JS_Undefined)
 
     def test_regex(self):
-        jsi = JSInterpreter('''
-        function x() { let a=/,,[/,913,/](,)}/; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), None)
+        self._test('function f() { let a=/,,[/,913,/](,)}/; }', None)
 
-        jsi = JSInterpreter('''
-        function x() { let a=/,,[/,913,/](,)}/; return a; }
-        ''')
-        self.assertIsInstance(jsi.call_function('x'), re.Pattern)
+        jsi = JSInterpreter('function f() { let a=/,,[/,913,/](,)}/; return a; }')
+        self.assertIsInstance(jsi.call_function('f'), re.Pattern)
 
-        jsi = JSInterpreter('''
-        function x() { let a=/,,[/,913,/](,)}/i; return a; }
-        ''')
-        self.assertEqual(jsi.call_function('x').flags & re.I, re.I)
+        jsi = JSInterpreter('function f() { let a=/,,[/,913,/](,)}/i; return a; }')
+        self.assertEqual(jsi.call_function('f').flags & re.I, re.I)
 
-        jsi = JSInterpreter(R'''
-        function x() { let a=/,][}",],()}(\[)/; return a; }
-        ''')
-        self.assertEqual(jsi.call_function('x').pattern, r',][}",],()}(\[)')
+        jsi = JSInterpreter(R'function f() { let a=/,][}",],()}(\[)/; return a; }')
+        self.assertEqual(jsi.call_function('f').pattern, r',][}",],()}(\[)')
 
-        jsi = JSInterpreter(R'''
-        function x() { let a=[/[)\\]/]; return a[0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x').pattern, r'[)\\]')
+        jsi = JSInterpreter(R'function f() { let a=[/[)\\]/]; return a[0]; }')
+        self.assertEqual(jsi.call_function('f').pattern, r'[)\\]')
 
     def test_char_code_at(self):
-        jsi = JSInterpreter('function x(i){return "test".charCodeAt(i)}')
-        self.assertEqual(jsi.call_function('x', 0), 116)
-        self.assertEqual(jsi.call_function('x', 1), 101)
-        self.assertEqual(jsi.call_function('x', 2), 115)
-        self.assertEqual(jsi.call_function('x', 3), 116)
-        self.assertEqual(jsi.call_function('x', 4), None)
-        self.assertEqual(jsi.call_function('x', 'not_a_number'), 116)
+        jsi = JSInterpreter('function f(i){return "test".charCodeAt(i)}')
+        self.assertEqual(jsi.call_function('f', 0), 116)
+        self.assertEqual(jsi.call_function('f', 1), 101)
+        self.assertEqual(jsi.call_function('f', 2), 115)
+        self.assertEqual(jsi.call_function('f', 3), 116)
+        self.assertEqual(jsi.call_function('f', 4), None)
+        self.assertEqual(jsi.call_function('f', 'not_a_number'), 116)
 
     def test_bitwise_operators_overflow(self):
-        jsi = JSInterpreter('function x(){return -524999584 << 5}')
-        self.assertEqual(jsi.call_function('x'), 379882496)
-
-        jsi = JSInterpreter('function x(){return 1236566549 << 5}')
-        self.assertEqual(jsi.call_function('x'), 915423904)
+        self._test('function f(){return -524999584 << 5}', 379882496)
+        self._test('function f(){return 1236566549 << 5}', 915423904)
 
     def test_bitwise_operators_typecast(self):
-        jsi = JSInterpreter('function x(){return null << 5}')
-        self.assertEqual(jsi.call_function('x'), 0)
-
-        jsi = JSInterpreter('function x(){return undefined >> 5}')
-        self.assertEqual(jsi.call_function('x'), 0)
-
-        jsi = JSInterpreter('function x(){return 42 << NaN}')
-        self.assertEqual(jsi.call_function('x'), 42)
+        self._test('function f(){return null << 5}', 0)
+        self._test('function f(){return undefined >> 5}', 0)
+        self._test('function f(){return 42 << NaN}', 42)
 
     def test_negative(self):
-        jsi = JSInterpreter("function f(){return 2    *    -2.0;}")
-        self.assertEqual(jsi.call_function('f'), -4)
-
-        jsi = JSInterpreter('function f(){return 2    -    - -2;}')
-        self.assertEqual(jsi.call_function('f'), 0)
-
-        jsi = JSInterpreter('function f(){return 2    -    - - -2;}')
-        self.assertEqual(jsi.call_function('f'), 4)
-
-        jsi = JSInterpreter('function f(){return 2    -    + + - -2;}')
-        self.assertEqual(jsi.call_function('f'), 0)
-
-        jsi = JSInterpreter('function f(){return 2    +    - + - -2;}')
-        self.assertEqual(jsi.call_function('f'), 0)
+        self._test('function f(){return 2    *    -2.0    ;}', -4)
+        self._test('function f(){return 2    -    - -2    ;}', 0)
+        self._test('function f(){return 2    -    - - -2  ;}', 4)
+        self._test('function f(){return 2    -    + + - -2;}', 0)
+        self._test('function f(){return 2    +    - + - -2;}', 0)
 
 
 if __name__ == '__main__':
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 8f52a71a9..91aec1fe6 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -13,6 +13,7 @@
 import random
 import re
 import shutil
+import string
 import subprocess
 import sys
 import tempfile
@@ -21,7 +22,6 @@
 import traceback
 import unicodedata
 import urllib.request
-from string import Formatter, ascii_letters
 
 from .cache import Cache
 from .compat import compat_os_name, compat_shlex_quote
@@ -1079,7 +1079,7 @@ def _outtmpl_expandpath(outtmpl):
         # correspondingly that is not what we want since we need to keep
         # '%%' intact for template dict substitution step. Working around
         # with boundary-alike separator hack.
-        sep = ''.join(random.choices(ascii_letters, k=32))
+        sep = ''.join(random.choices(string.ascii_letters, k=32))
         outtmpl = outtmpl.replace('%%', f'%{sep}%').replace('$$', f'${sep}$')
 
         # outtmpl should be expand_path'ed before template dict substitution
@@ -1238,7 +1238,7 @@ def _dumpjson_default(obj):
                 return list(obj)
             return repr(obj)
 
-        class _ReplacementFormatter(Formatter):
+        class _ReplacementFormatter(string.Formatter):
             def get_field(self, field_name, args, kwargs):
                 if field_name.isdigit():
                     return args[0], -1
@@ -2068,86 +2068,86 @@ def syntax_error(note, start):
 
         def _parse_filter(tokens):
             filter_parts = []
-            for type, string, start, _, _ in tokens:
-                if type == tokenize.OP and string == ']':
+            for type, string_, start, _, _ in tokens:
+                if type == tokenize.OP and string_ == ']':
                     return ''.join(filter_parts)
                 else:
-                    filter_parts.append(string)
+                    filter_parts.append(string_)
 
         def _remove_unused_ops(tokens):
             # Remove operators that we don't use and join them with the surrounding strings.
             # E.g. 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
             ALLOWED_OPS = ('/', '+', ',', '(', ')')
             last_string, last_start, last_end, last_line = None, None, None, None
-            for type, string, start, end, line in tokens:
-                if type == tokenize.OP and string == '[':
+            for type, string_, start, end, line in tokens:
+                if type == tokenize.OP and string_ == '[':
                     if last_string:
                         yield tokenize.NAME, last_string, last_start, last_end, last_line
                         last_string = None
-                    yield type, string, start, end, line
+                    yield type, string_, start, end, line
                     # everything inside brackets will be handled by _parse_filter
-                    for type, string, start, end, line in tokens:
-                        yield type, string, start, end, line
-                        if type == tokenize.OP and string == ']':
+                    for type, string_, start, end, line in tokens:
+                        yield type, string_, start, end, line
+                        if type == tokenize.OP and string_ == ']':
                             break
-                elif type == tokenize.OP and string in ALLOWED_OPS:
+                elif type == tokenize.OP and string_ in ALLOWED_OPS:
                     if last_string:
                         yield tokenize.NAME, last_string, last_start, last_end, last_line
                         last_string = None
-                    yield type, string, start, end, line
+                    yield type, string_, start, end, line
                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
                     if not last_string:
-                        last_string = string
+                        last_string = string_
                         last_start = start
                         last_end = end
                     else:
-                        last_string += string
+                        last_string += string_
             if last_string:
                 yield tokenize.NAME, last_string, last_start, last_end, last_line
 
         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
             selectors = []
             current_selector = None
-            for type, string, start, _, _ in tokens:
+            for type, string_, start, _, _ in tokens:
                 # ENCODING is only defined in python 3.x
                 if type == getattr(tokenize, 'ENCODING', None):
                     continue
                 elif type in [tokenize.NAME, tokenize.NUMBER]:
-                    current_selector = FormatSelector(SINGLE, string, [])
+                    current_selector = FormatSelector(SINGLE, string_, [])
                 elif type == tokenize.OP:
-                    if string == ')':
+                    if string_ == ')':
                         if not inside_group:
                             # ')' will be handled by the parentheses group
                             tokens.restore_last_token()
                         break
-                    elif inside_merge and string in ['/', ',']:
+                    elif inside_merge and string_ in ['/', ',']:
                         tokens.restore_last_token()
                         break
-                    elif inside_choice and string == ',':
+                    elif inside_choice and string_ == ',':
                         tokens.restore_last_token()
                         break
-                    elif string == ',':
+                    elif string_ == ',':
                         if not current_selector:
                             raise syntax_error('"," must follow a format selector', start)
                         selectors.append(current_selector)
                         current_selector = None
-                    elif string == '/':
+                    elif string_ == '/':
                         if not current_selector:
                             raise syntax_error('"/" must follow a format selector', start)
                         first_choice = current_selector
                         second_choice = _parse_format_selection(tokens, inside_choice=True)
                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
-                    elif string == '[':
+                    elif string_ == '[':
                         if not current_selector:
                             current_selector = FormatSelector(SINGLE, 'best', [])
                         format_filter = _parse_filter(tokens)
                         current_selector.filters.append(format_filter)
-                    elif string == '(':
+                    elif string_ == '(':
                         if current_selector:
                             raise syntax_error('Unexpected "("', start)
                         group = _parse_format_selection(tokens, inside_group=True)
                         current_selector = FormatSelector(GROUP, group, [])
-                    elif string == '+':
+                    elif string_ == '+':
                         if not current_selector:
                             raise syntax_error('Unexpected "+"', start)
                         selector_1 = current_selector
@@ -2156,7 +2156,7 @@ def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, ins
                             raise syntax_error('Expected a selector', start)
                         current_selector = FormatSelector(MERGE, (selector_1, selector_2), [])
                     else:
-                        raise syntax_error(f'Operator not recognized: "{string}"', start)
+                        raise syntax_error(f'Operator not recognized: "{string_}"', start)
                 elif type == tokenize.ENDMARKER:
                     break
             if current_selector:
@@ -2898,7 +2898,7 @@ def format_tmpl(tmpl):
 
             fmt = '%({})s'
             if tmpl.startswith('{'):
-                tmpl = f'.{tmpl}'
+                tmpl, fmt = f'.{tmpl}', '%({})j'
             if tmpl.endswith('='):
                 tmpl, fmt = tmpl[:-1], '{0} = %({0})#j'
             return '\n'.join(map(fmt.format, [tmpl] if mobj.group('dict') else tmpl.split(',')))
@@ -2937,7 +2937,8 @@ def print_field(field, actual_field=None, optional=False):
         print_field('url', 'urls')
         print_field('thumbnail', optional=True)
         print_field('description', optional=True)
-        print_field('filename', optional=True)
+        if filename:
+            print_field('filename')
         if self.params.get('forceduration') and info_copy.get('duration') is not None:
             self.to_stdout(formatSeconds(info_copy['duration']))
         print_field('format')
@@ -3419,8 +3420,8 @@ def sanitize_info(info_dict, remove_private_keys=False):
         if remove_private_keys:
             reject = lambda k, v: v is None or k.startswith('__') or k in {
                 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
-                'entries', 'filepath', '_filename', 'infojson_filename', 'original_url', 'playlist_autonumber',
-                '_format_sort_fields',
+                'entries', 'filepath', '_filename', 'filename', 'infojson_filename', 'original_url',
+                'playlist_autonumber', '_format_sort_fields',
             }
         else:
             reject = lambda k, v: False
@@ -3489,7 +3490,7 @@ def run_pp(self, pp, infodict):
                 *files_to_delete, info=infodict, msg='Deleting original file %s (pass -k to keep)')
         return infodict
 
-    def run_all_pps(self, key, info, *, additional_pps=None, fatal=True):
+    def run_all_pps(self, key, info, *, additional_pps=None):
         if key != 'video':
             self._forceprint(key, info)
         for pp in (additional_pps or []) + self._pps[key]:
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index 965b1c0f2..82974fb27 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -248,7 +248,7 @@ def _separate(expr, delim=',', max_split=None):
             return
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
-        in_quote, escaping, after_op, in_regex_char_group, in_unary_op = None, False, True, False, False
+        in_quote, escaping, after_op, in_regex_char_group = None, False, True, False
         for idx, char in enumerate(expr):
             if not in_quote and char in _MATCHING_PARENS:
                 counters[_MATCHING_PARENS[char]] += 1
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 47aa75c47..190af1b7d 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3281,7 +3281,7 @@ def is_iterable_like(x, allowed_types=collections.abc.Iterable, blocked_types=NO
 
 
 def variadic(x, allowed_types=NO_DEFAULT):
-    return x if is_iterable_like(x, blocked_types=allowed_types) else (x,)
+    return x if is_iterable_like(x, blocked_types=allowed_types) else (x, )
 
 
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
@@ -5404,7 +5404,7 @@ def to_high_limit_path(path):
 
 def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='', func=IDENTITY):
     val = traverse_obj(obj, *variadic(field))
-    if (not val and val != 0) if ignore is NO_DEFAULT else val in variadic(ignore):
+    if not val if ignore is NO_DEFAULT else val in variadic(ignore):
         return default
     return template % func(val)
 
@@ -5704,8 +5704,8 @@ def traverse_dict(dictn, keys, casesense=True):
     return traverse_obj(dictn, keys, casesense=casesense, is_user_input=True, traverse_string=True)
 
 
-def get_first(obj, keys, **kwargs):
-    return traverse_obj(obj, (..., *variadic(keys)), **kwargs, get_all=False)
+def get_first(obj, *paths, **kwargs):
+    return traverse_obj(obj, *((..., *variadic(keys)) for keys in paths), **kwargs, get_all=False)
 
 
 def time_seconds(**kwargs):

From 447afb9eaa65bc677e3245c83e53a8e69c174a3c Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 20 May 2023 19:11:03 +1200
Subject: [PATCH 261/871] [extractor/youtube] Support podcasts and releases
 tabs

Closes https://github.com/yt-dlp/yt-dlp/issues/6893

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 48 ++++++++++++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 2b17751e5..d089822f6 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4639,11 +4639,19 @@ def _playlist_entries(self, video_list_renderer):
 
     def _rich_entries(self, rich_grid_renderer):
         renderer = traverse_obj(
-            rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
+            rich_grid_renderer,
+            ('content', ('videoRenderer', 'reelItemRenderer', 'playlistRenderer')), get_all=False) or {}
         video_id = renderer.get('videoId')
-        if not video_id:
+        if video_id:
+            yield self._extract_video(renderer)
+            return
+        playlist_id = renderer.get('playlistId')
+        if playlist_id:
+            yield self.url_result(
+                f'https://www.youtube.com/playlist?list={playlist_id}',
+                ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
+                video_title=self._get_text(renderer, 'title'))
             return
-        yield self._extract_video(renderer)
 
     def _video_entry(self, video_renderer):
         video_id = video_renderer.get('videoId')
@@ -6185,6 +6193,40 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader': '3Blue1Brown',
         },
         'playlist_count': 0,
+    }, {
+        # Podcasts tab, with rich entry playlistRenderers
+        'url': 'https://www.youtube.com/@99percentinvisiblepodcast/podcasts',
+        'info_dict': {
+            'id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+            'channel_id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+            'uploader_url': 'https://www.youtube.com/@99percentinvisiblepodcast',
+            'description': 'md5:3a0ed38f1ad42a68ef0428c04a15695c',
+            'title': '99 Percent Invisible - Podcasts',
+            'uploader': '99 Percent Invisible',
+            'channel_follower_count': int,
+            'channel_url': 'https://www.youtube.com/channel/UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+            'tags': [],
+            'channel': '99 Percent Invisible',
+            'uploader_id': '@99percentinvisiblepodcast',
+        },
+        'playlist_count': 1,
+    }, {
+        # Releases tab, with rich entry playlistRenderers (same as Podcasts tab)
+        'url': 'https://www.youtube.com/@AHimitsu/releases',
+        'info_dict': {
+            'id': 'UCgFwu-j5-xNJml2FtTrrB3A',
+            'channel': 'A Himitsu',
+            'uploader_url': 'https://www.youtube.com/@AHimitsu',
+            'title': 'A Himitsu - Releases',
+            'uploader_id': '@AHimitsu',
+            'uploader': 'A Himitsu',
+            'channel_id': 'UCgFwu-j5-xNJml2FtTrrB3A',
+            'tags': 'count:16',
+            'description': 'I make music',
+            'channel_url': 'https://www.youtube.com/channel/UCgFwu-j5-xNJml2FtTrrB3A',
+            'channel_follower_count': int,
+        },
+        'playlist_mincount': 10,
     }]
 
     @classmethod

From d2e84d5eb01c66fc5304e8566348d65a7be24ed7 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Mon, 3 Apr 2023 07:01:03 +0200
Subject: [PATCH 262/871] [update] Better error handling

Authored by: pukkandan
---
 yt_dlp/__init__.py | 21 +++++++++++++--------
 yt_dlp/update.py   |  7 ++++---
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 47ee3cc02..8806106d3 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -13,6 +13,7 @@
 import os
 import re
 import sys
+import traceback
 
 from .compat import compat_shlex_quote
 from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS
@@ -937,14 +938,18 @@ def _real_main(argv=None):
         if opts.rm_cachedir:
             ydl.cache.remove()
 
-        updater = Updater(ydl, opts.update_self if isinstance(opts.update_self, str) else None)
-        if opts.update_self and updater.update() and actual_use:
-            if updater.cmd:
-                return updater.restart()
-            # This code is reachable only for zip variant in py < 3.10
-            # It makes sense to exit here, but the old behavior is to continue
-            ydl.report_warning('Restart yt-dlp to use the updated version')
-            # return 100, 'ERROR: The program must exit for the update to complete'
+        try:
+            updater = Updater(ydl, opts.update_self if isinstance(opts.update_self, str) else None)
+            if opts.update_self and updater.update() and actual_use:
+                if updater.cmd:
+                    return updater.restart()
+                # This code is reachable only for zip variant in py < 3.10
+                # It makes sense to exit here, but the old behavior is to continue
+                ydl.report_warning('Restart yt-dlp to use the updated version')
+                # return 100, 'ERROR: The program must exit for the update to complete'
+        except Exception:
+            traceback.print_exc()
+            ydl._download_retcode = 100
 
         if not actual_use:
             if pre_process:
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index 5a752d716..7914de832 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -16,6 +16,7 @@
     Popen,
     cached_method,
     deprecation_warning,
+    network_exceptions,
     remove_end,
     remove_start,
     sanitized_Request,
@@ -258,8 +259,8 @@ def check_update(self):
             self.ydl.to_screen((
                 f'Available version: {self._label(self.target_channel, self.latest_version)}, ' if self.target_tag == 'latest' else ''
             ) + f'Current version: {self._label(CHANNEL, self.current_version)}')
-        except Exception:
-            return self._report_network_error('obtain version info', delim='; Please try again later or')
+        except network_exceptions as e:
+            return self._report_network_error(f'obtain version info ({e})', delim='; Please try again later or')
 
         if not is_non_updateable():
             self.ydl.to_screen(f'Current Build Hash: {_sha256_file(self.filename)}')
@@ -303,7 +304,7 @@ def update(self):
 
         try:
             newcontent = self._download(self.release_name, self._tag)
-        except Exception as e:
+        except network_exceptions as e:
             if isinstance(e, urllib.error.HTTPError) and e.code == 404:
                 return self._report_error(
                     f'The requested tag {self._label(self.target_channel, self.target_tag)} does not exist', True)

From 665472a7de3880578c0b7b3f95c71570c056368e Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 20 May 2023 21:21:32 +0200
Subject: [PATCH 263/871] [update] Implement `--update-to` repo

Authored by: Grub4K, pukkandan
---
 README.md          | 13 ++++++++-----
 yt_dlp/__init__.py |  2 +-
 yt_dlp/options.py  |  8 ++++----
 yt_dlp/update.py   | 41 ++++++++++++++++++++++++++++-------------
 4 files changed, 41 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 6dff57b4c..d0eaba747 100644
--- a/README.md
+++ b/README.md
@@ -196,12 +196,15 @@ ## UPDATE
 The `nightly` channel has releases built after each push to the master branch, and will have the most recent fixes and additions, but also have more risk of regressions. They are available in [their own repo](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases).
 
 When using `--update`/`-U`, a release binary will only update to its current channel.
-This release channel can be changed by using the `--update-to` option. `--update-to` can also be used to upgrade or downgrade to specific tags from a channel.
+`--update-to CHANNEL` can be used to switch to a different channel when a newer version is available. `--update-to [CHANNEL@]TAG` can also be used to upgrade or downgrade to specific tags from a channel.
+
+You may also use `--update-to <repository>` (`<owner>/<repository>`) to update to a channel on a completely different repository. Be careful with what repository you are updating to though, there is no verification done for binaries from different repositories.
 
 Example usage:
 * `yt-dlp --update-to nightly` change to `nightly` channel and update to its latest release
 * `yt-dlp --update-to stable@2023.02.17` upgrade/downgrade to release to `stable` channel tag `2023.02.17`
 * `yt-dlp --update-to 2023.01.06` upgrade/downgrade to tag `2023.01.06` if it exists on the current channel
+* `yt-dlp --update-to example/yt-dlp@2023.03.01` upgrade/downgrade to the release from the `example/yt-dlp` repository, tag `2023.03.01`
 
 <!-- MANPAGE: BEGIN EXCLUDED SECTION -->
 ## RELEASE FILES
@@ -360,10 +363,10 @@ ## General Options:
     -U, --update                    Update this program to the latest version
     --no-update                     Do not check for updates (default)
     --update-to [CHANNEL]@[TAG]     Upgrade/downgrade to a specific version.
-                                    CHANNEL and TAG defaults to "stable" and
-                                    "latest" respectively if omitted; See
-                                    "UPDATE" for details. Supported channels:
-                                    stable, nightly
+                                    CHANNEL can be a repository as well. CHANNEL
+                                    and TAG default to "stable" and "latest"
+                                    respectively if omitted; See "UPDATE" for
+                                    details. Supported channels: stable, nightly
     -i, --ignore-errors             Ignore download and postprocessing errors.
                                     The download will be considered successful
                                     even if the postprocessing fails
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 8806106d3..9563d784a 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -939,7 +939,7 @@ def _real_main(argv=None):
             ydl.cache.remove()
 
         try:
-            updater = Updater(ydl, opts.update_self if isinstance(opts.update_self, str) else None)
+            updater = Updater(ydl, opts.update_self)
             if opts.update_self and updater.update() and actual_use:
                 if updater.cmd:
                     return updater.restart()
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index dc46ce998..838d79fcb 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -323,7 +323,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         help='Print program version and exit')
     general.add_option(
         '-U', '--update',
-        action='store_true', dest='update_self',
+        action='store_const', dest='update_self', const=CHANNEL,
         help=format_field(
             is_non_updateable(), None, 'Check if updates are available. %s',
             default=f'Update this program to the latest {CHANNEL} version'))
@@ -335,9 +335,9 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         '--update-to',
         action='store', dest='update_self', metavar='[CHANNEL]@[TAG]',
         help=(
-            'Upgrade/downgrade to a specific version. CHANNEL and TAG defaults to '
-            f'"{CHANNEL}" and "latest" respectively if omitted; See "UPDATE" for details. '
-            f'Supported channels: {", ".join(UPDATE_SOURCES)}'))
+            'Upgrade/downgrade to a specific version. CHANNEL can be a repository as well. '
+            f'CHANNEL and TAG default to "{CHANNEL.partition("@")[0]}" and "latest" respectively if omitted; '
+            f'See "UPDATE" for details. Supported channels: {", ".join(UPDATE_SOURCES)}'))
     general.add_option(
         '-i', '--ignore-errors',
         action='store_true', dest='ignoreerrors',
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index 7914de832..6c9bdaf1c 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -129,27 +129,36 @@ def __init__(self, ydl, target=None):
         self.ydl = ydl
 
         self.target_channel, sep, self.target_tag = (target or CHANNEL).rpartition('@')
-        if not sep and self.target_tag in UPDATE_SOURCES:  # stable => stable@latest
-            self.target_channel, self.target_tag = self.target_tag, None
+        # stable => stable@latest
+        if not sep and ('/' in self.target_tag or self.target_tag in UPDATE_SOURCES):
+            self.target_channel = self.target_tag
+            self.target_tag = None
         elif not self.target_channel:
-            self.target_channel = CHANNEL
+            self.target_channel = CHANNEL.partition('@')[0]
 
         if not self.target_tag:
-            self.target_tag, self._exact = 'latest', False
+            self.target_tag = 'latest'
+            self._exact = False
         elif self.target_tag != 'latest':
             self.target_tag = f'tags/{self.target_tag}'
 
-    @property
-    def _target_repo(self):
-        try:
-            return UPDATE_SOURCES[self.target_channel]
-        except KeyError:
-            return self._report_error(
-                f'Invalid update channel {self.target_channel!r} requested. '
-                f'Valid channels are {", ".join(UPDATE_SOURCES)}', True)
+        if '/' in self.target_channel:
+            self._target_repo = self.target_channel
+            if self.target_channel not in (CHANNEL, *UPDATE_SOURCES.values()):
+                self.ydl.report_warning(
+                    f'You are switching to an {self.ydl._format_err("unofficial", "red")} executable '
+                    f'from {self.ydl._format_err(self._target_repo, self.ydl.Styles.EMPHASIS)}. '
+                    f'Run {self.ydl._format_err("at your own risk", "light red")}')
+                self.restart = self._blocked_restart
+        else:
+            self._target_repo = UPDATE_SOURCES.get(self.target_channel)
+            if not self._target_repo:
+                self._report_error(
+                    f'Invalid update channel {self.target_channel!r} requested. '
+                    f'Valid channels are {", ".join(UPDATE_SOURCES)}', True)
 
     def _version_compare(self, a, b, channel=CHANNEL):
-        if channel != self.target_channel:
+        if self._exact and channel != self.target_channel:
             return False
 
         if _VERSION_RE.fullmatch(f'{a}.{b}'):
@@ -372,6 +381,12 @@ def restart(self):
         _, _, returncode = Popen.run(self.cmd)
         return returncode
 
+    def _blocked_restart(self):
+        self._report_error(
+            'Automatically restarting into custom builds is disabled for security reasons. '
+            'Restart yt-dlp to use the updated version', expected=True)
+        return self.ydl._download_retcode
+
 
 def run_update(ydl):
     """Update the program file with the latest version from the repository

From 44a79958f0b596ee71e1eb25f158610aada29d1b Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Mon, 3 Apr 2023 07:06:27 +0200
Subject: [PATCH 264/871] [build] Fix macOS target

Authored by: Grub4K
---
 .github/workflows/build.yml | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index aa11c6194..bec0576d1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -188,21 +188,23 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      # NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used
+      # NB: Building universal2 does not work with python from actions/setup-python
       - name: Install Requirements
         run: |
           brew install coreutils
-          /usr/bin/python3 -m pip install -U --user pip Pyinstaller==5.8 -r requirements.txt
+          python3 -m pip install -U --user pip setuptools wheel
+          # We need to ignore wheels otherwise we break universal2 builds
+          python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
 
       - name: Prepare
         run: |
-          /usr/bin/python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
-          /usr/bin/python3 devscripts/make_lazy_extractors.py
+          python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
-          /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir
+          python3 pyinst.py --target-architecture universal2 --onedir
           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
-          /usr/bin/python3 pyinst.py --target-architecture universal2
+          python3 pyinst.py --target-architecture universal2
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -232,7 +234,8 @@ jobs:
       - name: Install Requirements
         run: |
           brew install coreutils
-          python3 -m pip install -U --user pip Pyinstaller -r requirements.txt
+          python3 -m pip install -U --user pip setuptools wheel
+          python3 -m pip install -U --user Pyinstaller -r requirements.txt
 
       - name: Prepare
         run: |

From c4efa0aefec8daef1de62fd1693f13edf3c8b03c Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 20 May 2023 11:08:50 -0500
Subject: [PATCH 265/871] [build] Various build workflow improvements

- Wait for build before publishing to PyPI
- Do not run `meta_files` job if release is cancelled
- Customizable channel in release workflow
- Display badges above changelog

Authored by: bashonly, Grub4K
---
 .github/workflows/build.yml           |  4 +-
 .github/workflows/publish.yml         | 46 +++++++++++------
 .github/workflows/release-nightly.yml |  3 +-
 .github/workflows/release.yml         | 72 ++++++++++++++++++++-------
 devscripts/update-version.py          |  2 +-
 5 files changed, 90 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index bec0576d1..d038e693d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -41,7 +41,7 @@ on:
         required: true
         type: string
       channel:
-        description: Update channel (stable/nightly)
+        description: Update channel (stable/nightly/...)
         required: true
         default: stable
         type: string
@@ -316,7 +316,7 @@ jobs:
             dist/yt-dlp_x86.exe
 
   meta_files:
-    if: inputs.meta_files && always()
+    if: inputs.meta_files && always() && !cancelled()
     needs:
       - unix
       - linux_arm
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 8a1bd9a01..3ca5c6992 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -2,16 +2,20 @@ name: Publish
 on:
   workflow_call:
     inputs:
-      nightly:
-        default: false
-        required: false
-        type: boolean
+      channel:
+        default: stable
+        required: true
+        type: string
       version:
         required: true
         type: string
       target_commitish:
         required: true
         type: string
+      prerelease:
+        default: false
+        required: true
+        type: boolean
     secrets:
       ARCHIVE_REPO_TOKEN:
         required: false
@@ -34,6 +38,19 @@ jobs:
 
       - name: Generate release notes
         run: |
+          printf '%s' \
+            '[![Installation](https://img.shields.io/badge/-Which%20file%20should%20I%20download%3F-white.svg?style=for-the-badge)]' \
+              '(https://github.com/yt-dlp/yt-dlp#installation "Installation instructions") ' \
+            '[![Documentation](https://img.shields.io/badge/-Docs-brightgreen.svg?style=for-the-badge&logo=GitBook&labelColor=555555)]' \
+              '(https://github.com/yt-dlp/yt-dlp/tree/2023.03.04#readme "Documentation") ' \
+            '[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)]' \
+              '(https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators "Donate") ' \
+            '[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)]' \
+              '(https://discord.gg/H5MNcFW63r "Discord") ' \
+            ${{ inputs.channel != 'nightly' && '"[![Nightly](https://img.shields.io/badge/Get%20nightly%20builds-purple.svg?style=for-the-badge)]" \
+              "(https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest \"Nightly builds\")"' || '' }} \
+            > ./RELEASE_NOTES
+          printf '\n\n' >> ./RELEASE_NOTES
           cat >> ./RELEASE_NOTES << EOF
           #### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files)
           ---
@@ -41,9 +58,9 @@ jobs:
           $(python ./devscripts/make_changelog.py -vv)
           </details>
           EOF
-          echo "**This is an automated nightly pre-release build**" >> ./PRERELEASE_NOTES
-          cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES
-          echo "Generated from: https://github.com/${{ github.repository }}/commit/${{ inputs.target_commitish }}" >> ./ARCHIVE_NOTES
+          printf '%s\n\n' '**This is an automated nightly pre-release build**' >> ./NIGHTLY_NOTES
+          cat ./RELEASE_NOTES >> ./NIGHTLY_NOTES
+          printf '%s\n\n' 'Generated from: https://github.com/${{ github.repository }}/commit/${{ inputs.target_commitish }}' >> ./ARCHIVE_NOTES
           cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES
 
       - name: Archive nightly release
@@ -51,7 +68,7 @@ jobs:
           GH_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
           GH_REPO: ${{ vars.ARCHIVE_REPO }}
         if: |
-          inputs.nightly && env.GH_TOKEN != '' && env.GH_REPO != ''
+          inputs.channel == 'nightly' && env.GH_TOKEN != '' && env.GH_REPO != ''
         run: |
           gh release create \
             --notes-file ARCHIVE_NOTES \
@@ -60,7 +77,7 @@ jobs:
             artifact/*
 
       - name: Prune old nightly release
-        if: inputs.nightly && !vars.ARCHIVE_REPO
+        if: inputs.channel == 'nightly' && !vars.ARCHIVE_REPO
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
@@ -68,14 +85,15 @@ jobs:
           git tag --delete "nightly" || true
           sleep 5  # Enough time to cover deletion race condition
 
-      - name: Publish release${{ inputs.nightly && ' (nightly)' || '' }}
+      - name: Publish release${{ inputs.channel == 'nightly' && ' (nightly)' || '' }}
         env:
           GH_TOKEN: ${{ github.token }}
-        if: (inputs.nightly && !vars.ARCHIVE_REPO) || !inputs.nightly
+        if: (inputs.channel == 'nightly' && !vars.ARCHIVE_REPO) || inputs.channel != 'nightly'
         run: |
           gh release create \
-            --notes-file ${{ inputs.nightly && 'PRE' || '' }}RELEASE_NOTES \
+            --notes-file ${{ inputs.channel == 'nightly' && 'NIGHTLY_NOTES' || 'RELEASE_NOTES' }} \
             --target ${{ inputs.target_commitish }} \
-            --title "yt-dlp ${{ inputs.nightly && 'nightly ' || '' }}${{ inputs.version }}" \
-            ${{ inputs.nightly && '--prerelease "nightly"' || inputs.version }} \
+            --title "yt-dlp ${{ inputs.channel == 'nightly' && 'nightly ' || '' }}${{ inputs.version }}" \
+            ${{ inputs.prerelease && '--prerelease' || '' }} \
+            ${{ inputs.channel == 'nightly' && '"nightly"' || inputs.version }} \
             artifact/*
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index d4f01ab64..543e2e6f7 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -46,6 +46,7 @@ jobs:
     permissions:
       contents: write
     with:
-      nightly: true
+      channel: nightly
+      prerelease: true
       version: ${{ needs.prepare.outputs.version }}
       target_commitish: ${{ github.sha }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e07fc0c07..ada508be8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,5 +1,22 @@
 name: Release
-on: workflow_dispatch
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Version tag (YYYY.MM.DD[.REV])
+        required: false
+        default: ''
+        type: string
+      channel:
+        description: Update channel (stable/nightly/...)
+        required: false
+        default: ''
+        type: string
+      prerelease:
+        description: Pre-release
+        default: false
+        type: boolean
+
 permissions:
   contents: read
 
@@ -9,8 +26,9 @@ jobs:
       contents: write
     runs-on: ubuntu-latest
     outputs:
+      channel: ${{ steps.set_channel.outputs.channel }}
       version: ${{ steps.update_version.outputs.version }}
-      head_sha: ${{ steps.push_release.outputs.head_sha }}
+      head_sha: ${{ steps.get_target.outputs.head_sha }}
 
     steps:
       - uses: actions/checkout@v3
@@ -21,10 +39,18 @@ jobs:
         with:
           python-version: "3.10"
 
+      - name: Set channel
+        id: set_channel
+        run: |
+          CHANNEL="${{ github.repository == 'yt-dlp/yt-dlp' && 'stable' || github.repository }}"
+          echo "channel=${{ inputs.channel || '$CHANNEL' }}" > "$GITHUB_OUTPUT"
+
       - name: Update version
         id: update_version
         run: |
-          python devscripts/update-version.py ${{ vars.PUSH_VERSION_COMMIT == '' && '"$(date -u +"%H%M%S")"' || '' }} | \
+          REVISION="${{ vars.PUSH_VERSION_COMMIT == '' && '$(date -u +"%H%M%S")' || '' }}"
+          REVISION="${{ inputs.prerelease && '$(date -u +"%H%M%S")' || '$REVISION' }}"
+          python devscripts/update-version.py ${{ inputs.version || '$REVISION' }} | \
             grep -Po "version=\d+\.\d+\.\d+(\.\d+)?" >> "$GITHUB_OUTPUT"
 
       - name: Update documentation
@@ -39,6 +65,7 @@ jobs:
 
       - name: Push to release
         id: push_release
+        if: ${{ !inputs.prerelease }}
         run: |
           git config --global user.name github-actions
           git config --global user.email github-actions@example.com
@@ -46,14 +73,30 @@ jobs:
           git commit -m "Release ${{ steps.update_version.outputs.version }}" \
             -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
           git push origin --force ${{ github.event.ref }}:release
+
+      - name: Get target commitish
+        id: get_target
+        run: |
           echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
 
       - name: Update master
-        if: vars.PUSH_VERSION_COMMIT != ''
+        if: vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease
         run: git push origin ${{ github.event.ref }}
 
-  publish_pypi_homebrew:
+  build:
     needs: prepare
+    uses: ./.github/workflows/build.yml
+    with:
+      version: ${{ needs.prepare.outputs.version }}
+      channel: ${{ needs.prepare.outputs.channel }}
+    permissions:
+      contents: read
+      packages: write # For package cache
+    secrets:
+      GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+
+  publish_pypi_homebrew:
+    needs: [prepare, build]
     runs-on: ubuntu-latest
 
     steps:
@@ -77,7 +120,7 @@ jobs:
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        if: env.TWINE_PASSWORD != ''
+        if: env.TWINE_PASSWORD != '' && !inputs.prerelease
         run: |
           rm -rf dist/*
           make pypi-files
@@ -89,7 +132,7 @@ jobs:
         env:
           BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
           PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != ''
+        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' && !inputs.prerelease
         uses: actions/checkout@v3
         with:
           repository: yt-dlp/homebrew-taps
@@ -100,7 +143,7 @@ jobs:
         env:
           BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
           PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != ''
+        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' && !inputs.prerelease
         run: |
           python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.version }}"
           git -C taps/ config user.name github-actions
@@ -108,22 +151,13 @@ jobs:
           git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.version }}'
           git -C taps/ push
 
-  build:
-    needs: prepare
-    uses: ./.github/workflows/build.yml
-    with:
-      version: ${{ needs.prepare.outputs.version }}
-    permissions:
-      contents: read
-      packages: write # For package cache
-    secrets:
-      GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
-
   publish:
     needs: [prepare, build]
     uses: ./.github/workflows/publish.yml
     permissions:
       contents: write
     with:
+      channel: ${{ needs.prepare.outputs.channel }}
+      prerelease: ${{ inputs.prerelease }}
       version: ${{ needs.prepare.outputs.version }}
       target_commitish: ${{ needs.prepare.outputs.head_sha }}
diff --git a/devscripts/update-version.py b/devscripts/update-version.py
index d888be881..c873d10a5 100644
--- a/devscripts/update-version.py
+++ b/devscripts/update-version.py
@@ -51,7 +51,7 @@ def get_git_head():
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Update the version.py file')
     parser.add_argument(
-        '-c', '--channel', choices=['stable', 'nightly'], default='stable',
+        '-c', '--channel', default='stable',
         help='Select update channel (default: %(default)s)')
     parser.add_argument(
         '-o', '--output', default='yt_dlp/version.py',

From b73193c99aa23b135732408a5fcf655c68d731c6 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 20 May 2023 11:12:18 -0500
Subject: [PATCH 266/871] [build] Implement build verification using
 `--update-to`

Authored by: bashonly, Grub4K
---
 .github/workflows/build.yml | 69 +++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d038e693d..ac0cfdf7c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -127,6 +127,19 @@ jobs:
           mv ./dist/yt-dlp_linux ./yt-dlp_linux
           mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip
 
+      - name: Verify --update-to
+        if: vars.UPDATE_TO_VERIFICATION
+        run: |
+          binaries=("yt-dlp" "yt-dlp_linux")
+          for binary in "${binaries[@]}"; do
+            chmod +x ./${binary}
+            cp ./${binary} ./${binary}_downgraded
+            version="$(./${binary} --version)"
+            ./${binary}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
+            downgraded_version="$(./${binary}_downgraded --version)"
+            [[ "$version" != "$downgraded_version" ]]
+          done
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -176,6 +189,16 @@ jobs:
             python3.8 devscripts/make_lazy_extractors.py
             python3.8 pyinst.py
 
+            if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then
+              arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}"
+              chmod +x ./dist/yt-dlp_linux_${arch}
+              cp ./dist/yt-dlp_linux_${arch} ./dist/yt-dlp_linux_${arch}_downgraded
+              version="$(./dist/yt-dlp_linux_${arch} --version)"
+              ./dist/yt-dlp_linux_${arch}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
+              downgraded_version="$(./dist/yt-dlp_linux_${arch}_downgraded --version)"
+              [[ "$version" != "$downgraded_version" ]]
+            fi
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -206,6 +229,16 @@ jobs:
           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
           python3 pyinst.py --target-architecture universal2
 
+      - name: Verify --update-to
+        if: vars.UPDATE_TO_VERIFICATION
+        run: |
+          chmod +x ./dist/yt-dlp_macos
+          cp ./dist/yt-dlp_macos ./dist/yt-dlp_macos_downgraded
+          version="$(./dist/yt-dlp_macos --version)"
+          ./dist/yt-dlp_macos_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
+          downgraded_version="$(./dist/yt-dlp_macos_downgraded --version)"
+          [[ "$version" != "$downgraded_version" ]]
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -246,6 +279,16 @@ jobs:
           python3 pyinst.py
           mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
 
+      - name: Verify --update-to
+        if: vars.UPDATE_TO_VERIFICATION
+        run: |
+          chmod +x ./dist/yt-dlp_macos_legacy
+          cp ./dist/yt-dlp_macos_legacy ./dist/yt-dlp_macos_legacy_downgraded
+          version="$(./dist/yt-dlp_macos_legacy --version)"
+          ./dist/yt-dlp_macos_legacy_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
+          downgraded_version="$(./dist/yt-dlp_macos_legacy_downgraded --version)"
+          [[ "$version" != "$downgraded_version" ]]
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -278,6 +321,19 @@ jobs:
           python pyinst.py --onedir
           Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
 
+      - name: Verify --update-to
+        if: vars.UPDATE_TO_VERIFICATION
+        run: |
+          foreach ($name in @("yt-dlp","yt-dlp_min")) {
+            Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
+            $version = & "./dist/${name}.exe" --version
+            & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
+            $downgraded_version = & "./dist/${name}_downgraded.exe" --version
+            if ($version -eq $downgraded_version) {
+              exit 1
+            }
+          }
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -309,6 +365,19 @@ jobs:
         run: |
           python pyinst.py
 
+      - name: Verify --update-to
+        if: vars.UPDATE_TO_VERIFICATION
+        run: |
+          foreach ($name in @("yt-dlp_x86")) {
+            Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
+            $version = & "./dist/${name}.exe" --version
+            & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
+            $downgraded_version = & "./dist/${name}_downgraded.exe" --version
+            if ($version -eq $downgraded_version) {
+              exit 1
+            }
+          }
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:

From 23c39a4beadee382060bb47fdaa21316ca707d38 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Mon, 3 Apr 2023 07:22:11 +0200
Subject: [PATCH 267/871] [devscripts] `make_changelog`: Various improvements

- Make single items collapse into one line
- Don't hide "Important changes" in `<details>`
- Move upstream merge into priority
- Properly support comma separated prefixes

Authored by: Grub4K
---
 .github/workflows/publish.yml |   4 +-
 devscripts/make_changelog.py  | 187 +++++++++++++++++++---------------
 2 files changed, 106 insertions(+), 85 deletions(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 3ca5c6992..9ebf54e7f 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -54,9 +54,7 @@ jobs:
           cat >> ./RELEASE_NOTES << EOF
           #### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files)
           ---
-          <details><summary><h3>Changelog</h3></summary>
-          $(python ./devscripts/make_changelog.py -vv)
-          </details>
+          $(python ./devscripts/make_changelog.py -vv --collapsible)
           EOF
           printf '%s\n\n' '**This is an automated nightly pre-release build**' >> ./NIGHTLY_NOTES
           cat ./RELEASE_NOTES >> ./NIGHTLY_NOTES
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index b159bc1b9..1b7e251ee 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -26,7 +26,6 @@
 
 
 class CommitGroup(enum.Enum):
-    UPSTREAM = None
     PRIORITY = 'Important'
     CORE = 'Core'
     EXTRACTOR = 'Extractor'
@@ -34,6 +33,11 @@ class CommitGroup(enum.Enum):
     POSTPROCESSOR = 'Postprocessor'
     MISC = 'Misc.'
 
+    @classmethod
+    @property
+    def ignorable_prefixes(cls):
+        return ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream')
+
     @classmethod
     @lru_cache
     def commit_lookup(cls):
@@ -41,7 +45,6 @@ def commit_lookup(cls):
             name: group
             for group, names in {
                 cls.PRIORITY: {''},
-                cls.UPSTREAM: {'upstream'},
                 cls.CORE: {
                     'aes',
                     'cache',
@@ -54,6 +57,7 @@ def commit_lookup(cls):
                     'outtmpl',
                     'plugins',
                     'update',
+                    'upstream',
                     'utils',
                 },
                 cls.MISC: {
@@ -111,22 +115,36 @@ def key(self):
         return ((self.details or '').lower(), self.sub_details, self.message)
 
 
+def unique(items):
+    return sorted({item.strip().lower(): item for item in items if item}.values())
+
+
 class Changelog:
     MISC_RE = re.compile(r'(?:^|\b)(?:lint(?:ing)?|misc|format(?:ting)?|fixes)(?:\b|$)', re.IGNORECASE)
+    ALWAYS_SHOWN = (CommitGroup.PRIORITY,)
 
-    def __init__(self, groups, repo):
+    def __init__(self, groups, repo, collapsible=False):
         self._groups = groups
         self._repo = repo
+        self._collapsible = collapsible
 
     def __str__(self):
         return '\n'.join(self._format_groups(self._groups)).replace('\t', '    ')
 
     def _format_groups(self, groups):
+        first = True
         for item in CommitGroup:
+            if self._collapsible and item not in self.ALWAYS_SHOWN and first:
+                first = False
+                yield '\n<details><summary><h3>Changelog</h3></summary>\n'
+
             group = groups[item]
             if group:
                 yield self.format_module(item.value, group)
 
+        if self._collapsible:
+            yield '\n</details>'
+
     def format_module(self, name, group):
         result = f'\n#### {name} changes\n' if name else '\n'
         return result + '\n'.join(self._format_group(group))
@@ -137,62 +155,52 @@ def _format_group(self, group):
         for _, items in detail_groups:
             items = list(items)
             details = items[0].details
-            if not details:
-                indent = ''
-            else:
-                yield f'- {details}'
-                indent = '\t'
 
             if details == 'cleanup':
-                items, cleanup_misc_items = self._filter_cleanup_misc_items(items)
+                items = self._prepare_cleanup_misc_items(items)
+
+            prefix = '-'
+            if details:
+                if len(items) == 1:
+                    prefix = f'- **{details}**:'
+                else:
+                    yield f'- **{details}**'
+                    prefix = '\t-'
 
             sub_detail_groups = itertools.groupby(items, lambda item: tuple(map(str.lower, item.sub_details)))
             for sub_details, entries in sub_detail_groups:
                 if not sub_details:
                     for entry in entries:
-                        yield f'{indent}- {self.format_single_change(entry)}'
+                        yield f'{prefix} {self.format_single_change(entry)}'
                     continue
 
                 entries = list(entries)
-                prefix = f'{indent}- {", ".join(entries[0].sub_details)}'
+                sub_prefix = f'{prefix} {", ".join(entries[0].sub_details)}'
                 if len(entries) == 1:
-                    yield f'{prefix}: {self.format_single_change(entries[0])}'
+                    yield f'{sub_prefix}: {self.format_single_change(entries[0])}'
                     continue
 
-                yield prefix
+                yield sub_prefix
                 for entry in entries:
-                    yield f'{indent}\t- {self.format_single_change(entry)}'
+                    yield f'\t{prefix} {self.format_single_change(entry)}'
 
-            if details == 'cleanup' and cleanup_misc_items:
-                yield from self._format_cleanup_misc_sub_group(cleanup_misc_items)
-
-    def _filter_cleanup_misc_items(self, items):
+    def _prepare_cleanup_misc_items(self, items):
         cleanup_misc_items = defaultdict(list)
-        non_misc_items = []
+        sorted_items = []
         for item in items:
             if self.MISC_RE.search(item.message):
                 cleanup_misc_items[tuple(item.commit.authors)].append(item)
             else:
-                non_misc_items.append(item)
+                sorted_items.append(item)
 
-        return non_misc_items, cleanup_misc_items
+        for commit_infos in cleanup_misc_items.values():
+            sorted_items.append(CommitInfo(
+                'cleanup', ('Miscellaneous',), ', '.join(
+                    self._format_message_link(None, info.commit.hash)
+                    for info in sorted(commit_infos, key=lambda item: item.commit.hash or '')),
+                [], Commit(None, '', commit_infos[0].commit.authors), []))
 
-    def _format_cleanup_misc_sub_group(self, group):
-        prefix = '\t- Miscellaneous'
-        if len(group) == 1:
-            yield f'{prefix}: {next(self._format_cleanup_misc_items(group))}'
-            return
-
-        yield prefix
-        for message in self._format_cleanup_misc_items(group):
-            yield f'\t\t- {message}'
-
-    def _format_cleanup_misc_items(self, group):
-        for authors, infos in group.items():
-            message = ', '.join(
-                self._format_message_link(None, info.commit.hash)
-                for info in sorted(infos, key=lambda item: item.commit.hash or ''))
-            yield f'{message} by {self._format_authors(authors)}'
+        return sorted_items
 
     def format_single_change(self, info):
         message = self._format_message_link(info.message, info.commit.hash)
@@ -236,12 +244,8 @@ class CommitRange:
 
     AUTHOR_INDICATOR_RE = re.compile(r'Authored by:? ', re.IGNORECASE)
     MESSAGE_RE = re.compile(r'''
-        (?:\[
-            (?P<prefix>[^\]\/:,]+)
-            (?:/(?P<details>[^\]:,]+))?
-            (?:[:,](?P<sub_details>[^\]]+))?
-        \]\ )?
-        (?:(?P<sub_details_alt>`?[^:`]+`?): )?
+        (?:\[(?P<prefix>[^\]]+)\]\ )?
+        (?:(?P<sub_details>`?[^:`]+`?): )?
         (?P<message>.+?)
         (?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
         ''', re.VERBOSE | re.DOTALL)
@@ -340,60 +344,76 @@ def apply_overrides(self, overrides):
         self._commits = {key: value for key, value in reversed(self._commits.items())}
 
     def groups(self):
-        groups = defaultdict(list)
+        group_dict = defaultdict(list)
         for commit in self:
-            upstream_re = self.UPSTREAM_MERGE_RE.match(commit.short)
+            upstream_re = self.UPSTREAM_MERGE_RE.search(commit.short)
             if upstream_re:
-                commit.short = f'[upstream] Merge up to youtube-dl {upstream_re.group(1)}'
+                commit.short = f'[upstream] Merged with youtube-dl {upstream_re.group(1)}'
 
             match = self.MESSAGE_RE.fullmatch(commit.short)
             if not match:
                 logger.error(f'Error parsing short commit message: {commit.short!r}')
                 continue
 
-            prefix, details, sub_details, sub_details_alt, message, issues = match.groups()
-            group = None
-            if prefix:
-                if prefix == 'priority':
-                    prefix, _, details = (details or '').partition('/')
-                    logger.debug(f'Priority: {message!r}')
-                    group = CommitGroup.PRIORITY
-
-                if not details and prefix:
-                    if prefix not in ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream'):
-                        logger.debug(f'Replaced details with {prefix!r}')
-                        details = prefix or None
-
-                if details == 'common':
-                    details = None
-
-                if details:
-                    details = details.strip()
-
-            else:
-                group = CommitGroup.CORE
-
-            sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.replace(':', ',')
-            sub_details = tuple(filter(None, map(str.strip, sub_details.split(','))))
-
+            prefix, sub_details_alt, message, issues = match.groups()
             issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else []
 
+            if prefix:
+                groups, details, sub_details = zip(*map(self.details_from_prefix, prefix.split(',')))
+                group = next(iter(filter(None, groups)), None)
+                details = ', '.join(unique(details))
+                sub_details = list(itertools.chain.from_iterable(sub_details))
+            else:
+                group = CommitGroup.CORE
+                details = None
+                sub_details = []
+
+            if sub_details_alt:
+                sub_details.append(sub_details_alt)
+            sub_details = tuple(unique(sub_details))
+
             if not group:
-                group = CommitGroup.get(prefix.lower())
-                if not group:
-                    if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
-                        group = CommitGroup.EXTRACTOR
-                    else:
-                        group = CommitGroup.POSTPROCESSOR
-                    logger.warning(f'Failed to map {commit.short!r}, selected {group.name}')
+                if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
+                    group = CommitGroup.EXTRACTOR
+                else:
+                    group = CommitGroup.POSTPROCESSOR
+                logger.warning(f'Failed to map {commit.short!r}, selected {group.name.lower()}')
 
             commit_info = CommitInfo(
                 details, sub_details, message.strip(),
                 issues, commit, self._fixes[commit.hash])
-            logger.debug(f'Resolved {commit.short!r} to {commit_info!r}')
-            groups[group].append(commit_info)
 
-        return groups
+            logger.debug(f'Resolved {commit.short!r} to {commit_info!r}')
+            group_dict[group].append(commit_info)
+
+        return group_dict
+
+    @staticmethod
+    def details_from_prefix(prefix):
+        if not prefix:
+            return CommitGroup.CORE, None, ()
+
+        prefix, _, details = prefix.partition('/')
+        prefix = prefix.strip().lower()
+        details = details.strip()
+
+        group = CommitGroup.get(prefix)
+        if group is CommitGroup.PRIORITY:
+            prefix, _, details = details.partition('/')
+
+        if not details and prefix and prefix not in CommitGroup.ignorable_prefixes:
+            logger.debug(f'Replaced details with {prefix!r}')
+            details = prefix or None
+
+        if details == 'common':
+            details = None
+
+        if details:
+            details, *sub_details = details.split(':')
+        else:
+            sub_details = []
+
+        return group, details, sub_details
 
 
 def get_new_contributors(contributors_path, commits):
@@ -444,6 +464,9 @@ def get_new_contributors(contributors_path, commits):
     parser.add_argument(
         '--repo', default='yt-dlp/yt-dlp',
         help='the github repository to use for the operations (default: %(default)s)')
+    parser.add_argument(
+        '--collapsible', action='store_true',
+        help='make changelog collapsible (default: %(default)s)')
     args = parser.parse_args()
 
     logging.basicConfig(
@@ -467,4 +490,4 @@ def get_new_contributors(contributors_path, commits):
             write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a')
         logger.info(f'New contributors: {", ".join(new_contributors)}')
 
-    print(Changelog(commits.groups(), args.repo))
+    print(Changelog(commits.groups(), args.repo, args.collapsible))

From 69bec6730ec9d724bcedeab199d9d684d61423ba Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sun, 21 May 2023 09:56:23 +1200
Subject: [PATCH 268/871] [cleanup, utils] Split into submodules (#7090)

Closes https://github.com/yt-dlp/yt-dlp/pull/2173

Authored by: pukkandan, coletdjnz
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
---
 Makefile                             |   2 +-
 setup.cfg                            |   1 +
 yt_dlp/YoutubeDL.py                  |   2 -
 yt_dlp/utils/__init__.py             |  14 +
 yt_dlp/utils/_deprecated.py          |  30 ++
 yt_dlp/utils/_legacy.py              | 163 ++++++++++
 yt_dlp/{utils.py => utils/_utils.py} | 458 +--------------------------
 yt_dlp/utils/traversal.py            | 254 +++++++++++++++
 8 files changed, 480 insertions(+), 444 deletions(-)
 create mode 100644 yt_dlp/utils/__init__.py
 create mode 100644 yt_dlp/utils/_deprecated.py
 create mode 100644 yt_dlp/utils/_legacy.py
 rename yt_dlp/{utils.py => utils/_utils.py} (92%)
 create mode 100644 yt_dlp/utils/traversal.py

diff --git a/Makefile b/Makefile
index d5d47629b..f03fe2052 100644
--- a/Makefile
+++ b/Makefile
@@ -74,7 +74,7 @@ offlinetest: codetest
 	$(PYTHON) -m pytest -k "not download"
 
 # XXX: This is hard to maintain
-CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/dependencies
+CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/utils yt_dlp/dependencies
 yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
 	mkdir -p zip
 	for d in $(CODE_FOLDERS) ; do \
diff --git a/setup.cfg b/setup.cfg
index 6deaa7971..68d9e516d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -8,6 +8,7 @@ ignore = E402,E501,E731,E741,W503
 max_line_length = 120
 per_file_ignores =
     devscripts/lazy_load_template.py: F401
+    yt_dlp/utils/__init__.py: F401, F403
 
 
 [autoflake]
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 91aec1fe6..b8f1a05a0 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -124,7 +124,6 @@
     parse_filesize,
     preferredencoding,
     prepend_extension,
-    register_socks_protocols,
     remove_terminal_sequences,
     render_table,
     replace_extension,
@@ -739,7 +738,6 @@ def check_deprecated(param, option, suggestion):
                 when=when)
 
         self._setup_opener()
-        register_socks_protocols()
 
         def preload_download_archive(fn):
             """Preload the archive, if any is specified"""
diff --git a/yt_dlp/utils/__init__.py b/yt_dlp/utils/__init__.py
new file mode 100644
index 000000000..74b39e2c7
--- /dev/null
+++ b/yt_dlp/utils/__init__.py
@@ -0,0 +1,14 @@
+import warnings
+
+from ..compat.compat_utils import passthrough_module
+
+# XXX: Implement this the same way as other DeprecationWarnings without circular import
+passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
+    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=5))
+del passthrough_module
+
+# isort: off
+from .traversal import *
+from ._utils import *
+from ._utils import _configuration_args, _get_exe_version_output
+from ._deprecated import *
diff --git a/yt_dlp/utils/_deprecated.py b/yt_dlp/utils/_deprecated.py
new file mode 100644
index 000000000..4454d84a7
--- /dev/null
+++ b/yt_dlp/utils/_deprecated.py
@@ -0,0 +1,30 @@
+"""Deprecated - New code should avoid these"""
+
+from ._utils import preferredencoding
+
+
+def encodeFilename(s, for_subprocess=False):
+    assert isinstance(s, str)
+    return s
+
+
+def decodeFilename(b, for_subprocess=False):
+    return b
+
+
+def decodeArgument(b):
+    return b
+
+
+def decodeOption(optval):
+    if optval is None:
+        return optval
+    if isinstance(optval, bytes):
+        optval = optval.decode(preferredencoding())
+
+    assert isinstance(optval, str)
+    return optval
+
+
+def error_to_compat_str(err):
+    return str(err)
diff --git a/yt_dlp/utils/_legacy.py b/yt_dlp/utils/_legacy.py
new file mode 100644
index 000000000..cd009b504
--- /dev/null
+++ b/yt_dlp/utils/_legacy.py
@@ -0,0 +1,163 @@
+"""No longer used and new code should not use. Exists only for API compat."""
+
+import platform
+import struct
+import sys
+import urllib.parse
+import zlib
+
+from ._utils import decode_base_n, preferredencoding
+from .traversal import traverse_obj
+from ..dependencies import certifi, websockets
+
+has_certifi = bool(certifi)
+has_websockets = bool(websockets)
+
+
+def load_plugins(name, suffix, namespace):
+    from ..plugins import load_plugins
+    ret = load_plugins(name, suffix)
+    namespace.update(ret)
+    return ret
+
+
+def traverse_dict(dictn, keys, casesense=True):
+    return traverse_obj(dictn, keys, casesense=casesense, is_user_input=True, traverse_string=True)
+
+
+def decode_base(value, digits):
+    return decode_base_n(value, table=digits)
+
+
+def platform_name():
+    """ Returns the platform name as a str """
+    return platform.platform()
+
+
+def get_subprocess_encoding():
+    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+        # For subprocess calls, encode with locale encoding
+        # Refer to http://stackoverflow.com/a/9951851/35070
+        encoding = preferredencoding()
+    else:
+        encoding = sys.getfilesystemencoding()
+    if encoding is None:
+        encoding = 'utf-8'
+    return encoding
+
+
+# UNUSED
+# Based on png2str() written by @gdkchan and improved by @yokrysty
+# Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
+def decode_png(png_data):
+    # Reference: https://www.w3.org/TR/PNG/
+    header = png_data[8:]
+
+    if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
+        raise OSError('Not a valid PNG file.')
+
+    int_map = {1: '>B', 2: '>H', 4: '>I'}
+    unpack_integer = lambda x: struct.unpack(int_map[len(x)], x)[0]
+
+    chunks = []
+
+    while header:
+        length = unpack_integer(header[:4])
+        header = header[4:]
+
+        chunk_type = header[:4]
+        header = header[4:]
+
+        chunk_data = header[:length]
+        header = header[length:]
+
+        header = header[4:]  # Skip CRC
+
+        chunks.append({
+            'type': chunk_type,
+            'length': length,
+            'data': chunk_data
+        })
+
+    ihdr = chunks[0]['data']
+
+    width = unpack_integer(ihdr[:4])
+    height = unpack_integer(ihdr[4:8])
+
+    idat = b''
+
+    for chunk in chunks:
+        if chunk['type'] == b'IDAT':
+            idat += chunk['data']
+
+    if not idat:
+        raise OSError('Unable to read PNG data.')
+
+    decompressed_data = bytearray(zlib.decompress(idat))
+
+    stride = width * 3
+    pixels = []
+
+    def _get_pixel(idx):
+        x = idx % stride
+        y = idx // stride
+        return pixels[y][x]
+
+    for y in range(height):
+        basePos = y * (1 + stride)
+        filter_type = decompressed_data[basePos]
+
+        current_row = []
+
+        pixels.append(current_row)
+
+        for x in range(stride):
+            color = decompressed_data[1 + basePos + x]
+            basex = y * stride + x
+            left = 0
+            up = 0
+
+            if x > 2:
+                left = _get_pixel(basex - 3)
+            if y > 0:
+                up = _get_pixel(basex - stride)
+
+            if filter_type == 1:  # Sub
+                color = (color + left) & 0xff
+            elif filter_type == 2:  # Up
+                color = (color + up) & 0xff
+            elif filter_type == 3:  # Average
+                color = (color + ((left + up) >> 1)) & 0xff
+            elif filter_type == 4:  # Paeth
+                a = left
+                b = up
+                c = 0
+
+                if x > 2 and y > 0:
+                    c = _get_pixel(basex - stride - 3)
+
+                p = a + b - c
+
+                pa = abs(p - a)
+                pb = abs(p - b)
+                pc = abs(p - c)
+
+                if pa <= pb and pa <= pc:
+                    color = (color + a) & 0xff
+                elif pb <= pc:
+                    color = (color + b) & 0xff
+                else:
+                    color = (color + c) & 0xff
+
+            current_row.append(color)
+
+    return width, height, pixels
+
+
+def register_socks_protocols():
+    # "Register" SOCKS protocols
+    # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
+    # URLs with protocols not in urlparse.uses_netloc are not handled correctly
+    for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
+        if scheme not in urllib.parse.uses_netloc:
+            urllib.parse.uses_netloc.append(scheme)
diff --git a/yt_dlp/utils.py b/yt_dlp/utils/_utils.py
similarity index 92%
rename from yt_dlp/utils.py
rename to yt_dlp/utils/_utils.py
index 190af1b7d..f032af901 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -47,26 +47,18 @@
 import xml.etree.ElementTree
 import zlib
 
-from .compat import functools  # isort: split
-from .compat import (
+from . import traversal
+
+from ..compat import functools  # isort: split
+from ..compat import (
     compat_etree_fromstring,
     compat_expanduser,
     compat_HTMLParseError,
     compat_os_name,
     compat_shlex_quote,
 )
-from .dependencies import brotli, certifi, websockets, xattr
-from .socks import ProxyType, sockssocket
-
-
-def register_socks_protocols():
-    # "Register" SOCKS protocols
-    # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
-    # URLs with protocols not in urlparse.uses_netloc are not handled correctly
-    for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
-        if scheme not in urllib.parse.uses_netloc:
-            urllib.parse.uses_netloc.append(scheme)
-
+from ..dependencies import brotli, certifi, websockets, xattr
+from ..socks import ProxyType, sockssocket
 
 # This is not clearly defined otherwise
 compiled_regex_type = type(re.compile(''))
@@ -928,27 +920,6 @@ def run(cls, *args, timeout=None, **kwargs):
             return stdout or default, stderr or default, proc.returncode
 
 
-def get_subprocess_encoding():
-    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-        # For subprocess calls, encode with locale encoding
-        # Refer to http://stackoverflow.com/a/9951851/35070
-        encoding = preferredencoding()
-    else:
-        encoding = sys.getfilesystemencoding()
-    if encoding is None:
-        encoding = 'utf-8'
-    return encoding
-
-
-def encodeFilename(s, for_subprocess=False):
-    assert isinstance(s, str)
-    return s
-
-
-def decodeFilename(b, for_subprocess=False):
-    return b
-
-
 def encodeArgument(s):
     # Legacy code that uses byte strings
     # Uncomment the following line after fixing all post processors
@@ -956,20 +927,6 @@ def encodeArgument(s):
     return s if isinstance(s, str) else s.decode('ascii')
 
 
-def decodeArgument(b):
-    return b
-
-
-def decodeOption(optval):
-    if optval is None:
-        return optval
-    if isinstance(optval, bytes):
-        optval = optval.decode(preferredencoding())
-
-    assert isinstance(optval, str)
-    return optval
-
-
 _timetuple = collections.namedtuple('Time', ('hours', 'minutes', 'seconds', 'milliseconds'))
 
 
@@ -1034,7 +991,7 @@ def make_HTTPS_handler(params, **kwargs):
 
     context.verify_mode = ssl.CERT_REQUIRED if opts_check_certificate else ssl.CERT_NONE
     if opts_check_certificate:
-        if has_certifi and 'no-certifi' not in params.get('compat_opts', []):
+        if certifi and 'no-certifi' not in params.get('compat_opts', []):
             context.load_verify_locations(cafile=certifi.where())
         else:
             try:
@@ -1068,7 +1025,7 @@ def make_HTTPS_handler(params, **kwargs):
 
 
 def bug_reports_message(before=';'):
-    from .update import REPOSITORY
+    from ..update import REPOSITORY
 
     msg = (f'please report this issue on  https://github.com/{REPOSITORY}/issues?q= , '
            'filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U')
@@ -2019,12 +1976,6 @@ def __eq__(self, other):
                 and self.start == other.start and self.end == other.end)
 
 
-def platform_name():
-    """ Returns the platform name as a str """
-    deprecation_warning(f'"{__name__}.platform_name" is deprecated, use "platform.platform" instead')
-    return platform.platform()
-
-
 @functools.cache
 def system_identifier():
     python_implementation = platform.python_implementation()
@@ -2076,7 +2027,7 @@ def write_string(s, out=None, encoding=None):
 
 
 def deprecation_warning(msg, *, printer=None, stacklevel=0, **kwargs):
-    from . import _IN_CLI
+    from .. import _IN_CLI
     if _IN_CLI:
         if msg in deprecation_warning._cache:
             return
@@ -3284,13 +3235,6 @@ def variadic(x, allowed_types=NO_DEFAULT):
     return x if is_iterable_like(x, blocked_types=allowed_types) else (x, )
 
 
-def dict_get(d, key_or_keys, default=None, skip_false_values=True):
-    for val in map(d.get, variadic(key_or_keys)):
-        if val is not None and (val or not skip_false_values):
-            return val
-    return default
-
-
 def try_call(*funcs, expected_type=None, args=[], kwargs={}):
     for f in funcs:
         try:
@@ -3528,7 +3472,7 @@ def is_outdated_version(version, limit, assume_new=True):
 def ytdl_is_updateable():
     """ Returns if yt-dlp can be updated with -U """
 
-    from .update import is_non_updateable
+    from ..update import is_non_updateable
 
     return not is_non_updateable()
 
@@ -3538,10 +3482,6 @@ def args_to_str(args):
     return ' '.join(compat_shlex_quote(a) for a in args)
 
 
-def error_to_compat_str(err):
-    return str(err)
-
-
 def error_to_str(err):
     return f'{type(err).__name__}: {err}'
 
@@ -3628,7 +3568,7 @@ def mimetype2ext(mt, default=NO_DEFAULT):
     mimetype = mt.partition(';')[0].strip().lower()
     _, _, subtype = mimetype.rpartition('/')
 
-    ext = traverse_obj(MAP, mimetype, subtype, subtype.rsplit('+')[-1])
+    ext = traversal.traverse_obj(MAP, mimetype, subtype, subtype.rsplit('+')[-1])
     if ext:
         return ext
     elif default is not NO_DEFAULT:
@@ -3660,7 +3600,7 @@ def parse_codecs(codecs_str):
             vcodec = full_codec
             if parts[0] in ('dvh1', 'dvhe'):
                 hdr = 'DV'
-            elif parts[0] == 'av1' and traverse_obj(parts, 3) == '10':
+            elif parts[0] == 'av1' and traversal.traverse_obj(parts, 3) == '10':
                 hdr = 'HDR10'
             elif parts[:2] == ['vp9', '2']:
                 hdr = 'HDR10'
@@ -3706,8 +3646,7 @@ def get_compatible_ext(*, vcodecs, acodecs, vexts, aexts, preferences=None):
         },
     }
 
-    sanitize_codec = functools.partial(
-        try_get, getter=lambda x: x[0].split('.')[0].replace('0', '').lower())
+    sanitize_codec = functools.partial(try_get, getter=lambda x: x[0].split('.')[0].replace('0', ''))
     vcodec, acodec = sanitize_codec(vcodecs), sanitize_codec(acodecs)
 
     for ext in preferences or COMPATIBLE_CODECS.keys():
@@ -5088,12 +5027,6 @@ def decode_base_n(string, n=None, table=None):
     return result
 
 
-def decode_base(value, digits):
-    deprecation_warning(f'{__name__}.decode_base is deprecated and may be removed '
-                        f'in a future version. Use {__name__}.decode_base_n instead')
-    return decode_base_n(value, table=digits)
-
-
 def decode_packed_codes(code):
     mobj = re.search(PACKED_CODES_RE, code)
     obfuscated_code, base, count, symbols = mobj.groups()
@@ -5138,113 +5071,6 @@ def urshift(val, n):
     return val >> n if val >= 0 else (val + 0x100000000) >> n
 
 
-# Based on png2str() written by @gdkchan and improved by @yokrysty
-# Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
-def decode_png(png_data):
-    # Reference: https://www.w3.org/TR/PNG/
-    header = png_data[8:]
-
-    if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
-        raise OSError('Not a valid PNG file.')
-
-    int_map = {1: '>B', 2: '>H', 4: '>I'}
-    unpack_integer = lambda x: struct.unpack(int_map[len(x)], x)[0]
-
-    chunks = []
-
-    while header:
-        length = unpack_integer(header[:4])
-        header = header[4:]
-
-        chunk_type = header[:4]
-        header = header[4:]
-
-        chunk_data = header[:length]
-        header = header[length:]
-
-        header = header[4:]  # Skip CRC
-
-        chunks.append({
-            'type': chunk_type,
-            'length': length,
-            'data': chunk_data
-        })
-
-    ihdr = chunks[0]['data']
-
-    width = unpack_integer(ihdr[:4])
-    height = unpack_integer(ihdr[4:8])
-
-    idat = b''
-
-    for chunk in chunks:
-        if chunk['type'] == b'IDAT':
-            idat += chunk['data']
-
-    if not idat:
-        raise OSError('Unable to read PNG data.')
-
-    decompressed_data = bytearray(zlib.decompress(idat))
-
-    stride = width * 3
-    pixels = []
-
-    def _get_pixel(idx):
-        x = idx % stride
-        y = idx // stride
-        return pixels[y][x]
-
-    for y in range(height):
-        basePos = y * (1 + stride)
-        filter_type = decompressed_data[basePos]
-
-        current_row = []
-
-        pixels.append(current_row)
-
-        for x in range(stride):
-            color = decompressed_data[1 + basePos + x]
-            basex = y * stride + x
-            left = 0
-            up = 0
-
-            if x > 2:
-                left = _get_pixel(basex - 3)
-            if y > 0:
-                up = _get_pixel(basex - stride)
-
-            if filter_type == 1:  # Sub
-                color = (color + left) & 0xff
-            elif filter_type == 2:  # Up
-                color = (color + up) & 0xff
-            elif filter_type == 3:  # Average
-                color = (color + ((left + up) >> 1)) & 0xff
-            elif filter_type == 4:  # Paeth
-                a = left
-                b = up
-                c = 0
-
-                if x > 2 and y > 0:
-                    c = _get_pixel(basex - stride - 3)
-
-                p = a + b - c
-
-                pa = abs(p - a)
-                pb = abs(p - b)
-                pc = abs(p - c)
-
-                if pa <= pb and pa <= pc:
-                    color = (color + a) & 0xff
-                elif pb <= pc:
-                    color = (color + b) & 0xff
-                else:
-                    color = (color + c) & 0xff
-
-            current_row.append(color)
-
-    return width, height, pixels
-
-
 def write_xattr(path, key, value):
     # Windows: Write xattrs to NTFS Alternate Data Streams:
     # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
@@ -5403,7 +5229,7 @@ def to_high_limit_path(path):
 
 
 def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='', func=IDENTITY):
-    val = traverse_obj(obj, *variadic(field))
+    val = traversal.traverse_obj(obj, *variadic(field))
     if not val if ignore is NO_DEFAULT else val in variadic(ignore):
         return default
     return template % func(val)
@@ -5441,12 +5267,12 @@ def make_dir(path, to_screen=None):
         return True
     except OSError as err:
         if callable(to_screen) is not None:
-            to_screen('unable to create directory ' + error_to_compat_str(err))
+            to_screen(f'unable to create directory {err}')
         return False
 
 
 def get_executable_path():
-    from .update import _get_variant_and_executable_path
+    from ..update import _get_variant_and_executable_path
 
     return os.path.dirname(os.path.abspath(_get_variant_and_executable_path()[1]))
 
@@ -5470,244 +5296,6 @@ def get_system_config_dirs(package_name):
     yield os.path.join('/etc', package_name)
 
 
-def traverse_obj(
-        obj, *paths, default=NO_DEFAULT, expected_type=None, get_all=True,
-        casesense=True, is_user_input=False, traverse_string=False):
-    """
-    Safely traverse nested `dict`s and `Iterable`s
-
-    >>> obj = [{}, {"key": "value"}]
-    >>> traverse_obj(obj, (1, "key"))
-    "value"
-
-    Each of the provided `paths` is tested and the first producing a valid result will be returned.
-    The next path will also be tested if the path branched but no results could be found.
-    Supported values for traversal are `Mapping`, `Iterable` and `re.Match`.
-    Unhelpful values (`{}`, `None`) are treated as the absence of a value and discarded.
-
-    The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
-
-    The keys in the path can be one of:
-        - `None`:           Return the current object.
-        - `set`:            Requires the only item in the set to be a type or function,
-                            like `{type}`/`{func}`. If a `type`, returns only values
-                            of this type. If a function, returns `func(obj)`.
-        - `str`/`int`:      Return `obj[key]`. For `re.Match`, return `obj.group(key)`.
-        - `slice`:          Branch out and return all values in `obj[key]`.
-        - `Ellipsis`:       Branch out and return a list of all values.
-        - `tuple`/`list`:   Branch out and return a list of all matching values.
-                            Read as: `[traverse_obj(obj, branch) for branch in branches]`.
-        - `function`:       Branch out and return values filtered by the function.
-                            Read as: `[value for key, value in obj if function(key, value)]`.
-                            For `Iterable`s, `key` is the index of the value.
-                            For `re.Match`es, `key` is the group number (0 = full match)
-                            as well as additionally any group names, if given.
-        - `dict`            Transform the current object and return a matching dict.
-                            Read as: `{key: traverse_obj(obj, path) for key, path in dct.items()}`.
-
-        `tuple`, `list`, and `dict` all support nested paths and branches.
-
-    @params paths           Paths which to traverse by.
-    @param default          Value to return if the paths do not match.
-                            If the last key in the path is a `dict`, it will apply to each value inside
-                            the dict instead, depth first. Try to avoid if using nested `dict` keys.
-    @param expected_type    If a `type`, only accept final values of this type.
-                            If any other callable, try to call the function on each result.
-                            If the last key in the path is a `dict`, it will apply to each value inside
-                            the dict instead, recursively. This does respect branching paths.
-    @param get_all          If `False`, return the first matching result, otherwise all matching ones.
-    @param casesense        If `False`, consider string dictionary keys as case insensitive.
-
-    The following are only meant to be used by YoutubeDL.prepare_outtmpl and are not part of the API
-
-    @param is_user_input    Whether the keys are generated from user input.
-                            If `True` strings get converted to `int`/`slice` if needed.
-    @param traverse_string  Whether to traverse into objects as strings.
-                            If `True`, any non-compatible object will first be
-                            converted into a string and then traversed into.
-                            The return value of that path will be a string instead,
-                            not respecting any further branching.
-
-
-    @returns                The result of the object traversal.
-                            If successful, `get_all=True`, and the path branches at least once,
-                            then a list of results is returned instead.
-                            If no `default` is given and the last path branches, a `list` of results
-                            is always returned. If a path ends on a `dict` that result will always be a `dict`.
-    """
-    casefold = lambda k: k.casefold() if isinstance(k, str) else k
-
-    if isinstance(expected_type, type):
-        type_test = lambda val: val if isinstance(val, expected_type) else None
-    else:
-        type_test = lambda val: try_call(expected_type or IDENTITY, args=(val,))
-
-    def apply_key(key, obj, is_last):
-        branching = False
-        result = None
-
-        if obj is None and traverse_string:
-            if key is ... or callable(key) or isinstance(key, slice):
-                branching = True
-                result = ()
-
-        elif key is None:
-            result = obj
-
-        elif isinstance(key, set):
-            assert len(key) == 1, 'Set should only be used to wrap a single item'
-            item = next(iter(key))
-            if isinstance(item, type):
-                if isinstance(obj, item):
-                    result = obj
-            else:
-                result = try_call(item, args=(obj,))
-
-        elif isinstance(key, (list, tuple)):
-            branching = True
-            result = itertools.chain.from_iterable(
-                apply_path(obj, branch, is_last)[0] for branch in key)
-
-        elif key is ...:
-            branching = True
-            if isinstance(obj, collections.abc.Mapping):
-                result = obj.values()
-            elif is_iterable_like(obj):
-                result = obj
-            elif isinstance(obj, re.Match):
-                result = obj.groups()
-            elif traverse_string:
-                branching = False
-                result = str(obj)
-            else:
-                result = ()
-
-        elif callable(key):
-            branching = True
-            if isinstance(obj, collections.abc.Mapping):
-                iter_obj = obj.items()
-            elif is_iterable_like(obj):
-                iter_obj = enumerate(obj)
-            elif isinstance(obj, re.Match):
-                iter_obj = itertools.chain(
-                    enumerate((obj.group(), *obj.groups())),
-                    obj.groupdict().items())
-            elif traverse_string:
-                branching = False
-                iter_obj = enumerate(str(obj))
-            else:
-                iter_obj = ()
-
-            result = (v for k, v in iter_obj if try_call(key, args=(k, v)))
-            if not branching:  # string traversal
-                result = ''.join(result)
-
-        elif isinstance(key, dict):
-            iter_obj = ((k, _traverse_obj(obj, v, False, is_last)) for k, v in key.items())
-            result = {
-                k: v if v is not None else default for k, v in iter_obj
-                if v is not None or default is not NO_DEFAULT
-            } or None
-
-        elif isinstance(obj, collections.abc.Mapping):
-            result = (try_call(obj.get, args=(key,)) if casesense or try_call(obj.__contains__, args=(key,)) else
-                      next((v for k, v in obj.items() if casefold(k) == key), None))
-
-        elif isinstance(obj, re.Match):
-            if isinstance(key, int) or casesense:
-                with contextlib.suppress(IndexError):
-                    result = obj.group(key)
-
-            elif isinstance(key, str):
-                result = next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
-
-        elif isinstance(key, (int, slice)):
-            if is_iterable_like(obj, collections.abc.Sequence):
-                branching = isinstance(key, slice)
-                with contextlib.suppress(IndexError):
-                    result = obj[key]
-            elif traverse_string:
-                with contextlib.suppress(IndexError):
-                    result = str(obj)[key]
-
-        return branching, result if branching else (result,)
-
-    def lazy_last(iterable):
-        iterator = iter(iterable)
-        prev = next(iterator, NO_DEFAULT)
-        if prev is NO_DEFAULT:
-            return
-
-        for item in iterator:
-            yield False, prev
-            prev = item
-
-        yield True, prev
-
-    def apply_path(start_obj, path, test_type):
-        objs = (start_obj,)
-        has_branched = False
-
-        key = None
-        for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
-            if is_user_input and isinstance(key, str):
-                if key == ':':
-                    key = ...
-                elif ':' in key:
-                    key = slice(*map(int_or_none, key.split(':')))
-                elif int_or_none(key) is not None:
-                    key = int(key)
-
-            if not casesense and isinstance(key, str):
-                key = key.casefold()
-
-            if __debug__ and callable(key):
-                # Verify function signature
-                inspect.signature(key).bind(None, None)
-
-            new_objs = []
-            for obj in objs:
-                branching, results = apply_key(key, obj, last)
-                has_branched |= branching
-                new_objs.append(results)
-
-            objs = itertools.chain.from_iterable(new_objs)
-
-        if test_type and not isinstance(key, (dict, list, tuple)):
-            objs = map(type_test, objs)
-
-        return objs, has_branched, isinstance(key, dict)
-
-    def _traverse_obj(obj, path, allow_empty, test_type):
-        results, has_branched, is_dict = apply_path(obj, path, test_type)
-        results = LazyList(item for item in results if item not in (None, {}))
-        if get_all and has_branched:
-            if results:
-                return results.exhaust()
-            if allow_empty:
-                return [] if default is NO_DEFAULT else default
-            return None
-
-        return results[0] if results else {} if allow_empty and is_dict else None
-
-    for index, path in enumerate(paths, 1):
-        result = _traverse_obj(obj, path, index == len(paths), True)
-        if result is not None:
-            return result
-
-    return None if default is NO_DEFAULT else default
-
-
-def traverse_dict(dictn, keys, casesense=True):
-    deprecation_warning(f'"{__name__}.traverse_dict" is deprecated and may be removed '
-                        f'in a future version. Use "{__name__}.traverse_obj" instead')
-    return traverse_obj(dictn, keys, casesense=casesense, is_user_input=True, traverse_string=True)
-
-
-def get_first(obj, *paths, **kwargs):
-    return traverse_obj(obj, *((..., *variadic(keys)) for keys in paths), **kwargs, get_all=False)
-
-
 def time_seconds(**kwargs):
     """
     Returns TZ-aware time in seconds since the epoch (1970-01-01T00:00:00Z)
@@ -5803,7 +5391,7 @@ def number_of_digits(number):
 
 def join_nonempty(*values, delim='-', from_dict=None):
     if from_dict is not None:
-        values = (traverse_obj(from_dict, variadic(v)) for v in values)
+        values = (traversal.traverse_obj(from_dict, variadic(v)) for v in values)
     return delim.join(map(str, filter(None, values)))
 
 
@@ -6514,15 +6102,3 @@ def calculate_preference(self, format):
                 format['abr'] = format.get('tbr') - format.get('vbr', 0)
 
         return tuple(self._calculate_field_preference(format, field) for field in self._order)
-
-
-# Deprecated
-has_certifi = bool(certifi)
-has_websockets = bool(websockets)
-
-
-def load_plugins(name, suffix, namespace):
-    from .plugins import load_plugins
-    ret = load_plugins(name, suffix)
-    namespace.update(ret)
-    return ret
diff --git a/yt_dlp/utils/traversal.py b/yt_dlp/utils/traversal.py
new file mode 100644
index 000000000..462c3ba5d
--- /dev/null
+++ b/yt_dlp/utils/traversal.py
@@ -0,0 +1,254 @@
+import collections.abc
+import contextlib
+import inspect
+import itertools
+import re
+
+from ._utils import (
+    IDENTITY,
+    NO_DEFAULT,
+    LazyList,
+    int_or_none,
+    is_iterable_like,
+    try_call,
+    variadic,
+)
+
+
+def traverse_obj(
+        obj, *paths, default=NO_DEFAULT, expected_type=None, get_all=True,
+        casesense=True, is_user_input=False, traverse_string=False):
+    """
+    Safely traverse nested `dict`s and `Iterable`s
+
+    >>> obj = [{}, {"key": "value"}]
+    >>> traverse_obj(obj, (1, "key"))
+    "value"
+
+    Each of the provided `paths` is tested and the first producing a valid result will be returned.
+    The next path will also be tested if the path branched but no results could be found.
+    Supported values for traversal are `Mapping`, `Iterable` and `re.Match`.
+    Unhelpful values (`{}`, `None`) are treated as the absence of a value and discarded.
+
+    The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
+
+    The keys in the path can be one of:
+        - `None`:           Return the current object.
+        - `set`:            Requires the only item in the set to be a type or function,
+                            like `{type}`/`{func}`. If a `type`, returns only values
+                            of this type. If a function, returns `func(obj)`.
+        - `str`/`int`:      Return `obj[key]`. For `re.Match`, return `obj.group(key)`.
+        - `slice`:          Branch out and return all values in `obj[key]`.
+        - `Ellipsis`:       Branch out and return a list of all values.
+        - `tuple`/`list`:   Branch out and return a list of all matching values.
+                            Read as: `[traverse_obj(obj, branch) for branch in branches]`.
+        - `function`:       Branch out and return values filtered by the function.
+                            Read as: `[value for key, value in obj if function(key, value)]`.
+                            For `Iterable`s, `key` is the index of the value.
+                            For `re.Match`es, `key` is the group number (0 = full match)
+                            as well as additionally any group names, if given.
+        - `dict`            Transform the current object and return a matching dict.
+                            Read as: `{key: traverse_obj(obj, path) for key, path in dct.items()}`.
+
+        `tuple`, `list`, and `dict` all support nested paths and branches.
+
+    @params paths           Paths which to traverse by.
+    @param default          Value to return if the paths do not match.
+                            If the last key in the path is a `dict`, it will apply to each value inside
+                            the dict instead, depth first. Try to avoid if using nested `dict` keys.
+    @param expected_type    If a `type`, only accept final values of this type.
+                            If any other callable, try to call the function on each result.
+                            If the last key in the path is a `dict`, it will apply to each value inside
+                            the dict instead, recursively. This does respect branching paths.
+    @param get_all          If `False`, return the first matching result, otherwise all matching ones.
+    @param casesense        If `False`, consider string dictionary keys as case insensitive.
+
+    The following are only meant to be used by YoutubeDL.prepare_outtmpl and are not part of the API
+
+    @param is_user_input    Whether the keys are generated from user input.
+                            If `True` strings get converted to `int`/`slice` if needed.
+    @param traverse_string  Whether to traverse into objects as strings.
+                            If `True`, any non-compatible object will first be
+                            converted into a string and then traversed into.
+                            The return value of that path will be a string instead,
+                            not respecting any further branching.
+
+
+    @returns                The result of the object traversal.
+                            If successful, `get_all=True`, and the path branches at least once,
+                            then a list of results is returned instead.
+                            If no `default` is given and the last path branches, a `list` of results
+                            is always returned. If a path ends on a `dict` that result will always be a `dict`.
+    """
+    casefold = lambda k: k.casefold() if isinstance(k, str) else k
+
+    if isinstance(expected_type, type):
+        type_test = lambda val: val if isinstance(val, expected_type) else None
+    else:
+        type_test = lambda val: try_call(expected_type or IDENTITY, args=(val,))
+
+    def apply_key(key, obj, is_last):
+        branching = False
+        result = None
+
+        if obj is None and traverse_string:
+            if key is ... or callable(key) or isinstance(key, slice):
+                branching = True
+                result = ()
+
+        elif key is None:
+            result = obj
+
+        elif isinstance(key, set):
+            assert len(key) == 1, 'Set should only be used to wrap a single item'
+            item = next(iter(key))
+            if isinstance(item, type):
+                if isinstance(obj, item):
+                    result = obj
+            else:
+                result = try_call(item, args=(obj,))
+
+        elif isinstance(key, (list, tuple)):
+            branching = True
+            result = itertools.chain.from_iterable(
+                apply_path(obj, branch, is_last)[0] for branch in key)
+
+        elif key is ...:
+            branching = True
+            if isinstance(obj, collections.abc.Mapping):
+                result = obj.values()
+            elif is_iterable_like(obj):
+                result = obj
+            elif isinstance(obj, re.Match):
+                result = obj.groups()
+            elif traverse_string:
+                branching = False
+                result = str(obj)
+            else:
+                result = ()
+
+        elif callable(key):
+            branching = True
+            if isinstance(obj, collections.abc.Mapping):
+                iter_obj = obj.items()
+            elif is_iterable_like(obj):
+                iter_obj = enumerate(obj)
+            elif isinstance(obj, re.Match):
+                iter_obj = itertools.chain(
+                    enumerate((obj.group(), *obj.groups())),
+                    obj.groupdict().items())
+            elif traverse_string:
+                branching = False
+                iter_obj = enumerate(str(obj))
+            else:
+                iter_obj = ()
+
+            result = (v for k, v in iter_obj if try_call(key, args=(k, v)))
+            if not branching:  # string traversal
+                result = ''.join(result)
+
+        elif isinstance(key, dict):
+            iter_obj = ((k, _traverse_obj(obj, v, False, is_last)) for k, v in key.items())
+            result = {
+                k: v if v is not None else default for k, v in iter_obj
+                if v is not None or default is not NO_DEFAULT
+            } or None
+
+        elif isinstance(obj, collections.abc.Mapping):
+            result = (try_call(obj.get, args=(key,)) if casesense or try_call(obj.__contains__, args=(key,)) else
+                      next((v for k, v in obj.items() if casefold(k) == key), None))
+
+        elif isinstance(obj, re.Match):
+            if isinstance(key, int) or casesense:
+                with contextlib.suppress(IndexError):
+                    result = obj.group(key)
+
+            elif isinstance(key, str):
+                result = next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
+
+        elif isinstance(key, (int, slice)):
+            if is_iterable_like(obj, collections.abc.Sequence):
+                branching = isinstance(key, slice)
+                with contextlib.suppress(IndexError):
+                    result = obj[key]
+            elif traverse_string:
+                with contextlib.suppress(IndexError):
+                    result = str(obj)[key]
+
+        return branching, result if branching else (result,)
+
+    def lazy_last(iterable):
+        iterator = iter(iterable)
+        prev = next(iterator, NO_DEFAULT)
+        if prev is NO_DEFAULT:
+            return
+
+        for item in iterator:
+            yield False, prev
+            prev = item
+
+        yield True, prev
+
+    def apply_path(start_obj, path, test_type):
+        objs = (start_obj,)
+        has_branched = False
+
+        key = None
+        for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
+            if is_user_input and isinstance(key, str):
+                if key == ':':
+                    key = ...
+                elif ':' in key:
+                    key = slice(*map(int_or_none, key.split(':')))
+                elif int_or_none(key) is not None:
+                    key = int(key)
+
+            if not casesense and isinstance(key, str):
+                key = key.casefold()
+
+            if __debug__ and callable(key):
+                # Verify function signature
+                inspect.signature(key).bind(None, None)
+
+            new_objs = []
+            for obj in objs:
+                branching, results = apply_key(key, obj, last)
+                has_branched |= branching
+                new_objs.append(results)
+
+            objs = itertools.chain.from_iterable(new_objs)
+
+        if test_type and not isinstance(key, (dict, list, tuple)):
+            objs = map(type_test, objs)
+
+        return objs, has_branched, isinstance(key, dict)
+
+    def _traverse_obj(obj, path, allow_empty, test_type):
+        results, has_branched, is_dict = apply_path(obj, path, test_type)
+        results = LazyList(item for item in results if item not in (None, {}))
+        if get_all and has_branched:
+            if results:
+                return results.exhaust()
+            if allow_empty:
+                return [] if default is NO_DEFAULT else default
+            return None
+
+        return results[0] if results else {} if allow_empty and is_dict else None
+
+    for index, path in enumerate(paths, 1):
+        result = _traverse_obj(obj, path, index == len(paths), True)
+        if result is not None:
+            return result
+
+    return None if default is NO_DEFAULT else default
+
+
+def get_first(obj, *paths, **kwargs):
+    return traverse_obj(obj, *((..., *variadic(keys)) for keys in paths), **kwargs, get_all=False)
+
+
+def dict_get(d, key_or_keys, default=None, skip_false_values=True):
+    for val in map(d.get, variadic(key_or_keys)):
+        if val is not None and (val or not skip_false_values):
+            return val
+    return default

From 955c89584b66fcd0fcfab3e611f1edeb1ca63886 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sun, 21 May 2023 10:55:09 +1200
Subject: [PATCH 269/871] [core] Deprecate internal `Youtubedl-no-compression`
 header (#6876)

Authored by: coletdjnz
---
 yt_dlp/YoutubeDL.py           |  4 +++-
 yt_dlp/downloader/external.py |  4 +---
 yt_dlp/downloader/http.py     |  4 ++--
 yt_dlp/extractor/litv.py      |  2 +-
 yt_dlp/utils/_legacy.py       | 10 ++++++++++
 yt_dlp/utils/_utils.py        | 23 ++++++-----------------
 6 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index b8f1a05a0..1162d2df1 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2380,7 +2380,9 @@ def restore_last_token(self):
 
     def _calc_headers(self, info_dict):
         res = merge_headers(self.params['http_headers'], info_dict.get('http_headers') or {})
-
+        if 'Youtubedl-No-Compression' in res:  # deprecated
+            res.pop('Youtubedl-No-Compression', None)
+            res['Accept-Encoding'] = 'identity'
         cookies = self._calc_cookies(info_dict['url'])
         if cookies:
             res['Cookie'] = cookies
diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index ee130c827..007689a8c 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -23,7 +23,6 @@
     encodeArgument,
     encodeFilename,
     find_available_port,
-    handle_youtubedl_headers,
     remove_end,
     sanitized_Request,
     traverse_obj,
@@ -529,10 +528,9 @@ def _call_downloader(self, tmpfilename, info_dict):
         selected_formats = info_dict.get('requested_formats') or [info_dict]
         for i, fmt in enumerate(selected_formats):
             if fmt.get('http_headers') and re.match(r'^https?://', fmt['url']):
-                headers_dict = handle_youtubedl_headers(fmt['http_headers'])
                 # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
                 # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
-                args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in headers_dict.items())])
+                args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in fmt['http_headers'].items())])
 
             if start_time:
                 args += ['-ss', str(start_time)]
diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index fa72d5722..79f69b5d0 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -45,8 +45,8 @@ class DownloadContext(dict):
         ctx.tmpfilename = self.temp_name(filename)
         ctx.stream = None
 
-        # Do not include the Accept-Encoding header
-        headers = {'Youtubedl-no-compression': 'True'}
+        # Disable compression
+        headers = {'Accept-Encoding': 'identity'}
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
diff --git a/yt_dlp/extractor/litv.py b/yt_dlp/extractor/litv.py
index 31826ac99..0b792fb96 100644
--- a/yt_dlp/extractor/litv.py
+++ b/yt_dlp/extractor/litv.py
@@ -113,7 +113,7 @@ def _real_extract(self, url):
             entry_protocol='m3u8_native', m3u8_id='hls')
         for a_format in formats:
             # LiTV HLS segments doesn't like compressions
-            a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = True
+            a_format.setdefault('http_headers', {})['Accept-Encoding'] = 'identity'
 
         title = program_info['title'] + program_info.get('secondaryMark', '')
         description = program_info.get('description')
diff --git a/yt_dlp/utils/_legacy.py b/yt_dlp/utils/_legacy.py
index cd009b504..b0578a1d6 100644
--- a/yt_dlp/utils/_legacy.py
+++ b/yt_dlp/utils/_legacy.py
@@ -161,3 +161,13 @@ def register_socks_protocols():
     for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
         if scheme not in urllib.parse.uses_netloc:
             urllib.parse.uses_netloc.append(scheme)
+
+
+def handle_youtubedl_headers(headers):
+    filtered_headers = headers
+
+    if 'Youtubedl-no-compression' in filtered_headers:
+        filtered_headers = {k: v for k, v in filtered_headers.items() if k.lower() != 'accept-encoding'}
+        del filtered_headers['Youtubedl-no-compression']
+
+    return filtered_headers
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index f032af901..9f1a127cd 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -1308,25 +1308,12 @@ def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_a
     return hc
 
 
-def handle_youtubedl_headers(headers):
-    filtered_headers = headers
-
-    if 'Youtubedl-no-compression' in filtered_headers:
-        filtered_headers = {k: v for k, v in filtered_headers.items() if k.lower() != 'accept-encoding'}
-        del filtered_headers['Youtubedl-no-compression']
-
-    return filtered_headers
-
-
 class YoutubeDLHandler(urllib.request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
     This class, when installed with an OpenerDirector, automatically adds
-    the standard headers to every HTTP request and handles gzipped and
-    deflated responses from web servers. If compression is to be avoided in
-    a particular request, the original request in the program code only has
-    to include the HTTP header "Youtubedl-no-compression", which will be
-    removed before making the real request.
+    the standard headers to every HTTP request and handles gzipped, deflated and
+    brotli responses from web servers.
 
     Part of this code was copied from:
 
@@ -1389,11 +1376,13 @@ def http_request(self, req):
             if h.capitalize() not in req.headers:
                 req.add_header(h, v)
 
+        if 'Youtubedl-no-compression' in req.headers:  # deprecated
+            req.headers.pop('Youtubedl-no-compression', None)
+            req.add_header('Accept-encoding', 'identity')
+
         if 'Accept-encoding' not in req.headers:
             req.add_header('Accept-encoding', ', '.join(SUPPORTED_ENCODINGS))
 
-        req.headers = handle_youtubedl_headers(req.headers)
-
         return super().do_request_(req)
 
     def http_response(self, req, resp):

From 69a40e4a7f6caa5662527ebd2f3c4e8aa02857a2 Mon Sep 17 00:00:00 2001
From: kangalio <jannik.a.schaper@web.de>
Date: Mon, 22 May 2023 13:47:06 +0200
Subject: [PATCH 270/871] [extractor/youtube:music:search_url] Extract title
 (#7102)

Authored by: kangalio
Closes #7095
---
 yt_dlp/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index d089822f6..bd38900f2 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4579,8 +4579,11 @@ def _grid_entries(self, grid_renderer):
     def _music_reponsive_list_entry(self, renderer):
         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
         if video_id:
+            title = traverse_obj(renderer, (
+                'flexColumns', 0, 'musicResponsiveListItemFlexColumnRenderer',
+                'text', 'runs', 0, 'text'))
             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
-                                   ie=YoutubeIE.ie_key(), video_id=video_id)
+                                   ie=YoutubeIE.ie_key(), video_id=video_id, title=title)
         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
         if playlist_id:
             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))

From 46f1370e9af6f8af8762f67e27e5acb8f0c48a47 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 24 May 2023 23:29:30 +0530
Subject: [PATCH 271/871] [devscripts/cli_to_api] Add script

---
 devscripts/cli_to_api.py      | 48 +++++++++++++++++++++++++++++++++++
 yt_dlp/YoutubeDL.py           |  8 +++---
 yt_dlp/downloader/common.py   |  7 ++---
 yt_dlp/downloader/fragment.py |  4 +--
 yt_dlp/utils/_utils.py        |  6 +++--
 5 files changed, 62 insertions(+), 11 deletions(-)
 create mode 100644 devscripts/cli_to_api.py

diff --git a/devscripts/cli_to_api.py b/devscripts/cli_to_api.py
new file mode 100644
index 000000000..b8b7cbcf1
--- /dev/null
+++ b/devscripts/cli_to_api.py
@@ -0,0 +1,48 @@
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import yt_dlp
+import yt_dlp.options
+
+create_parser = yt_dlp.options.create_parser
+
+
+def parse_patched_options(opts):
+    patched_parser = create_parser()
+    patched_parser.defaults.update({
+        'ignoreerrors': False,
+        'retries': 0,
+        'fragment_retries': 0,
+        'extract_flat': False,
+        'concat_playlist': 'never',
+    })
+    yt_dlp.options.__dict__['create_parser'] = lambda: patched_parser
+    try:
+        return yt_dlp.parse_options(opts)
+    finally:
+        yt_dlp.options.__dict__['create_parser'] = create_parser
+
+
+default_opts = parse_patched_options([]).ydl_opts
+
+
+def cli_to_api(opts, cli_defaults=False):
+    opts = (yt_dlp.parse_options if cli_defaults else parse_patched_options)(opts).ydl_opts
+
+    diff = {k: v for k, v in opts.items() if default_opts[k] != v}
+    if 'postprocessors' in diff:
+        diff['postprocessors'] = [pp for pp in diff['postprocessors']
+                                  if pp not in default_opts['postprocessors']]
+    return diff
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+
+    print('\nThe arguments passed translate to:\n')
+    pprint(cli_to_api(sys.argv[1:]))
+    print('\nCombining these with the CLI defaults gives:\n')
+    pprint(cli_to_api(sys.argv[1:], True))
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 1162d2df1..cd82b2772 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -280,7 +280,7 @@ class YoutubeDL:
                        subtitles. The language can be prefixed with a "-" to
                        exclude it from the requested languages, e.g. ['all', '-live_chat']
     keepvideo:         Keep the video file after post-processing
-    daterange:         A DateRange object, download only if the upload_date is in the range.
+    daterange:         A utils.DateRange object, download only if the upload_date is in the range.
     skip_download:     Skip the actual download of the video file
     cachedir:          Location of the cache files in the filesystem.
                        False to disable filesystem cache.
@@ -329,13 +329,13 @@ class YoutubeDL:
                        'auto' for elaborate guessing
     encoding:          Use this encoding instead of the system-specified.
     extract_flat:      Whether to resolve and process url_results further
-                       * False:     Always process (default)
+                       * False:     Always process. Default for API
                        * True:      Never process
                        * 'in_playlist': Do not process inside playlist/multi_video
                        * 'discard': Always process, but don't return the result
                                     from inside playlist/multi_video
                        * 'discard_in_playlist': Same as "discard", but only for
-                                    playlists (not multi_video)
+                                    playlists (not multi_video). Default for CLI
     wait_for_video:    If given, wait for scheduled streams to become available.
                        The value should be a tuple containing the range
                        (min_secs, max_secs) to wait between retries
@@ -472,7 +472,7 @@ class YoutubeDL:
                        can also be used
 
     The following options are used by the extractors:
-    extractor_retries: Number of times to retry for known errors
+    extractor_retries: Number of times to retry for known errors (default: 3)
     dynamic_mpd:       Whether to process dynamic DASH manifests (default: True)
     hls_split_discontinuity: Split HLS playlists to different formats at
                        discontinuities such as ad breaks (default: False)
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index 077b29b41..8f9bc05d6 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -51,8 +51,9 @@ class FileDownloader:
     ratelimit:          Download speed limit, in bytes/sec.
     continuedl:         Attempt to continue downloads if possible
     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
-    retries:            Number of times to retry for HTTP error 5xx
-    file_access_retries:   Number of times to retry on file access error
+    retries:            Number of times to retry for expected network errors.
+                        Default is 0 for API, but 10 for CLI
+    file_access_retries:   Number of times to retry on file access error (default: 3)
     buffersize:         Size of download buffer in bytes.
     noresizebuffer:     Do not automatically resize the download buffer.
     continuedl:         Try to continue downloads if possible.
@@ -225,7 +226,7 @@ def error_callback(err, count, retries, *, fd):
                 sleep_func=fd.params.get('retry_sleep_functions', {}).get('file_access'))
 
         def wrapper(self, func, *args, **kwargs):
-            for retry in RetryManager(self.params.get('file_access_retries'), error_callback, fd=self):
+            for retry in RetryManager(self.params.get('file_access_retries', 3), error_callback, fd=self):
                 try:
                     return func(self, *args, **kwargs)
                 except OSError as err:
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 3dc638f52..8abf7760b 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -34,8 +34,8 @@ class FragmentFD(FileDownloader):
 
     Available options:
 
-    fragment_retries:   Number of times to retry a fragment for HTTP error (DASH
-                        and hlsnative only)
+    fragment_retries:   Number of times to retry a fragment for HTTP error
+                        (DASH and hlsnative only). Default is 0 for API, but 10 for CLI
     skip_unavailable_fragments:
                         Skip unavailable fragments (DASH and hlsnative only)
     keep_fragments:     Keep downloaded fragments on disk after downloading is
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 9f1a127cd..afcb2a164 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -60,6 +60,8 @@
 from ..dependencies import brotli, certifi, websockets, xattr
 from ..socks import ProxyType, sockssocket
 
+__name__ = __name__.rsplit('.', 1)[0]  # Pretend to be the parent module
+
 # This is not clearly defined otherwise
 compiled_regex_type = type(re.compile(''))
 
@@ -1957,8 +1959,8 @@ def __contains__(self, date):
             date = date_from_str(date)
         return self.start <= date <= self.end
 
-    def __str__(self):
-        return f'{self.start.isoformat()} - {self.end.isoformat()}'
+    def __repr__(self):
+        return f'{__name__}.{type(self).__name__}({self.start.isoformat()!r}, {self.end.isoformat()!r})'
 
     def __eq__(self, other):
         return (isinstance(other, DateRange)

From 4823ec9f461512daa1b8ab362893bb86a6320b26 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 24 May 2023 23:30:43 +0530
Subject: [PATCH 272/871] Update to ytdl-commit-d1c6c5

[YouTube] [core] Improve platform debug log, based on yt-dlp
https://github.com/ytdl-org/youtube-dl/commit/d1c6c5c4d618fa950813c0c71aede34a5ac851e9

Except:
    * 6ed34338285f722d0da312ce0af3a15a077a3e2a [jsinterp] Add short-cut evaluation for common expression
        * There was no performance improvement when tested with https://github.com/ytdl-org/youtube-dl/issues/30641
    * e8de54bce50f6f77a4d7e8e80675f7003d5bf630 [core] Handle `/../` sequences in HTTP URLs
        * We plan to implement this differently
---
 test/test_jsinterp.py          | 32 ++++++++++++++++++++++++++------
 test/test_utils.py             | 32 ++++++++++++++++++++++++++++++++
 yt_dlp/downloader/common.py    | 24 +++++++++++++++++-------
 yt_dlp/downloader/fragment.py  | 33 ++++++++++++++++++++-------------
 yt_dlp/downloader/http.py      |  3 ++-
 yt_dlp/extractor/aenetworks.py | 15 +++++++++++++--
 yt_dlp/extractor/litv.py       |  2 +-
 yt_dlp/extractor/youtube.py    | 10 +++-------
 yt_dlp/jsinterp.py             |  2 +-
 yt_dlp/utils/_utils.py         | 12 ++++++++++--
 10 files changed, 125 insertions(+), 40 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 444909b84..96274116b 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -66,9 +66,8 @@ def test_assignments(self):
         self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51)
         self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11)
 
+    @unittest.skip('Not implemented')
     def test_comments(self):
-        'Skipping: Not yet fully implemented'
-        return
         self._test('''
             function f() {
                 var x = /* 1 + */ 2;
@@ -100,10 +99,13 @@ def test_builtins(self):
         jsi = JSInterpreter('function f() { return NaN }')
         self.assertTrue(math.isnan(jsi.call_function('f')))
 
-        self._test('function f() { return new Date("Wednesday 31 December 1969 18:01:26 MDT") - 0; }',
-                   86000)
-        self._test('function f(dt) { return new Date(dt) - 0; }',
-                   86000, args=['Wednesday 31 December 1969 18:01:26 MDT'])
+    def test_date(self):
+        self._test('function f() { return new Date("Wednesday 31 December 1969 18:01:26 MDT") - 0; }', 86000)
+
+        jsi = JSInterpreter('function f(dt) { return new Date(dt) - 0; }')
+        self.assertEqual(jsi.call_function('f', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
+        self.assertEqual(jsi.call_function('f', '12/31/1969 18:01:26 MDT'), 86000)  # m/d/y
+        self.assertEqual(jsi.call_function('f', '1 January 1970 00:00:00 UTC'), 0)
 
     def test_call(self):
         jsi = JSInterpreter('''
@@ -286,6 +288,19 @@ def test_regex(self):
         jsi = JSInterpreter(R'function f() { let a=[/[)\\]/]; return a[0]; }')
         self.assertEqual(jsi.call_function('f').pattern, r'[)\\]')
 
+    @unittest.skip('Not implemented')
+    def test_replace(self):
+        self._test('function f() { let a="data-name".replace("data-", ""); return a }',
+                   'name')
+        self._test('function f() { let a="data-name".replace(new RegExp("^.+-"), ""); return a; }',
+                   'name')
+        self._test('function f() { let a="data-name".replace(/^.+-/, ""); return a; }',
+                   'name')
+        self._test('function f() { let a="data-name".replace(/a/g, "o"); return a; }',
+                   'doto-nome')
+        self._test('function f() { let a="data-name".replaceAll("a", "o"); return a; }',
+                   'doto-nome')
+
     def test_char_code_at(self):
         jsi = JSInterpreter('function f(i){return "test".charCodeAt(i)}')
         self.assertEqual(jsi.call_function('f', 0), 116)
@@ -311,6 +326,11 @@ def test_negative(self):
         self._test('function f(){return 2    -    + + - -2;}', 0)
         self._test('function f(){return 2    +    - + - -2;}', 0)
 
+    @unittest.skip('Not implemented')
+    def test_packed(self):
+        jsi = JSInterpreter('''function f(p,a,c,k,e,d){while(c--)if(k[c])p=p.replace(new RegExp('\\b'+c.toString(a)+'\\b','g'),k[c]);return p}''')
+        self.assertEqual(jsi.call_function('f', '''h 7=g("1j");7.7h({7g:[{33:"w://7f-7e-7d-7c.v.7b/7a/79/78/77/76.74?t=73&s=2s&e=72&f=2t&71=70.0.0.1&6z=6y&6x=6w"}],6v:"w://32.v.u/6u.31",16:"r%",15:"r%",6t:"6s",6r:"",6q:"l",6p:"l",6o:"6n",6m:\'6l\',6k:"6j",9:[{33:"/2u?b=6i&n=50&6h=w://32.v.u/6g.31",6f:"6e"}],1y:{6d:1,6c:\'#6b\',6a:\'#69\',68:"67",66:30,65:r,},"64":{63:"%62 2m%m%61%5z%5y%5x.u%5w%5v%5u.2y%22 2k%m%1o%22 5t%m%1o%22 5s%m%1o%22 2j%m%5r%22 16%m%5q%22 15%m%5p%22 5o%2z%5n%5m%2z",5l:"w://v.u/d/1k/5k.2y",5j:[]},\'5i\':{"5h":"5g"},5f:"5e",5d:"w://v.u",5c:{},5b:l,1x:[0.25,0.50,0.75,1,1.25,1.5,2]});h 1m,1n,5a;h 59=0,58=0;h 7=g("1j");h 2x=0,57=0,56=0;$.55({54:{\'53-52\':\'2i-51\'}});7.j(\'4z\',6(x){c(5>0&&x.1l>=5&&1n!=1){1n=1;$(\'q.4y\').4x(\'4w\')}});7.j(\'13\',6(x){2x=x.1l});7.j(\'2g\',6(x){2w(x)});7.j(\'4v\',6(){$(\'q.2v\').4u()});6 2w(x){$(\'q.2v\').4t();c(1m)19;1m=1;17=0;c(4s.4r===l){17=1}$.4q(\'/2u?b=4p&2l=1k&4o=2t-4n-4m-2s-4l&4k=&4j=&4i=&17=\'+17,6(2r){$(\'#4h\').4g(2r)});$(\'.3-8-4f-4e:4d("4c")\').2h(6(e){2q();g().4b(0);g().4a(l)});6 2q(){h $14=$("<q />").2p({1l:"49",16:"r%",15:"r%",48:0,2n:0,2o:47,46:"45(10%, 10%, 10%, 0.4)","44-43":"42"});$("<41 />").2p({16:"60%",15:"60%",2o:40,"3z-2n":"3y"}).3x({\'2m\':\'/?b=3w&2l=1k\',\'2k\':\'0\',\'2j\':\'2i\'}).2f($14);$14.2h(6(){$(3v).3u();g().2g()});$14.2f($(\'#1j\'))}g().13(0);}6 3t(){h 9=7.1b(2e);2d.2c(9);c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==2e){2d.2c(\'!!=\'+i);7.1p(i)}}}}7.j(\'3s\',6(){g().1h("/2a/3r.29","3q 10 28",6(){g().13(g().27()+10)},"2b");$("q[26=2b]").23().21(\'.3-20-1z\');g().1h("/2a/3p.29","3o 10 28",6(){h 12=g().27()-10;c(12<0)12=0;g().13(12)},"24");$("q[26=24]").23().21(\'.3-20-1z\');});6 1i(){}7.j(\'3n\',6(){1i()});7.j(\'3m\',6(){1i()});7.j("k",6(y){h 9=7.1b();c(9.n<2)19;$(\'.3-8-3l-3k\').3j(6(){$(\'#3-8-a-k\').1e(\'3-8-a-z\');$(\'.3-a-k\').p(\'o-1f\',\'11\')});7.1h("/3i/3h.3g","3f 3e",6(){$(\'.3-1w\').3d(\'3-8-1v\');$(\'.3-8-1y, .3-8-1x\').p(\'o-1g\',\'11\');c($(\'.3-1w\').3c(\'3-8-1v\')){$(\'.3-a-k\').p(\'o-1g\',\'l\');$(\'.3-a-k\').p(\'o-1f\',\'l\');$(\'.3-8-a\').1e(\'3-8-a-z\');$(\'.3-8-a:1u\').3b(\'3-8-a-z\')}3a{$(\'.3-a-k\').p(\'o-1g\',\'11\');$(\'.3-a-k\').p(\'o-1f\',\'11\');$(\'.3-8-a:1u\').1e(\'3-8-a-z\')}},"39");7.j("38",6(y){1d.37(\'1c\',y.9[y.36].1a)});c(1d.1t(\'1c\')){35("1s(1d.1t(\'1c\'));",34)}});h 18;6 1s(1q){h 9=7.1b();c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==1q){c(i==18){19}18=i;7.1p(i)}}}}',36,270,'|||jw|||function|player|settings|tracks|submenu||if||||jwplayer|var||on|audioTracks|true|3D|length|aria|attr|div|100|||sx|filemoon|https||event|active||false|tt|seek|dd|height|width|adb|current_audio|return|name|getAudioTracks|default_audio|localStorage|removeClass|expanded|checked|addButton|callMeMaybe|vplayer|0fxcyc2ajhp1|position|vvplay|vvad|220|setCurrentAudioTrack|audio_name|for|audio_set|getItem|last|open|controls|playbackRates|captions|rewind|icon|insertAfter||detach|ff00||button|getPosition|sec|png|player8|ff11|log|console|track_name|appendTo|play|click|no|scrolling|frameborder|file_code|src|top|zIndex|css|showCCform|data|1662367683|383371|dl|video_ad|doPlay|prevt|mp4|3E||jpg|thumbs|file|300|setTimeout|currentTrack|setItem|audioTrackChanged|dualSound|else|addClass|hasClass|toggleClass|Track|Audio|svg|dualy|images|mousedown|buttons|topbar|playAttemptFailed|beforePlay|Rewind|fr|Forward|ff|ready|set_audio_track|remove|this|upload_srt|prop|50px|margin|1000001|iframe|center|align|text|rgba|background|1000000|left|absolute|pause|setCurrentCaptions|Upload|contains|item|content|html|fviews|referer|prem|embed|3e57249ef633e0d03bf76ceb8d8a4b65|216|83|hash|view|get|TokenZir|window|hide|show|complete|slow|fadeIn|video_ad_fadein|time||cache|Cache|Content|headers|ajaxSetup|v2done|tott|vastdone2|vastdone1|vvbefore|playbackRateControls|cast|aboutlink|FileMoon|abouttext|UHD|1870|qualityLabels|sites|GNOME_POWER|link|2Fiframe|3C|allowfullscreen|22360|22640|22no|marginheight|marginwidth|2FGNOME_POWER|2F0fxcyc2ajhp1|2Fe|2Ffilemoon|2F|3A||22https|3Ciframe|code|sharing|fontOpacity|backgroundOpacity|Tahoma|fontFamily|303030|backgroundColor|FFFFFF|color|userFontScale|thumbnails|kind|0fxcyc2ajhp10000|url|get_slides|start|startparam|none|preload|html5|primary|hlshtml|androidhls|duration|uniform|stretching|0fxcyc2ajhp1_xt|image|2048|sp|6871|asn|127|srv|43200|_g3XlBcu2lmD9oDexD2NLWSmah2Nu3XcDrl93m9PwXY|m3u8||master|0fxcyc2ajhp1_x|00076|01|hls2|to|s01|delivery|storage|moon|sources|setup'''.split('|')))
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_utils.py b/test/test_utils.py
index e1bf6ac20..a22f25d73 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -5,6 +5,7 @@
 import re
 import sys
 import unittest
+import warnings
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
@@ -112,6 +113,7 @@
     subtitles_filename,
     timeconvert,
     traverse_obj,
+    try_call,
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
@@ -123,6 +125,7 @@
     urlencode_postdata,
     urljoin,
     urshift,
+    variadic,
     version_tuple,
     xpath_attr,
     xpath_element,
@@ -1974,6 +1977,35 @@ def test_get_compatible_ext(self):
         self.assertEqual(get_compatible_ext(
             vcodecs=['av1'], acodecs=['mp4a'], vexts=['webm'], aexts=['m4a'], preferences=('webm', 'mkv')), 'mkv')
 
+    def test_try_call(self):
+        def total(*x, **kwargs):
+            return sum(x) + sum(kwargs.values())
+
+        self.assertEqual(try_call(None), None,
+                         msg='not a fn should give None')
+        self.assertEqual(try_call(lambda: 1), 1,
+                         msg='int fn with no expected_type should give int')
+        self.assertEqual(try_call(lambda: 1, expected_type=int), 1,
+                         msg='int fn with expected_type int should give int')
+        self.assertEqual(try_call(lambda: 1, expected_type=dict), None,
+                         msg='int fn with wrong expected_type should give None')
+        self.assertEqual(try_call(total, args=(0, 1, 0, ), expected_type=int), 1,
+                         msg='fn should accept arglist')
+        self.assertEqual(try_call(total, kwargs={'a': 0, 'b': 1, 'c': 0}, expected_type=int), 1,
+                         msg='fn should accept kwargs')
+        self.assertEqual(try_call(lambda: 1, expected_type=dict), None,
+                         msg='int fn with no expected_type should give None')
+        self.assertEqual(try_call(lambda x: {}, total, args=(42, ), expected_type=int), 42,
+                         msg='expect first int result with expected_type int')
+
+    def test_variadic(self):
+        self.assertEqual(variadic(None), (None, ))
+        self.assertEqual(variadic('spam'), ('spam', ))
+        self.assertEqual(variadic('spam', allowed_types=dict), 'spam')
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            self.assertEqual(variadic('spam', allowed_types=[dict]), 'spam')
+
     def test_traverse_obj(self):
         _TEST_DATA = {
             100: 100,
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index 8f9bc05d6..c48a2ff8a 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -139,17 +139,21 @@ def calc_percent(byte_counter, data_len):
     def format_percent(percent):
         return '  N/A%' if percent is None else f'{percent:>5.1f}%'
 
-    @staticmethod
-    def calc_eta(start, now, total, current):
+    @classmethod
+    def calc_eta(cls, start_or_rate, now_or_remaining, total=NO_DEFAULT, current=NO_DEFAULT):
+        if total is NO_DEFAULT:
+            rate, remaining = start_or_rate, now_or_remaining
+            if None in (rate, remaining):
+                return None
+            return int(float(remaining) / rate)
+
+        start, now = start_or_rate, now_or_remaining
         if total is None:
             return None
         if now is None:
             now = time.time()
-        dif = now - start
-        if current == 0 or dif < 0.001:  # One millisecond
-            return None
-        rate = float(current) / dif
-        return int((float(total) - float(current)) / rate)
+        rate = cls.calc_speed(start, now, current)
+        return rate and int((float(total) - float(current)) / rate)
 
     @staticmethod
     def calc_speed(start, now, bytes):
@@ -166,6 +170,12 @@ def format_speed(speed):
     def format_retries(retries):
         return 'inf' if retries == float('inf') else int(retries)
 
+    @staticmethod
+    def filesize_or_none(unencoded_filename):
+        if os.path.isfile(unencoded_filename):
+            return os.path.getsize(unencoded_filename)
+        return 0
+
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 8abf7760b..6770815ab 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -121,6 +121,11 @@ def _download_fragment(self, ctx, frag_url, info_dict, headers=None, request_dat
             'request_data': request_data,
             'ctx_id': ctx.get('ctx_id'),
         }
+        frag_resume_len = 0
+        if ctx['dl'].params.get('continuedl', True):
+            frag_resume_len = self.filesize_or_none(self.temp_name(fragment_filename))
+        fragment_info_dict['frag_resume_len'] = ctx['frag_resume_len'] = frag_resume_len
+
         success, _ = ctx['dl'].download(fragment_filename, fragment_info_dict)
         if not success:
             return False
@@ -155,9 +160,7 @@ def _append_fragment(self, ctx, frag_content):
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
-        if 'live' not in ctx:
-            ctx['live'] = False
-        if not ctx['live']:
+        if not ctx.setdefault('live', False):
             total_frags_str = '%d' % ctx['total_frags']
             ad_frags = ctx.get('ad_frags', 0)
             if ad_frags:
@@ -173,12 +176,11 @@ def _prepare_frag_download(self, ctx):
         })
         tmpfilename = self.temp_name(ctx['filename'])
         open_mode = 'wb'
-        resume_len = 0
 
         # Establish possible resume length
-        if os.path.isfile(encodeFilename(tmpfilename)):
+        resume_len = self.filesize_or_none(tmpfilename)
+        if resume_len > 0:
             open_mode = 'ab'
-            resume_len = os.path.getsize(encodeFilename(tmpfilename))
 
         # Should be initialized before ytdl file check
         ctx.update({
@@ -187,7 +189,9 @@ def _prepare_frag_download(self, ctx):
         })
 
         if self.__do_ytdl_file(ctx):
-            if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+            ytdl_file_exists = os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename'])))
+            continuedl = self.params.get('continuedl', True)
+            if continuedl and ytdl_file_exists:
                 self._read_ytdl_file(ctx)
                 is_corrupt = ctx.get('ytdl_corrupt') is True
                 is_inconsistent = ctx['fragment_index'] > 0 and resume_len == 0
@@ -201,7 +205,12 @@ def _prepare_frag_download(self, ctx):
                     if 'ytdl_corrupt' in ctx:
                         del ctx['ytdl_corrupt']
                     self._write_ytdl_file(ctx)
+
             else:
+                if not continuedl:
+                    if ytdl_file_exists:
+                        self._read_ytdl_file(ctx)
+                    ctx['fragment_index'] = resume_len = 0
                 self._write_ytdl_file(ctx)
                 assert ctx['fragment_index'] == 0
 
@@ -274,12 +283,10 @@ def frag_progress_hook(s):
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
                 state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
-                if not ctx['live']:
-                    state['eta'] = self.calc_eta(
-                        start, time_now, estimated_size - resume_len,
-                        state['downloaded_bytes'] - resume_len)
                 ctx['speed'] = state['speed'] = self.calc_speed(
-                    ctx['fragment_started'], time_now, frag_downloaded_bytes)
+                    ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx['frag_resume_len'])
+                if not ctx['live']:
+                    state['eta'] = self.calc_eta(state['speed'], estimated_size - state['downloaded_bytes'])
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state, info_dict)
 
@@ -297,7 +304,7 @@ def _finish_frag_download(self, ctx, info_dict):
 
         to_file = ctx['tmpfilename'] != '-'
         if to_file:
-            downloaded_bytes = os.path.getsize(encodeFilename(ctx['tmpfilename']))
+            downloaded_bytes = self.filesize_or_none(ctx['filename'])
         else:
             downloaded_bytes = ctx['complete_frags_downloaded_bytes']
 
diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index 79f69b5d0..e785f0d4e 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -150,7 +150,8 @@ def establish_connection():
                     # Content-Range is either not present or invalid. Assuming remote webserver is
                     # trying to send the whole file, resume is not possible, so wiping the local file
                     # and performing entire redownload
-                    self.report_unable_to_resume()
+                    elif range_start > 0:
+                        self.report_unable_to_resume()
                     ctx.resume_len = 0
                     ctx.open_mode = 'wb'
                 ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
diff --git a/yt_dlp/extractor/aenetworks.py b/yt_dlp/extractor/aenetworks.py
index d7c401016..f049a0fb3 100644
--- a/yt_dlp/extractor/aenetworks.py
+++ b/yt_dlp/extractor/aenetworks.py
@@ -3,6 +3,8 @@
     ExtractorError,
     GeoRestrictedError,
     int_or_none,
+    remove_start,
+    traverse_obj,
     update_url_query,
     urlencode_postdata,
 )
@@ -72,7 +74,14 @@ def _extract_aetn_info(self, domain, filter_key, filter_value, url):
         requestor_id, brand = self._DOMAIN_MAP[domain]
         result = self._download_json(
             'https://feeds.video.aetnd.com/api/v2/%s/videos' % brand,
-            filter_value, query={'filter[%s]' % filter_key: filter_value})['results'][0]
+            filter_value, query={'filter[%s]' % filter_key: filter_value})
+        result = traverse_obj(
+            result, ('results',
+                     lambda k, v: k == 0 and v[filter_key] == filter_value),
+            get_all=False)
+        if not result:
+            raise ExtractorError('Show not found in A&E feed (too new?)', expected=True,
+                                 video_id=remove_start(filter_value, '/'))
         title = result['title']
         video_id = result['id']
         media_url = result['publicUrl']
@@ -123,7 +132,7 @@ class AENetworksIE(AENetworksBaseIE):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
-        'skip': 'This video is only available for users of participating TV providers.',
+        'skip': 'Geo-restricted - This content is not available in your location.'
     }, {
         'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
         'info_dict': {
@@ -140,6 +149,7 @@ class AENetworksIE(AENetworksBaseIE):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+        'skip': 'This video is only available for users of participating TV providers.',
     }, {
         'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
         'only_matching': True
@@ -303,6 +313,7 @@ def _real_extract(self, url):
 class HistoryPlayerIE(AENetworksBaseIE):
     IE_NAME = 'history:player'
     _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|biography)\.com)/player/(?P<id>\d+)'
+    _TESTS = []
 
     def _real_extract(self, url):
         domain, video_id = self._match_valid_url(url).groups()
diff --git a/yt_dlp/extractor/litv.py b/yt_dlp/extractor/litv.py
index 0b792fb96..19b298ec6 100644
--- a/yt_dlp/extractor/litv.py
+++ b/yt_dlp/extractor/litv.py
@@ -4,8 +4,8 @@
 from ..utils import (
     ExtractorError,
     int_or_none,
-    traverse_obj,
     smuggle_url,
+    traverse_obj,
     unsmuggle_url,
 )
 
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index bd38900f2..654bf5e6b 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -66,7 +66,6 @@
     variadic,
 )
 
-
 STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client'
 # any clients starting with _ cannot be explicitly requested by the user
 INNERTUBE_CLIENTS = {
@@ -2994,17 +2993,14 @@ def _parse_sig_js(self, jscode):
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              # Obsolete patterns
-             r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'("|\')signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 
@@ -4883,7 +4879,7 @@ def _extract_metadata_from_tabs(self, item_id, data):
         metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
         if metadata_renderer:
             channel_id = traverse_obj(metadata_renderer, ('externalId', {self.ucid_or_none}),
-                                                         ('channelUrl', {self.ucid_from_url}))
+                                      ('channelUrl', {self.ucid_from_url}))
             info.update({
                 'channel': metadata_renderer.get('title'),
                 'channel_id': channel_id,
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index 82974fb27..1ef1f0823 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -443,7 +443,7 @@ def dict_item(key, val):
                 err = e
 
             pending = (None, False)
-            m = re.match(r'catch\s*(?P<err>\(\s*{_NAME_RE}\s*\))?\{{'.format(**globals()), expr)
+            m = re.match(fr'catch\s*(?P<err>\(\s*{_NAME_RE}\s*\))?\{{', expr)
             if m:
                 sub_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
                 if err:
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index afcb2a164..238b0fe69 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -130,8 +130,13 @@ def random_user_agent():
 }
 
 
-NO_DEFAULT = object()
-IDENTITY = lambda x: x
+class NO_DEFAULT:
+    pass
+
+
+def IDENTITY(x):
+    return x
+
 
 ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
@@ -3223,6 +3228,9 @@ def is_iterable_like(x, allowed_types=collections.abc.Iterable, blocked_types=NO
 
 
 def variadic(x, allowed_types=NO_DEFAULT):
+    if not isinstance(allowed_types, (tuple, type)):
+        deprecation_warning('allowed_types should be a tuple or a type')
+        allowed_types = tuple(allowed_types)
     return x if is_iterable_like(x, blocked_types=allowed_types) else (x, )
 
 

From 15b2d3db1d40b0437fca79d8874d392aa54b3cdd Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 24 May 2023 22:13:24 +0530
Subject: [PATCH 273/871] [misc] Add automatic duplicate issue detection

---
 .github/workflows/potential-duplicates.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 .github/workflows/potential-duplicates.yml

diff --git a/.github/workflows/potential-duplicates.yml b/.github/workflows/potential-duplicates.yml
new file mode 100644
index 000000000..1521ae20c
--- /dev/null
+++ b/.github/workflows/potential-duplicates.yml
@@ -0,0 +1,20 @@
+name: Potential Duplicates
+on:
+  issues:
+    types: [opened, edited]
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: wow-actions/potential-duplicates@v1
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          label: potential-duplicate
+          state: all
+          threshold: 0.7
+          comment: |
+            This issue is potentially a duplicate of one of the following issues:
+            {{#issues}}
+              - #{{ number }} ({{ accuracy }}%)
+            {{/issues}}

From 7aeda6cc9e73ada0b0a0b6a6748c66bef63a20a8 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 24 May 2023 23:05:20 +0530
Subject: [PATCH 274/871] [jsinterp] Do not compile regex

---
 test/test_jsinterp.py | 4 +++-
 yt_dlp/jsinterp.py    | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 96274116b..4d44e6efe 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -8,7 +8,6 @@
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import math
-import re
 
 from yt_dlp.jsinterp import JS_Undefined, JSInterpreter
 
@@ -275,7 +274,9 @@ def test_object(self):
 
     def test_regex(self):
         self._test('function f() { let a=/,,[/,913,/](,)}/; }', None)
+        self._test('function f() { let a=/,,[/,913,/](,)}/; return a; }', R'/,,[/,913,/](,)}/0')
 
+        R'''  # We are not compiling regex
         jsi = JSInterpreter('function f() { let a=/,,[/,913,/](,)}/; return a; }')
         self.assertIsInstance(jsi.call_function('f'), re.Pattern)
 
@@ -287,6 +288,7 @@ def test_regex(self):
 
         jsi = JSInterpreter(R'function f() { let a=[/[)\\]/]; return a[0]; }')
         self.assertEqual(jsi.call_function('f').pattern, r'[)\\]')
+        '''
 
     @unittest.skip('Not implemented')
     def test_replace(self):
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index 1ef1f0823..7c7940efd 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -352,8 +352,10 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100):
             inner, outer = self._separate(expr, expr[0], 1)
             if expr[0] == '/':
                 flags, outer = self._regex_flags(outer)
+                # We don't support regex methods yet, so no point compiling it
+                inner = f'{inner}/{flags}'
                 # Avoid https://github.com/python/cpython/issues/74534
-                inner = re.compile(inner[1:].replace('[[', r'[\['), flags=flags)
+                # inner = re.compile(inner[1:].replace('[[', r'[\['), flags=flags)
             else:
                 inner = json.loads(js_to_json(f'{inner}{expr[0]}', strict=True))
             if not outer:

From 8417f26b8a819cd7ffcd4e000ca3e45033e670fb Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 24 May 2023 20:35:07 +0200
Subject: [PATCH 275/871] [core] Implement `--color` flag (#6904)

Authored by: Grub4K
---
 README.md                   |  9 +++++++--
 yt_dlp/YoutubeDL.py         | 36 +++++++++++++++++++++++++++++++-----
 yt_dlp/__init__.py          |  6 +++++-
 yt_dlp/downloader/common.py |  3 ++-
 yt_dlp/options.py           | 24 +++++++++++++++++++++---
 5 files changed, 66 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index d0eaba747..25ed3b844 100644
--- a/README.md
+++ b/README.md
@@ -425,8 +425,12 @@ ## General Options:
     --no-wait-for-video             Do not wait for scheduled streams (default)
     --mark-watched                  Mark videos watched (even with --simulate)
     --no-mark-watched               Do not mark videos watched (default)
-    --no-colors                     Do not emit color codes in output (Alias:
-                                    --no-colours)
+    --color [STREAM:]POLICY         Whether to emit color codes in output,
+                                    optionally prefixed by the STREAM (stdout or
+                                    stderr) to apply the setting to. Can be one
+                                    of "always", "auto" (default), "never", or
+                                    "no_color" (use non color terminal
+                                    sequences). Can be used multiple times
     --compat-options OPTS           Options that can help keep compatibility
                                     with youtube-dl or youtube-dlc
                                     configurations by reverting some of the
@@ -2148,6 +2152,7 @@ #### Redundant options
     --playlist-end NUMBER            -I :NUMBER
     --playlist-reverse               -I ::-1
     --no-playlist-reverse            Default
+    --no-colors                      --color no_color
 
 
 #### Not recommended
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index cd82b2772..e1e558836 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -415,7 +415,12 @@ class YoutubeDL:
                        - Raise utils.DownloadCancelled(msg) to abort remaining
                          downloads when a video is rejected.
                        match_filter_func in utils.py is one example for this.
-    no_color:          Do not emit color codes in output.
+    color:             A Dictionary with output stream names as keys
+                       and their respective color policy as values.
+                       Can also just be a single color policy,
+                       in which case it applies to all outputs.
+                       Valid stream names are 'stdout' and 'stderr'.
+                       Valid color policies are one of 'always', 'auto', 'no_color' or 'never'.
     geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
                        HTTP header
     geo_bypass_country:
@@ -537,6 +542,7 @@ class YoutubeDL:
                        data will be downloaded and processed by extractor.
                        You can reduce network I/O by disabling it if you don't
                        care about HLS. (only for youtube)
+    no_color:          Same as `color='no_color'`
     """
 
     _NUMERIC_FIELDS = {
@@ -603,9 +609,24 @@ def __init__(self, params=None, auto_init=True):
         except Exception as e:
             self.write_debug(f'Failed to enable VT mode: {e}')
 
+        if self.params.get('no_color'):
+            if self.params.get('color') is not None:
+                self.report_warning('Overwriting params from "color" with "no_color"')
+            self.params['color'] = 'no_color'
+
+        term_allow_color = os.environ.get('TERM', '').lower() != 'dumb'
+
+        def process_color_policy(stream):
+            stream_name = {sys.stdout: 'stdout', sys.stderr: 'stderr'}[stream]
+            policy = traverse_obj(self.params, ('color', (stream_name, None), {str}), get_all=False)
+            if policy in ('auto', None):
+                return term_allow_color and supports_terminal_sequences(stream)
+            assert policy in ('always', 'never', 'no_color')
+            return {'always': True, 'never': False}.get(policy, policy)
+
         self._allow_colors = Namespace(**{
-            type_: not self.params.get('no_color') and supports_terminal_sequences(stream)
-            for type_, stream in self._out_files.items_ if type_ != 'console'
+            name: process_color_policy(stream)
+            for name, stream in self._out_files.items_ if name != 'console'
         })
 
         # The code is left like this to be reused for future deprecations
@@ -974,7 +995,7 @@ def _format_text(self, handle, allow_colors, text, f, fallback=None, *, test_enc
             text = text.encode(encoding, 'ignore').decode(encoding)
             if fallback is not None and text != original_text:
                 text = fallback
-        return format_text(text, f) if allow_colors else text if fallback is None else fallback
+        return format_text(text, f) if allow_colors is True else text if fallback is None else fallback
 
     def _format_out(self, *args, **kwargs):
         return self._format_text(self._out_files.out, self._allow_colors.out, *args, **kwargs)
@@ -3769,9 +3790,14 @@ def print_debug_header(self):
 
         def get_encoding(stream):
             ret = str(getattr(stream, 'encoding', 'missing (%s)' % type(stream).__name__))
+            additional_info = []
+            if os.environ.get('TERM', '').lower() == 'dumb':
+                additional_info.append('dumb')
             if not supports_terminal_sequences(stream):
                 from .utils import WINDOWS_VT_MODE  # Must be imported locally
-                ret += ' (No VT)' if WINDOWS_VT_MODE is False else ' (No ANSI)'
+                additional_info.append('No VT' if WINDOWS_VT_MODE is False else 'No ANSI')
+            if additional_info:
+                ret = f'{ret} ({",".join(additional_info)})'
             return ret
 
         encoding_str = 'Encodings: locale %s, fs %s, pref %s, %s' % (
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 9563d784a..137c9503f 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -436,6 +436,10 @@ def metadataparser_actions(f):
         elif ed and proto == 'default':
             default_downloader = ed.get_basename()
 
+    for policy in opts.color.values():
+        if policy not in ('always', 'auto', 'no_color', 'never'):
+            raise ValueError(f'"{policy}" is not a valid color policy')
+
     warnings, deprecation_warnings = [], []
 
     # Common mistake: -f best
@@ -894,7 +898,7 @@ def parse_options(argv=None):
         'playlist_items': opts.playlist_items,
         'xattr_set_filesize': opts.xattr_set_filesize,
         'match_filter': opts.match_filter,
-        'no_color': opts.no_color,
+        'color': opts.color,
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
         'hls_use_mpegts': opts.hls_use_mpegts,
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index c48a2ff8a..477ec3c8a 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -296,7 +296,8 @@ def _prepare_multiline_status(self, lines=1):
             self._multiline = BreaklineStatusPrinter(self.ydl._out_files.out, lines)
         else:
             self._multiline = MultilinePrinter(self.ydl._out_files.out, lines, not self.params.get('quiet'))
-        self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self.params.get('no_color')
+        self._multiline.allow_colors = self.ydl._allow_colors.out and self.ydl._allow_colors.out != 'no_color'
+        self._multiline._HAVE_FULLCAP = self.ydl._allow_colors.out
 
     def _finish_multiline_status(self):
         self._multiline.end()
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 838d79fcb..fecc27403 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -34,6 +34,7 @@
     join_nonempty,
     orderedSet_from_options,
     remove_end,
+    variadic,
     write_string,
 )
 from .version import CHANNEL, __version__
@@ -250,7 +251,7 @@ def _dict_from_options_callback(
             if multiple_args:
                 val = [val, *value[1:]]
         elif default_key is not None:
-            keys, val = [default_key], value
+            keys, val = variadic(default_key), value
         else:
             raise optparse.OptionValueError(
                 f'wrong {opt_str} formatting; it should be {option.metavar}, not "{value}"')
@@ -440,8 +441,25 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         help='Do not mark videos watched (default)')
     general.add_option(
         '--no-colors', '--no-colours',
-        action='store_true', dest='no_color', default=False,
-        help='Do not emit color codes in output (Alias: --no-colours)')
+        action='store_const', dest='color', const={
+            'stdout': 'no_color',
+            'stderr': 'no_color',
+        },
+        help=optparse.SUPPRESS_HELP)
+    general.add_option(
+        '--color',
+        dest='color', metavar='[STREAM:]POLICY', default={}, type='str',
+        action='callback', callback=_dict_from_options_callback,
+        callback_kwargs={
+            'allowed_keys': 'stdout|stderr',
+            'default_key': ['stdout', 'stderr'],
+            'process': str.strip,
+        }, help=(
+            'Whether to emit color codes in output, optionally prefixed by '
+            'the STREAM (stdout or stderr) to apply the setting to. '
+            'Can be one of "always", "auto" (default), "never", or '
+            '"no_color" (use non color terminal sequences). '
+            'Can be used multiple times'))
     general.add_option(
         '--compat-options',
         metavar='OPTS', dest='compat_opts', default=set(), type='str',

From 032de83ea9ff2f4977d9c71a93bbc1775597b762 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 24 May 2023 20:45:15 +0200
Subject: [PATCH 276/871] [extractor/crunchyroll] Rework with support for
 movies, music and artists (#6237)

This adds `CrunchyrollMusicIE` and `CrunchyrollArtistIE` extractors using the new, reworked base class and expands the `CrunchyrollBetaIE` with support for movies and movie listings and more complete metadata extraction

Authored by: Grub4K
---
 yt_dlp/extractor/_extractors.py |   2 +
 yt_dlp/extractor/crunchyroll.py | 692 +++++++++++++++++++++++---------
 2 files changed, 499 insertions(+), 195 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index fd2bfa9a1..8984d4b16 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -406,6 +406,8 @@
 from .crunchyroll import (
     CrunchyrollBetaIE,
     CrunchyrollBetaShowIE,
+    CrunchyrollMusicIE,
+    CrunchyrollArtistIE,
 )
 from .cspan import CSpanIE, CSpanCongressIE
 from .ctsnews import CtsNewsIE
diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py
index 1abffcd74..d4a21616b 100644
--- a/yt_dlp/extractor/crunchyroll.py
+++ b/yt_dlp/extractor/crunchyroll.py
@@ -1,28 +1,37 @@
 import base64
-import urllib.parse
+import urllib.error
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     float_or_none,
     format_field,
+    int_or_none,
     join_nonempty,
+    parse_age_limit,
+    parse_count,
     parse_iso8601,
     qualities,
+    remove_start,
+    time_seconds,
     traverse_obj,
-    try_get,
+    url_or_none,
+    urlencode_postdata,
 )
 
 
 class CrunchyrollBaseIE(InfoExtractor):
-    _LOGIN_URL = 'https://www.crunchyroll.com/welcome/login'
+    _BASE_URL = 'https://www.crunchyroll.com'
     _API_BASE = 'https://api.crunchyroll.com'
     _NETRC_MACHINE = 'crunchyroll'
-    params = None
+    _AUTH_HEADERS = None
+    _API_ENDPOINT = None
+    _BASIC_AUTH = None
+    _QUERY = {}
 
     @property
     def is_logged_in(self):
-        return self._get_cookies(self._LOGIN_URL).get('etp_rt')
+        return self._get_cookies(self._BASE_URL).get('etp_rt')
 
     def _perform_login(self, username, password):
         if self.is_logged_in:
@@ -35,7 +44,7 @@ def _perform_login(self, username, password):
                 'device_id': 'whatvalueshouldbeforweb',
                 'device_type': 'com.crunchyroll.static',
                 'access_token': 'giKq5eY27ny3cqz',
-                'referer': self._LOGIN_URL
+                'referer': f'{self._BASE_URL}/welcome/login'
             })
         if upsell_response['code'] != 'ok':
             raise ExtractorError('Could not get session id')
@@ -43,149 +52,89 @@ def _perform_login(self, username, password):
 
         login_response = self._download_json(
             f'{self._API_BASE}/login.1.json', None, 'Logging in',
-            data=urllib.parse.urlencode({
+            data=urlencode_postdata({
                 'account': username,
                 'password': password,
                 'session_id': session_id
-            }).encode('ascii'))
+            }))
         if login_response['code'] != 'ok':
             raise ExtractorError('Login failed. Server message: %s' % login_response['message'], expected=True)
         if not self.is_logged_in:
             raise ExtractorError('Login succeeded but did not set etp_rt cookie')
 
-    def _get_embedded_json(self, webpage, display_id):
-        initial_state = self._parse_json(self._search_regex(
-            r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'initial state'), display_id)
-        app_config = self._parse_json(self._search_regex(
-            r'__APP_CONFIG__\s*=\s*({.+?})\s*;', webpage, 'app config'), display_id)
-        return initial_state, app_config
+    def _update_query(self, lang):
+        if lang in CrunchyrollBaseIE._QUERY:
+            return
 
-    def _get_params(self, lang):
-        if not CrunchyrollBaseIE.params:
-            if self._get_cookies(f'https://www.crunchyroll.com/{lang}').get('etp_rt'):
-                grant_type, key = 'etp_rt_cookie', 'accountAuthClientId'
-            else:
-                grant_type, key = 'client_id', 'anonClientId'
+        webpage = self._download_webpage(
+            f'{self._BASE_URL}/{lang}', None, note=f'Retrieving main page (lang={lang or None})')
 
-            initial_state, app_config = self._get_embedded_json(self._download_webpage(
-                f'https://www.crunchyroll.com/{lang}', None, note='Retrieving main page'), None)
-            api_domain = app_config['cxApiParams']['apiDomain'].replace('beta.crunchyroll.com', 'www.crunchyroll.com')
+        initial_state = self._search_json(r'__INITIAL_STATE__\s*=', webpage, 'initial state', None)
+        CrunchyrollBaseIE._QUERY[lang] = traverse_obj(initial_state, {
+            'locale': ('localization', 'locale'),
+        }) or None
 
-            auth_response = self._download_json(
-                f'{api_domain}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
-                headers={
-                    'Authorization': 'Basic ' + str(base64.b64encode(('%s:' % app_config['cxApiParams'][key]).encode('ascii')), 'ascii')
-                }, data=f'grant_type={grant_type}'.encode('ascii'))
-            policy_response = self._download_json(
-                f'{api_domain}/index/v2', None, note='Retrieving signed policy',
-                headers={
-                    'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']
-                })
-            cms = policy_response.get('cms_web')
-            bucket = cms['bucket']
-            params = {
-                'Policy': cms['policy'],
-                'Signature': cms['signature'],
-                'Key-Pair-Id': cms['key_pair_id']
-            }
-            locale = traverse_obj(initial_state, ('localization', 'locale'))
-            if locale:
-                params['locale'] = locale
-            CrunchyrollBaseIE.params = (api_domain, bucket, params)
-        return CrunchyrollBaseIE.params
+        if CrunchyrollBaseIE._BASIC_AUTH:
+            return
 
+        app_config = self._search_json(r'__APP_CONFIG__\s*=', webpage, 'app config', None)
+        cx_api_param = app_config['cxApiParams']['accountAuthClientId' if self.is_logged_in else 'anonClientId']
+        self.write_debug(f'Using cxApiParam={cx_api_param}')
+        CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode()
 
-class CrunchyrollBetaIE(CrunchyrollBaseIE):
-    IE_NAME = 'crunchyroll'
-    _VALID_URL = r'''(?x)
-        https?://(?:beta|www)\.crunchyroll\.com/
-        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
-        watch/(?P<id>\w+)
-        (?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)'''
-    _TESTS = [{
-        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y/to-the-future',
-        'info_dict': {
-            'id': 'GY2P1Q98Y',
-            'ext': 'mp4',
-            'duration': 1380.241,
-            'timestamp': 1459632600,
-            'description': 'md5:a022fbec4fbb023d43631032c91ed64b',
-            'title': 'World Trigger Episode 73 – To the Future',
-            'upload_date': '20160402',
-            'series': 'World Trigger',
-            'series_id': 'GR757DMKY',
-            'season': 'World Trigger',
-            'season_id': 'GR9P39NJ6',
-            'season_number': 1,
-            'episode': 'To the Future',
-            'episode_number': 73,
-            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$',
-            'chapters': 'count:2',
-        },
-        'params': {'skip_download': 'm3u8', 'format': 'all[format_id~=hardsub]'},
-    }, {
-        'url': 'https://www.crunchyroll.com/watch/GYE5WKQGR',
-        'info_dict': {
-            'id': 'GYE5WKQGR',
-            'ext': 'mp4',
-            'duration': 366.459,
-            'timestamp': 1476788400,
-            'description': 'md5:74b67283ffddd75f6e224ca7dc031e76',
-            'title': 'SHELTER Episode  – Porter Robinson presents Shelter the Animation',
-            'upload_date': '20161018',
-            'series': 'SHELTER',
-            'series_id': 'GYGG09WWY',
-            'season': 'SHELTER',
-            'season_id': 'GR09MGK4R',
-            'season_number': 1,
-            'episode': 'Porter Robinson presents Shelter the Animation',
-            'episode_number': 0,
-            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$',
-            'chapters': 'count:0',
-        },
-        'params': {'skip_download': True},
-        'skip': 'Video is Premium only',
-    }, {
-        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y',
-        'only_matching': True,
-    }, {
-        'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
-        'only_matching': True,
-    }]
+    def _update_auth(self):
+        if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_REFRESH > time_seconds():
+            return
 
-    def _real_extract(self, url):
-        lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
-        api_domain, bucket, params = self._get_params(lang)
+        assert CrunchyrollBaseIE._BASIC_AUTH, '_update_query needs to be called at least one time beforehand'
+        grant_type = 'etp_rt_cookie' if self.is_logged_in else 'client_id'
+        auth_response = self._download_json(
+            f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
+            headers={'Authorization': CrunchyrollBaseIE._BASIC_AUTH}, data=f'grant_type={grant_type}'.encode())
 
-        episode_response = self._download_json(
-            f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
-            note='Retrieving episode metadata', query=params)
-        if episode_response.get('is_premium_only') and not bucket.endswith('crunchyroll'):
-            if self.is_logged_in:
-                raise ExtractorError('This video is for premium members only', expected=True)
-            else:
-                self.raise_login_required('This video is for premium members only')
+        CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']}
+        CrunchyrollBaseIE._AUTH_REFRESH = time_seconds(seconds=traverse_obj(auth_response, ('expires_in', {float_or_none}), default=300) - 10)
 
-        stream_response = self._download_json(
-            f'{api_domain}{episode_response["__links__"]["streams"]["href"]}', display_id,
-            note='Retrieving stream info', query=params)
-        get_streams = lambda name: (traverse_obj(stream_response, name) or {}).items()
+    def _call_base_api(self, endpoint, internal_id, lang, note=None, query={}):
+        self._update_query(lang)
+        self._update_auth()
 
-        requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
-        hardsub_preference = qualities(requested_hardsubs[::-1])
+        if not endpoint.startswith('/'):
+            endpoint = f'/{endpoint}'
+
+        return self._download_json(
+            f'{self._BASE_URL}{endpoint}', internal_id, note or f'Calling API: {endpoint}',
+            headers=CrunchyrollBaseIE._AUTH_HEADERS, query={**CrunchyrollBaseIE._QUERY[lang], **query})
+
+    def _call_api(self, path, internal_id, lang, note='api', query={}):
+        if not path.startswith(f'/content/v2/{self._API_ENDPOINT}/'):
+            path = f'/content/v2/{self._API_ENDPOINT}/{path}'
+
+        try:
+            result = self._call_base_api(
+                path, internal_id, lang, f'Downloading {note} JSON ({self._API_ENDPOINT})', query=query)
+        except ExtractorError as error:
+            if isinstance(error.cause, urllib.error.HTTPError) and error.cause.code == 404:
+                return None
+            raise
+
+        if not result:
+            raise ExtractorError(f'Unexpected response when downloading {note} JSON')
+        return result
+
+    def _extract_formats(self, stream_response, display_id=None):
         requested_formats = self._configuration_arg('format') or ['adaptive_hls']
-
         available_formats = {}
-        for stream_type, streams in get_streams('streams'):
+        for stream_type, streams in traverse_obj(
+                stream_response, (('streams', ('data', 0)), {dict.items}, ...)):
             if stream_type not in requested_formats:
                 continue
-            for stream in streams.values():
-                if not stream.get('url'):
-                    continue
+            for stream in traverse_obj(streams, lambda _, v: v['url']):
                 hardsub_lang = stream.get('hardsub_locale') or ''
                 format_id = join_nonempty(stream_type, format_field(stream, 'hardsub_locale', 'hardsub-%s'))
                 available_formats[hardsub_lang] = (stream_type, format_id, hardsub_lang, stream['url'])
 
+        requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
         if '' in available_formats and 'all' not in requested_hardsubs:
             full_format_langs = set(requested_hardsubs)
             self.to_screen(
@@ -196,6 +145,8 @@ def _real_extract(self, url):
         else:
             full_format_langs = set(map(str.lower, available_formats))
 
+        audio_locale = traverse_obj(stream_response, ((None, 'meta'), 'audio_locale'), get_all=False)
+        hardsub_preference = qualities(requested_hardsubs[::-1])
         formats = []
         for stream_type, format_id, hardsub_lang, stream_url in available_formats.values():
             if stream_type.endswith('hls'):
@@ -214,63 +165,292 @@ def _real_extract(self, url):
                 continue
             for f in adaptive_formats:
                 if f.get('acodec') != 'none':
-                    f['language'] = stream_response.get('audio_locale')
+                    f['language'] = audio_locale
                 f['quality'] = hardsub_preference(hardsub_lang.lower())
             formats.extend(adaptive_formats)
 
-        chapters = None
+        return formats
+
+    def _extract_subtitles(self, data):
+        subtitles = {}
+
+        for locale, subtitle in traverse_obj(data, ((None, 'meta'), 'subtitles', {dict.items}, ...)):
+            subtitles[locale] = [traverse_obj(subtitle, {'url': 'url', 'ext': 'format'})]
+
+        return subtitles
+
+
+class CrunchyrollCmsBaseIE(CrunchyrollBaseIE):
+    _API_ENDPOINT = 'cms'
+    _CMS_EXPIRY = None
+
+    def _call_cms_api_signed(self, path, internal_id, lang, note='api'):
+        if not CrunchyrollCmsBaseIE._CMS_EXPIRY or CrunchyrollCmsBaseIE._CMS_EXPIRY <= time_seconds():
+            response = self._call_base_api('index/v2', None, lang, 'Retrieving signed policy')['cms_web']
+            CrunchyrollCmsBaseIE._CMS_QUERY = {
+                'Policy': response['policy'],
+                'Signature': response['signature'],
+                'Key-Pair-Id': response['key_pair_id'],
+            }
+            CrunchyrollCmsBaseIE._CMS_BUCKET = response['bucket']
+            CrunchyrollCmsBaseIE._CMS_EXPIRY = parse_iso8601(response['expires']) - 10
+
+        if not path.startswith('/cms/v2'):
+            path = f'/cms/v2{CrunchyrollCmsBaseIE._CMS_BUCKET}/{path}'
+
+        return self._call_base_api(
+            path, internal_id, lang, f'Downloading {note} JSON (signed cms)', query=CrunchyrollCmsBaseIE._CMS_QUERY)
+
+
+class CrunchyrollBetaIE(CrunchyrollCmsBaseIE):
+    IE_NAME = 'crunchyroll'
+    _VALID_URL = r'''(?x)
+        https?://(?:beta\.|www\.)?crunchyroll\.com/
+        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        watch/(?!concert|musicvideo)(?P<id>\w+)'''
+    _TESTS = [{
+        # Premium only
+        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y/to-the-future',
+        'info_dict': {
+            'id': 'GY2P1Q98Y',
+            'ext': 'mp4',
+            'duration': 1380.241,
+            'timestamp': 1459632600,
+            'description': 'md5:a022fbec4fbb023d43631032c91ed64b',
+            'title': 'World Trigger Episode 73 – To the Future',
+            'upload_date': '20160402',
+            'series': 'World Trigger',
+            'series_id': 'GR757DMKY',
+            'season': 'World Trigger',
+            'season_id': 'GR9P39NJ6',
+            'season_number': 1,
+            'episode': 'To the Future',
+            'episode_number': 73,
+            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'chapters': 'count:2',
+            'age_limit': 14,
+            'like_count': int,
+            'dislike_count': int,
+        },
+        'params': {'skip_download': 'm3u8', 'format': 'all[format_id~=hardsub]'},
+    }, {
+        # Premium only
+        'url': 'https://www.crunchyroll.com/watch/GYE5WKQGR',
+        'info_dict': {
+            'id': 'GYE5WKQGR',
+            'ext': 'mp4',
+            'duration': 366.459,
+            'timestamp': 1476788400,
+            'description': 'md5:74b67283ffddd75f6e224ca7dc031e76',
+            'title': 'SHELTER – Porter Robinson presents Shelter the Animation',
+            'upload_date': '20161018',
+            'series': 'SHELTER',
+            'series_id': 'GYGG09WWY',
+            'season': 'SHELTER',
+            'season_id': 'GR09MGK4R',
+            'season_number': 1,
+            'episode': 'Porter Robinson presents Shelter the Animation',
+            'episode_number': 0,
+            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'age_limit': 14,
+            'like_count': int,
+            'dislike_count': int,
+        },
+        'params': {'skip_download': True},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/GJWU2VKK3/cherry-blossom-meeting-and-a-coming-blizzard',
+        'info_dict': {
+            'id': 'GJWU2VKK3',
+            'ext': 'mp4',
+            'duration': 1420.054,
+            'description': 'md5:2d1c67c0ec6ae514d9c30b0b99a625cd',
+            'title': 'The Ice Guy and His Cool Female Colleague Episode 1 – Cherry Blossom Meeting and a Coming Blizzard',
+            'series': 'The Ice Guy and His Cool Female Colleague',
+            'series_id': 'GW4HM75NP',
+            'season': 'The Ice Guy and His Cool Female Colleague',
+            'season_id': 'GY9PC21VE',
+            'season_number': 1,
+            'episode': 'Cherry Blossom Meeting and a Coming Blizzard',
+            'episode_number': 1,
+            'chapters': 'count:2',
+            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'timestamp': 1672839000,
+            'upload_date': '20230104',
+            'age_limit': 14,
+            'like_count': int,
+            'dislike_count': int,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/GM8F313NQ',
+        'info_dict': {
+            'id': 'GM8F313NQ',
+            'ext': 'mp4',
+            'title': 'Garakowa -Restore the World-',
+            'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608',
+            'duration': 3996.104,
+            'age_limit': 13,
+            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/G62PEZ2E6',
+        'info_dict': {
+            'id': 'G62PEZ2E6',
+            'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608',
+            'age_limit': 13,
+            'duration': 65.138,
+            'title': 'Garakowa -Restore the World-',
+        },
+        'playlist_mincount': 5,
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y',
+        'only_matching': True,
+    }, {
+        'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
+        'only_matching': True,
+    }]
+    # We want to support lazy playlist filtering and movie listings cannot be inside a playlist
+    _RETURN_TYPE = 'video'
+
+    def _real_extract(self, url):
+        lang, internal_id = self._match_valid_url(url).group('lang', 'id')
+
+        # We need to use unsigned API call to allow ratings query string
+        response = traverse_obj(self._call_api(
+            f'objects/{internal_id}', internal_id, lang, 'object info', {'ratings': 'true'}), ('data', 0, {dict}))
+        if not response:
+            raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True)
+
+        object_type = response.get('type')
+        if object_type == 'episode':
+            result = self._transform_episode_response(response)
+
+        elif object_type == 'movie':
+            result = self._transform_movie_response(response)
+
+        elif object_type == 'movie_listing':
+            first_movie_id = traverse_obj(response, ('movie_listing_metadata', 'first_movie_id'))
+            if not self._yes_playlist(internal_id, first_movie_id):
+                return self.url_result(f'{self._BASE_URL}/{lang}watch/{first_movie_id}', CrunchyrollBetaIE, first_movie_id)
+
+            def entries():
+                movies = self._call_api(f'movie_listings/{internal_id}/movies', internal_id, lang, 'movie list')
+                for movie_response in traverse_obj(movies, ('data', ...)):
+                    yield self.url_result(
+                        f'{self._BASE_URL}/{lang}watch/{movie_response["id"]}',
+                        CrunchyrollBetaIE, **self._transform_movie_response(movie_response))
+
+            return self.playlist_result(entries(), **self._transform_movie_response(response))
+
+        else:
+            raise ExtractorError(f'Unknown object type {object_type}')
+
+        # There might be multiple audio languages for one object (`<object>_metadata.versions`),
+        # so we need to get the id from `streams_link` instead or we dont know which language to choose
+        streams_link = response.get('streams_link')
+        if not streams_link and traverse_obj(response, (f'{object_type}_metadata', 'is_premium_only')):
+            message = f'This {object_type} is for premium members only'
+            if self.is_logged_in:
+                raise ExtractorError(message, expected=True)
+            self.raise_login_required(message)
+
+        # We need go from unsigned to signed api to avoid getting soft banned
+        stream_response = self._call_cms_api_signed(remove_start(
+            streams_link, '/content/v2/cms/'), internal_id, lang, 'stream info')
+        result['formats'] = self._extract_formats(stream_response, internal_id)
+        result['subtitles'] = self._extract_subtitles(stream_response)
+
         # if no intro chapter is available, a 403 without usable data is returned
-        intro_chapter = self._download_json(f'https://static.crunchyroll.com/datalab-intro-v2/{internal_id}.json',
-                                            display_id, fatal=False, errnote=False)
+        intro_chapter = self._download_json(
+            f'https://static.crunchyroll.com/datalab-intro-v2/{internal_id}.json',
+            internal_id, note='Downloading chapter info', fatal=False, errnote=False)
         if isinstance(intro_chapter, dict):
-            chapters = [{
+            result['chapters'] = [{
                 'title': 'Intro',
                 'start_time': float_or_none(intro_chapter.get('startTime')),
-                'end_time': float_or_none(intro_chapter.get('endTime'))
+                'end_time': float_or_none(intro_chapter.get('endTime')),
             }]
 
+        def calculate_count(item):
+            return parse_count(''.join((item['displayed'], item.get('unit') or '')))
+
+        result.update(traverse_obj(response, ('rating', {
+            'like_count': ('up', {calculate_count}),
+            'dislike_count': ('down', {calculate_count}),
+        })))
+
+        return result
+
+    @staticmethod
+    def _transform_episode_response(data):
+        metadata = traverse_obj(data, (('episode_metadata', None), {dict}), get_all=False) or {}
         return {
-            'id': internal_id,
-            'title': '%s Episode %s – %s' % (
-                episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
-            'description': try_get(episode_response, lambda x: x['description'].replace(r'\r\n', '\n')),
-            'duration': float_or_none(episode_response.get('duration_ms'), 1000),
-            'timestamp': parse_iso8601(episode_response.get('upload_date')),
-            'series': episode_response.get('series_title'),
-            'series_id': episode_response.get('series_id'),
-            'season': episode_response.get('season_title'),
-            'season_id': episode_response.get('season_id'),
-            'season_number': episode_response.get('season_number'),
-            'episode': episode_response.get('title'),
-            'episode_number': episode_response.get('sequence_number'),
-            'formats': formats,
-            'thumbnails': [{
-                'url': thumb.get('source'),
-                'width': thumb.get('width'),
-                'height': thumb.get('height'),
-            } for thumb in traverse_obj(episode_response, ('images', 'thumbnail', ..., ...)) or []],
-            'subtitles': {
-                lang: [{
-                    'url': subtitle_data.get('url'),
-                    'ext': subtitle_data.get('format')
-                }] for lang, subtitle_data in get_streams('subtitles')
-            },
-            'chapters': chapters
+            'id': data['id'],
+            'title': ' \u2013 '.join((
+                ('%s%s' % (
+                    format_field(metadata, 'season_title'),
+                    format_field(metadata, 'episode', ' Episode %s'))),
+                format_field(data, 'title'))),
+            **traverse_obj(data, {
+                'episode': ('title', {str}),
+                'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}),
+                'thumbnails': ('images', 'thumbnail', ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                }),
+            }),
+            **traverse_obj(metadata, {
+                'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}),
+                'timestamp': ('upload_date', {parse_iso8601}),
+                'series': ('series_title', {str}),
+                'series_id': ('series_id', {str}),
+                'season': ('season_title', {str}),
+                'season_id': ('season_id', {str}),
+                'season_number': ('season_number', ({int}, {float_or_none})),
+                'episode_number': ('sequence_number', ({int}, {float_or_none})),
+                'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
+                'language': ('audio_locale', {str}),
+            }, get_all=False),
+        }
+
+    @staticmethod
+    def _transform_movie_response(data):
+        metadata = traverse_obj(data, (('movie_metadata', 'movie_listing_metadata', None), {dict}), get_all=False) or {}
+        return {
+            'id': data['id'],
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}),
+                'thumbnails': ('images', 'thumbnail', ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                }),
+            }),
+            **traverse_obj(metadata, {
+                'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}),
+                'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
+            }),
         }
 
 
-class CrunchyrollBetaShowIE(CrunchyrollBaseIE):
+class CrunchyrollBetaShowIE(CrunchyrollCmsBaseIE):
     IE_NAME = 'crunchyroll:playlist'
     _VALID_URL = r'''(?x)
-        https?://(?:beta|www)\.crunchyroll\.com/
+        https?://(?:beta\.|www\.)?crunchyroll\.com/
         (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
-        series/(?P<id>\w+)
-        (?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)'''
+        series/(?P<id>\w+)'''
     _TESTS = [{
         'url': 'https://www.crunchyroll.com/series/GY19NQ2QR/Girl-Friend-BETA',
         'info_dict': {
             'id': 'GY19NQ2QR',
             'title': 'Girl Friend BETA',
+            'description': 'md5:99c1b22ee30a74b536a8277ced8eb750',
+            # XXX: `thumbnail` does not get set from `thumbnails` in playlist
+            #  'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'age_limit': 14,
         },
         'playlist_mincount': 10,
     }, {
@@ -279,41 +459,163 @@ class CrunchyrollBetaShowIE(CrunchyrollBaseIE):
     }]
 
     def _real_extract(self, url):
-        lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
-        api_domain, bucket, params = self._get_params(lang)
-
-        series_response = self._download_json(
-            f'{api_domain}/cms/v2{bucket}/series/{internal_id}', display_id,
-            note='Retrieving series metadata', query=params)
-
-        seasons_response = self._download_json(
-            f'{api_domain}/cms/v2{bucket}/seasons?series_id={internal_id}', display_id,
-            note='Retrieving season list', query=params)
+        lang, internal_id = self._match_valid_url(url).group('lang', 'id')
 
         def entries():
-            for season in seasons_response['items']:
-                episodes_response = self._download_json(
-                    f'{api_domain}/cms/v2{bucket}/episodes?season_id={season["id"]}', display_id,
-                    note=f'Retrieving episode list for {season.get("slug_title")}', query=params)
-                for episode in episodes_response['items']:
-                    episode_id = episode['id']
-                    episode_display_id = episode['slug_title']
-                    yield {
-                        '_type': 'url',
-                        'url': f'https://www.crunchyroll.com/{lang}watch/{episode_id}/{episode_display_id}',
-                        'ie_key': CrunchyrollBetaIE.ie_key(),
-                        'id': episode_id,
-                        'title': '%s Episode %s – %s' % (episode.get('season_title'), episode.get('episode'), episode.get('title')),
-                        'description': try_get(episode, lambda x: x['description'].replace(r'\r\n', '\n')),
-                        'duration': float_or_none(episode.get('duration_ms'), 1000),
-                        'series': episode.get('series_title'),
-                        'series_id': episode.get('series_id'),
-                        'season': episode.get('season_title'),
-                        'season_id': episode.get('season_id'),
-                        'season_number': episode.get('season_number'),
-                        'episode': episode.get('title'),
-                        'episode_number': episode.get('sequence_number'),
-                        'language': episode.get('audio_locale'),
-                    }
+            seasons_response = self._call_cms_api_signed(f'seasons?series_id={internal_id}', internal_id, lang, 'seasons')
+            for season in traverse_obj(seasons_response, ('items', ..., {dict})):
+                episodes_response = self._call_cms_api_signed(
+                    f'episodes?season_id={season["id"]}', season["id"], lang, 'episode list')
+                for episode_response in traverse_obj(episodes_response, ('items', ..., {dict})):
+                    yield self.url_result(
+                        f'{self._BASE_URL}/{lang}watch/{episode_response["id"]}',
+                        CrunchyrollBetaIE, **CrunchyrollBetaIE._transform_episode_response(episode_response))
 
-        return self.playlist_result(entries(), internal_id, series_response.get('title'))
+        return self.playlist_result(
+            entries(), internal_id,
+            **traverse_obj(self._call_api(f'series/{internal_id}', internal_id, lang, 'series'), ('data', 0, {
+                'title': ('title', {str}),
+                'description': ('description', {lambda x: x.replace(r'\r\n', '\n')}),
+                'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
+                'thumbnails': ('images', ..., ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                })
+            })))
+
+
+class CrunchyrollMusicIE(CrunchyrollBaseIE):
+    IE_NAME = 'crunchyroll:music'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?crunchyroll\.com/
+        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        watch/(?P<type>concert|musicvideo)/(?P<id>\w{10})'''
+    _TESTS = [{
+        'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': 'MV88BB7F2C',
+            'display_id': 'crossing-field',
+            'title': 'Crossing Field',
+            'track': 'Crossing Field',
+            'artist': 'LiSA',
+            'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'genre': ['Anime'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': 'MC2E2AC135',
+            'display_id': 'live-is-smile-always-364joker-at-yokohama-arena',
+            'title': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA',
+            'track': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA',
+            'artist': 'LiSA',
+            'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'description': 'md5:747444e7e6300907b7a43f0a0503072e',
+            'genre': ['J-Pop'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C/crossing-field',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135/live-is-smile-always-364joker-at-yokohama-arena',
+        'only_matching': True,
+    }]
+    _API_ENDPOINT = 'music'
+
+    def _real_extract(self, url):
+        lang, internal_id, object_type = self._match_valid_url(url).group('lang', 'id', 'type')
+        path, name = {
+            'concert': ('concerts', 'concert info'),
+            'musicvideo': ('music_videos', 'music video info'),
+        }[object_type]
+        response = traverse_obj(self._call_api(f'{path}/{internal_id}', internal_id, lang, name), ('data', 0, {dict}))
+        if not response:
+            raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True)
+
+        streams_link = response.get('streams_link')
+        if not streams_link and response.get('isPremiumOnly'):
+            message = f'This {response.get("type") or "media"} is for premium members only'
+            if self.is_logged_in:
+                raise ExtractorError(message, expected=True)
+            self.raise_login_required(message)
+
+        result = self._transform_music_response(response)
+        stream_response = self._call_api(streams_link, internal_id, lang, 'stream info')
+        result['formats'] = self._extract_formats(stream_response, internal_id)
+
+        return result
+
+    @staticmethod
+    def _transform_music_response(data):
+        return {
+            'id': data['id'],
+            **traverse_obj(data, {
+                'display_id': 'slug',
+                'title': 'title',
+                'track': 'title',
+                'artist': ('artist', 'name'),
+                'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n') or None}),
+                'thumbnails': ('images', ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                }),
+                'genre': ('genres', ..., 'displayValue'),
+                'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
+            }),
+        }
+
+
+class CrunchyrollArtistIE(CrunchyrollBaseIE):
+    IE_NAME = 'crunchyroll:artist'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?crunchyroll\.com/
+        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        artist/(?P<id>\w{10})'''
+    _TESTS = [{
+        'url': 'https://www.crunchyroll.com/artist/MA179CB50D',
+        'info_dict': {
+            'id': 'MA179CB50D',
+            'title': 'LiSA',
+            'genre': ['J-Pop', 'Anime', 'Rock'],
+            'description': 'md5:16d87de61a55c3f7d6c454b73285938e',
+        },
+        'playlist_mincount': 83,
+    }, {
+        'url': 'https://www.crunchyroll.com/artist/MA179CB50D/lisa',
+        'only_matching': True,
+    }]
+    _API_ENDPOINT = 'music'
+
+    def _real_extract(self, url):
+        lang, internal_id = self._match_valid_url(url).group('lang', 'id')
+        response = traverse_obj(self._call_api(
+            f'artists/{internal_id}', internal_id, lang, 'artist info'), ('data', 0))
+
+        def entries():
+            for attribute, path in [('concerts', 'concert'), ('videos', 'musicvideo')]:
+                for internal_id in traverse_obj(response, (attribute, ...)):
+                    yield self.url_result(f'{self._BASE_URL}/watch/{path}/{internal_id}', CrunchyrollMusicIE, internal_id)
+
+        return self.playlist_result(entries(), **self._transform_artist_response(response))
+
+    @staticmethod
+    def _transform_artist_response(data):
+        return {
+            'id': data['id'],
+            **traverse_obj(data, {
+                'title': 'name',
+                'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}),
+                'thumbnails': ('images', ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                }),
+                'genre': ('genres', ..., 'displayValue'),
+            }),
+        }

From edbe5b589dd0860a67b4e03f58db3cd2539d91c2 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Thu, 25 May 2023 22:52:44 +0200
Subject: [PATCH 277/871] Bugfixes for 4823ec9f461512daa1b8ab362893bb86a6320b26

Hotfix for fragmented downloads

Authored by: bashonly
---
 yt_dlp/downloader/fragment.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 6770815ab..53b4b604e 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -284,7 +284,7 @@ def frag_progress_hook(s):
                 frag_downloaded_bytes = s['downloaded_bytes']
                 state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['speed'] = state['speed'] = self.calc_speed(
-                    ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx['frag_resume_len'])
+                    ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx.get('frag_resume_len', 0))
                 if not ctx['live']:
                     state['eta'] = self.calc_eta(state['speed'], estimated_size - state['downloaded_bytes'])
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
@@ -304,7 +304,7 @@ def _finish_frag_download(self, ctx, info_dict):
 
         to_file = ctx['tmpfilename'] != '-'
         if to_file:
-            downloaded_bytes = self.filesize_or_none(ctx['filename'])
+            downloaded_bytes = self.filesize_or_none(ctx['tmpfilename'])
         else:
             downloaded_bytes = ctx['complete_frags_downloaded_bytes']
 

From 4ad58667c102bd82a7c4cca8aa395ec1682e3b4c Mon Sep 17 00:00:00 2001
From: MMM <flashdagger@googlemail.com>
Date: Thu, 25 May 2023 23:06:58 +0200
Subject: [PATCH 278/871] [extractor/bibeltv] Fix extraction, support live
 streams and series (#6505)

Authored by: flashdagger
---
 yt_dlp/extractor/_extractors.py |   6 +-
 yt_dlp/extractor/bibeltv.py     | 208 +++++++++++++++++++++++++++++---
 2 files changed, 194 insertions(+), 20 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 8984d4b16..6a1406dc5 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -204,7 +204,11 @@
     BFMTVLiveIE,
     BFMTVArticleIE,
 )
-from .bibeltv import BibelTVIE
+from .bibeltv import (
+    BibelTVLiveIE,
+    BibelTVSeriesIE,
+    BibelTVVideoIE,
+)
 from .bigflix import BigflixIE
 from .bigo import BigoIE
 from .bild import BildIE
diff --git a/yt_dlp/extractor/bibeltv.py b/yt_dlp/extractor/bibeltv.py
index fd20aadad..34464daa1 100644
--- a/yt_dlp/extractor/bibeltv.py
+++ b/yt_dlp/extractor/bibeltv.py
@@ -1,27 +1,197 @@
+from functools import partial
+
 from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    determine_ext,
+    format_field,
+    int_or_none,
+    js_to_json,
+    orderedSet,
+    parse_iso8601,
+    traverse_obj,
+    url_or_none,
+)
 
 
-class BibelTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/videos/(?:crn/)?(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://www.bibeltv.de/mediathek/videos/329703-sprachkurs-in-malaiisch',
-        'md5': '252f908192d611de038b8504b08bf97f',
-        'info_dict': {
-            'id': 'ref:329703',
-            'ext': 'mp4',
-            'title': 'Sprachkurs in Malaiisch',
-            'description': 'md5:3e9f197d29ee164714e67351cf737dfe',
-            'timestamp': 1608316701,
-            'uploader_id': '5840105145001',
-            'upload_date': '20201218',
+class BibelTVBaseIE(InfoExtractor):
+    _GEO_COUNTRIES = ['AT', 'CH', 'DE']
+    _GEO_BYPASS = False
+
+    API_URL = 'https://www.bibeltv.de/mediathek/api'
+    AUTH_TOKEN = 'j88bRXY8DsEqJ9xmTdWhrByVi5Hm'
+
+    def _extract_formats_and_subtitles(self, data, crn_id, *, is_live=False):
+        formats = []
+        subtitles = {}
+        for media_url in traverse_obj(data, (..., 'src', {url_or_none})):
+            media_ext = determine_ext(media_url)
+            if media_ext == 'm3u8':
+                m3u8_formats, m3u8_subs = self._extract_m3u8_formats_and_subtitles(
+                    media_url, crn_id, live=is_live)
+                formats.extend(m3u8_formats)
+                subtitles.update(m3u8_subs)
+            elif media_ext == 'mpd':
+                mpd_formats, mpd_subs = self._extract_mpd_formats_and_subtitles(media_url, crn_id)
+                formats.extend(mpd_formats)
+                subtitles.update(mpd_subs)
+            elif media_ext == 'mp4':
+                formats.append({'url': media_url})
+            else:
+                self.report_warning(f'Unknown format {media_ext!r}')
+
+        return formats, subtitles
+
+    @staticmethod
+    def _extract_base_info(data):
+        return {
+            'id': data['crn'],
+            **traverse_obj(data, {
+                'title': 'title',
+                'description': 'description',
+                'duration': ('duration', {partial(int_or_none, scale=1000)}),
+                'timestamp': ('schedulingStart', {parse_iso8601}),
+                'season_number': 'seasonNumber',
+                'episode_number': 'episodeNumber',
+                'view_count': 'viewCount',
+                'like_count': 'likeCount',
+            }),
+            'thumbnails': orderedSet(traverse_obj(data, ('images', ..., {
+                'url': ('url', {url_or_none}),
+            }))),
         }
-    }, {
-        'url': 'https://www.bibeltv.de/mediathek/videos/crn/326374',
-        'only_matching': True,
+
+    def _extract_url_info(self, data):
+        return {
+            '_type': 'url',
+            'url': format_field(data, 'slug', 'https://www.bibeltv.de/mediathek/videos/%s'),
+            **self._extract_base_info(data),
+        }
+
+    def _extract_video_info(self, data):
+        crn_id = data['crn']
+
+        if data.get('drm'):
+            self.report_drm(crn_id)
+
+        json_data = self._download_json(
+            format_field(data, 'id', f'{self.API_URL}/video/%s'), crn_id,
+            headers={'Authorization': self.AUTH_TOKEN}, fatal=False,
+            errnote='No formats available') or {}
+
+        formats, subtitles = self._extract_formats_and_subtitles(
+            traverse_obj(json_data, ('video', 'videoUrls', ...)), crn_id)
+
+        return {
+            '_type': 'video',
+            **self._extract_base_info(data),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+
+class BibelTVVideoIE(BibelTVBaseIE):
+    IE_DESC = 'BibelTV single video'
+    _VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/videos/(?P<id>\d+)[\w-]+'
+    IE_NAME = 'bibeltv:video'
+
+    _TESTS = [{
+        'url': 'https://www.bibeltv.de/mediathek/videos/344436-alte-wege',
+        'md5': 'ec1c07efe54353780512e8a4103b612e',
+        'info_dict': {
+            'id': '344436',
+            'ext': 'mp4',
+            'title': 'Alte Wege',
+            'description': 'md5:2f4eb7294c9797a47b8fd13cccca22e9',
+            'timestamp': 1677877071,
+            'duration': 150.0,
+            'upload_date': '20230303',
+            'thumbnail': r're:https://bibeltv\.imgix\.net/[\w-]+\.jpg',
+            'episode': 'Episode 1',
+            'episode_number': 1,
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {
+            'format': '6',
+        },
     }]
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5840105145001/default_default/index.html?videoId=ref:%s'
 
     def _real_extract(self, url):
         crn_id = self._match_id(url)
-        return self.url_result(
-            self.BRIGHTCOVE_URL_TEMPLATE % crn_id, 'BrightcoveNew')
+        video_data = traverse_obj(
+            self._search_nextjs_data(self._download_webpage(url, crn_id), crn_id),
+            ('props', 'pageProps', 'videoPageData', 'videos', 0, {dict}))
+        if not video_data:
+            raise ExtractorError('Missing video data.')
+
+        return self._extract_video_info(video_data)
+
+
+class BibelTVSeriesIE(BibelTVBaseIE):
+    IE_DESC = 'BibelTV series playlist'
+    _VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/serien/(?P<id>\d+)[\w-]+'
+    IE_NAME = 'bibeltv:series'
+
+    _TESTS = [{
+        'url': 'https://www.bibeltv.de/mediathek/serien/333485-ein-wunder-fuer-jeden-tag',
+        'playlist_mincount': 400,
+        'info_dict': {
+            'id': '333485',
+            'title': 'Ein Wunder für jeden Tag',
+            'description': 'Tägliche Kurzandacht mit Déborah Rosenkranz.',
+        },
+    }]
+
+    def _real_extract(self, url):
+        crn_id = self._match_id(url)
+        webpage = self._download_webpage(url, crn_id)
+        nextjs_data = self._search_nextjs_data(webpage, crn_id)
+        series_data = traverse_obj(nextjs_data, ('props', 'pageProps', 'seriePageData', {dict}))
+        if not series_data:
+            raise ExtractorError('Missing series data.')
+
+        return self.playlist_result(
+            traverse_obj(series_data, ('videos', ..., {dict}, {self._extract_url_info})),
+            crn_id, series_data.get('title'), clean_html(series_data.get('description')))
+
+
+class BibelTVLiveIE(BibelTVBaseIE):
+    IE_DESC = 'BibelTV live program'
+    _VALID_URL = r'https?://(?:www\.)?bibeltv\.de/livestreams/(?P<id>[\w-]+)'
+    IE_NAME = 'bibeltv:live'
+
+    _TESTS = [{
+        'url': 'https://www.bibeltv.de/livestreams/bibeltv/',
+        'info_dict': {
+            'id': 'bibeltv',
+            'ext': 'mp4',
+            'title': 're:Bibel TV',
+            'live_status': 'is_live',
+            'thumbnail': 'https://streampreview.bibeltv.de/bibeltv.webp',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.bibeltv.de/livestreams/impuls/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        stream_id = self._match_id(url)
+        webpage = self._download_webpage(url, stream_id)
+        stream_data = self._search_json(
+            r'\\"video\\":', webpage, 'bibeltvData', stream_id,
+            transform_source=lambda jstring: js_to_json(jstring.replace('\\"', '"')))
+
+        formats, subtitles = self._extract_formats_and_subtitles(
+            traverse_obj(stream_data, ('src', ...)), stream_id, is_live=True)
+
+        return {
+            'id': stream_id,
+            'title': stream_data.get('title'),
+            'thumbnail': stream_data.get('poster'),
+            'is_live': True,
+            'formats': formats,
+            'subtitles': subtitles,
+        }

From 5caf30dbc34f10b0be60676fece635b5c59f0d72 Mon Sep 17 00:00:00 2001
From: Audrey <45548254+tntmod54321@users.noreply.github.com>
Date: Fri, 26 May 2023 08:24:39 -0400
Subject: [PATCH 279/871] [extractor/youtube] Extract `heatmap` data (#7100)

Closes #3888
Authored by: tntmod54321
---
 yt_dlp/extractor/common.py  |  4 ++++
 yt_dlp/extractor/youtube.py | 15 +++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 78288f809..1b1dd560f 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -350,6 +350,10 @@ class InfoExtractor:
                         * "start_time" - The start time of the chapter in seconds
                         * "end_time" - The end time of the chapter in seconds
                         * "title" (optional, string)
+    heatmap:        A list of dictionaries, with the following entries:
+                        * "start_time" - The start time of the data point in seconds
+                        * "end_time" - The end time of the data point in seconds
+                        * "value" - The normalized value of the data point (float between 0 and 1)
     playable_in_embed: Whether this video is allowed to play in embedded
                     players on other sites. Can be True (=always allowed),
                     False (=never allowed), None (=unknown), or a string
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 654bf5e6b..80edcd77d 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -1273,6 +1273,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
                 'uploader_id': '@PhilippHagemeister',
+                'heatmap': 'count:100',
             }
         },
         {
@@ -1426,6 +1427,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'FlyingKitty',
                 'uploader_url': 'https://www.youtube.com/@FlyingKitty900',
                 'uploader_id': '@FlyingKitty900',
+                'comment_count': int,
             },
         },
         {
@@ -3244,6 +3246,17 @@ def _extract_chapters_from_engagement_panel(self, data, duration):
                                           chapter_time, chapter_title, duration)
             for contents in content_list)), [])
 
+    def _extract_heatmap_from_player_overlay(self, data):
+        content_list = traverse_obj(data, (
+            'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer', 'decoratedPlayerBarRenderer', 'playerBar',
+            'multiMarkersPlayerBarRenderer', 'markersMap', ..., 'value', 'heatmap', 'heatmapRenderer', 'heatMarkers', {list}))
+        return next(filter(None, (
+            traverse_obj(contents, (..., 'heatMarkerRenderer', {
+                'start_time': ('timeRangeStartMillis', {functools.partial(float_or_none, scale=1000)}),
+                'end_time': {lambda x: (x['timeRangeStartMillis'] + x['markerDurationMillis']) / 1000},
+                'value': ('heatMarkerIntensityScoreNormalized', {float_or_none}),
+            })) for contents in content_list)), None)
+
     def _extract_comment(self, comment_renderer, parent=None):
         comment_id = comment_renderer.get('commentId')
         if not comment_id:
@@ -4313,6 +4326,8 @@ def process_language(container, base_url, lang_code, sub_name, query):
                 or self._extract_chapters_from_description(video_description, duration)
                 or None)
 
+            info['heatmap'] = self._extract_heatmap_from_player_overlay(initial_data)
+
         contents = traverse_obj(
             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
             expected_type=list, default=[])

From b844a3f8b16500663e7ab6c6ec061cc9b30f71ac Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 26 May 2023 07:57:10 -0500
Subject: [PATCH 280/871] [extractor/weverse] Add extractors (#6711)

Closes #4786
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   8 +
 yt_dlp/extractor/naver.py       |   2 +-
 yt_dlp/extractor/weverse.py     | 604 ++++++++++++++++++++++++++++++++
 3 files changed, 613 insertions(+), 1 deletion(-)
 create mode 100644 yt_dlp/extractor/weverse.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 6a1406dc5..49dd9aecd 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2320,6 +2320,14 @@
     WeiboMobileIE
 )
 from .weiqitv import WeiqiTVIE
+from .weverse import (
+    WeverseIE,
+    WeverseMediaIE,
+    WeverseMomentIE,
+    WeverseLiveTabIE,
+    WeverseMediaTabIE,
+    WeverseLiveIE,
+)
 from .wevidi import WeVidiIE
 from .whyp import WhypIE
 from .wikimedia import WikimediaIE
diff --git a/yt_dlp/extractor/naver.py b/yt_dlp/extractor/naver.py
index 7a1890a61..d79caf5f3 100644
--- a/yt_dlp/extractor/naver.py
+++ b/yt_dlp/extractor/naver.py
@@ -21,7 +21,7 @@
 class NaverBaseIE(InfoExtractor):
     _CAPTION_EXT_RE = r'\.(?:ttml|vtt)'
 
-    @staticmethod  # NB: Used in VLiveWebArchiveIE
+    @staticmethod  # NB: Used in VLiveWebArchiveIE, WeverseIE
     def process_subtitles(vod_data, process_url):
         ret = {'subtitles': {}, 'automatic_captions': {}}
         for caption in traverse_obj(vod_data, ('captions', 'list', ...)):
diff --git a/yt_dlp/extractor/weverse.py b/yt_dlp/extractor/weverse.py
new file mode 100644
index 000000000..ab629c885
--- /dev/null
+++ b/yt_dlp/extractor/weverse.py
@@ -0,0 +1,604 @@
+import base64
+import hashlib
+import hmac
+import itertools
+import json
+import re
+import time
+import urllib.error
+import urllib.parse
+import uuid
+
+from .common import InfoExtractor
+from .naver import NaverBaseIE
+from .youtube import YoutubeIE
+from ..utils import (
+    ExtractorError,
+    UserNotLive,
+    float_or_none,
+    int_or_none,
+    str_or_none,
+    traverse_obj,
+    try_call,
+    update_url_query,
+    url_or_none,
+)
+
+
+class WeverseBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'weverse'
+    _ACCOUNT_API_BASE = 'https://accountapi.weverse.io/web/api/v2'
+    _API_HEADERS = {
+        'Referer': 'https://weverse.io/',
+        'WEV-device-Id': str(uuid.uuid4()),
+    }
+
+    def _perform_login(self, username, password):
+        headers = {
+            'x-acc-app-secret': '5419526f1c624b38b10787e5c10b2a7a',
+            'x-acc-app-version': '2.2.6',
+            'x-acc-language': 'en',
+            'x-acc-service-id': 'weverse',
+            'x-acc-trace-id': str(uuid.uuid4()),
+            'x-clog-user-device-id': str(uuid.uuid4()),
+        }
+        check_username = self._download_json(
+            f'{self._ACCOUNT_API_BASE}/signup/email/status', None,
+            note='Checking username', query={'email': username}, headers=headers)
+        if not check_username.get('hasPassword'):
+            raise ExtractorError('Invalid username provided', expected=True)
+
+        headers['content-type'] = 'application/json'
+        try:
+            auth = self._download_json(
+                f'{self._ACCOUNT_API_BASE}/auth/token/by-credentials', None, data=json.dumps({
+                    'email': username,
+                    'password': password,
+                }, separators=(',', ':')).encode(), headers=headers, note='Logging in')
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                raise ExtractorError('Invalid password provided', expected=True)
+            raise
+
+        WeverseBaseIE._API_HEADERS['Authorization'] = f'Bearer {auth["accessToken"]}'
+
+    def _real_initialize(self):
+        if self._API_HEADERS.get('Authorization'):
+            return
+
+        token = try_call(lambda: self._get_cookies('https://weverse.io/')['we2_access_token'].value)
+        if not token:
+            self.raise_login_required()
+
+        WeverseBaseIE._API_HEADERS['Authorization'] = f'Bearer {token}'
+
+    def _call_api(self, ep, video_id, data=None, note='Downloading API JSON'):
+        # Ref: https://ssl.pstatic.net/static/wevweb/2_3_2_11101725/public/static/js/2488.a09b41ff.chunk.js
+        # From https://ssl.pstatic.net/static/wevweb/2_3_2_11101725/public/static/js/main.e206f7c1.js:
+        key = b'1b9cb6378d959b45714bec49971ade22e6e24e42'
+        api_path = update_url_query(ep, {
+            'appId': 'be4d79eb8fc7bd008ee82c8ec4ff6fd4',
+            'language': 'en',
+            'platform': 'WEB',
+            'wpf': 'pc',
+        })
+        wmsgpad = int(time.time() * 1000)
+        wmd = base64.b64encode(hmac.HMAC(
+            key, f'{api_path[:255]}{wmsgpad}'.encode(), digestmod=hashlib.sha1).digest()).decode()
+        headers = {'Content-Type': 'application/json'} if data else {}
+        try:
+            return self._download_json(
+                f'https://global.apis.naver.com/weverse/wevweb{api_path}', video_id, note=note,
+                data=data, headers={**self._API_HEADERS, **headers}, query={
+                    'wmsgpad': wmsgpad,
+                    'wmd': wmd,
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                self.raise_login_required(
+                    'Session token has expired. Log in again or refresh cookies in browser')
+            elif isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                raise ExtractorError('Your account does not have access to this content', expected=True)
+            raise
+
+    def _call_post_api(self, video_id):
+        return self._call_api(f'/post/v1.0/post-{video_id}?fieldSet=postV1', video_id)
+
+    def _get_community_id(self, channel):
+        return str(self._call_api(
+            f'/community/v1.0/communityIdUrlPathByUrlPathArtistCode?keyword={channel}',
+            channel, note='Fetching community ID')['communityId'])
+
+    def _get_formats(self, data, video_id):
+        formats = traverse_obj(data, ('videos', 'list', lambda _, v: url_or_none(v['source']), {
+            'url': 'source',
+            'width': ('encodingOption', 'width', {int_or_none}),
+            'height': ('encodingOption', 'height', {int_or_none}),
+            'vcodec': 'type',
+            'vbr': ('bitrate', 'video', {int_or_none}),
+            'abr': ('bitrate', 'audio', {int_or_none}),
+            'filesize': ('size', {int_or_none}),
+            'format_id': ('encodingOption', 'id', {str_or_none}),
+        }))
+
+        for stream in traverse_obj(data, ('streams', lambda _, v: v['type'] == 'HLS' and url_or_none(v['source']))):
+            query = {}
+            for param in traverse_obj(stream, ('keys', lambda _, v: v['type'] == 'param' and v['name'])):
+                query[param['name']] = param.get('value', '')
+            fmts = self._extract_m3u8_formats(
+                stream['source'], video_id, 'mp4', m3u8_id='hls', fatal=False, query=query)
+            if query:
+                for fmt in fmts:
+                    fmt['url'] = update_url_query(fmt['url'], query)
+                    fmt['extra_param_to_segment_url'] = urllib.parse.urlencode(query)
+            formats.extend(fmts)
+
+        return formats
+
+    def _get_subs(self, caption_url):
+        subs_ext_re = r'\.(?:ttml|vtt)'
+        replace_ext = lambda x, y: re.sub(subs_ext_re, y, x)
+        if re.search(subs_ext_re, caption_url):
+            return [replace_ext(caption_url, '.ttml'), replace_ext(caption_url, '.vtt')]
+        return [caption_url]
+
+    def _parse_post_meta(self, metadata):
+        return traverse_obj(metadata, {
+            'title': ((('extension', 'mediaInfo', 'title'), 'title'), {str}),
+            'description': ((('extension', 'mediaInfo', 'body'), 'body'), {str}),
+            'uploader': ('author', 'profileName', {str}),
+            'uploader_id': ('author', 'memberId', {str}),
+            'creator': ('community', 'communityName', {str}),
+            'channel_id': (('community', 'author'), 'communityId', {str_or_none}),
+            'duration': ('extension', 'video', 'playTime', {float_or_none}),
+            'timestamp': ('publishedAt', {lambda x: int_or_none(x, 1000)}),
+            'release_timestamp': ('extension', 'video', 'onAirStartAt', {lambda x: int_or_none(x, 1000)}),
+            'thumbnail': ('extension', (('mediaInfo', 'thumbnail', 'url'), ('video', 'thumb')), {url_or_none}),
+            'view_count': ('extension', 'video', 'playCount', {int_or_none}),
+            'like_count': ('extension', 'video', 'likeCount', {int_or_none}),
+            'comment_count': ('commentCount', {int_or_none}),
+        }, get_all=False)
+
+    def _extract_availability(self, data):
+        return self._availability(**traverse_obj(data, ((('extension', 'video'), None), {
+            'needs_premium': 'paid',
+            'needs_subscription': 'membershipOnly',
+        }), get_all=False, expected_type=bool), needs_auth=True)
+
+    def _extract_live_status(self, data):
+        data = traverse_obj(data, ('extension', 'video', {dict})) or {}
+        if data.get('type') == 'LIVE':
+            return traverse_obj({
+                'ONAIR': 'is_live',
+                'DONE': 'post_live',
+                'STANDBY': 'is_upcoming',
+                'DELAY': 'is_upcoming',
+            }, (data.get('status'), {str})) or 'is_live'
+        return 'was_live' if data.get('liveToVod') else 'not_live'
+
+
+class WeverseIE(WeverseBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/live/(?P<id>[\d-]+)'
+    _TESTS = [{
+        'url': 'https://weverse.io/billlie/live/0-107323480',
+        'md5': '1fa849f00181eef9100d3c8254c47979',
+        'info_dict': {
+            'id': '0-107323480',
+            'ext': 'mp4',
+            'title': '행복한 평이루💜',
+            'description': '',
+            'uploader': 'Billlie',
+            'uploader_id': '5ae14aed7b7cdc65fa87c41fe06cc936',
+            'channel': 'billlie',
+            'channel_id': '72',
+            'channel_url': 'https://weverse.io/billlie',
+            'creator': 'Billlie',
+            'timestamp': 1666262062,
+            'upload_date': '20221020',
+            'release_timestamp': 1666262058,
+            'release_date': '20221020',
+            'duration': 3102,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'was_live',
+        },
+    }, {
+        'url': 'https://weverse.io/lesserafim/live/2-102331763',
+        'md5': 'e46125c08b13a6c8c1f4565035cca987',
+        'info_dict': {
+            'id': '2-102331763',
+            'ext': 'mp4',
+            'title': '🎂김채원 생신🎂',
+            'description': '🎂김채원 생신🎂',
+            'uploader': 'LE SSERAFIM ',
+            'uploader_id': 'd26ddc1e258488a0a2b795218d14d59d',
+            'channel': 'lesserafim',
+            'channel_id': '47',
+            'channel_url': 'https://weverse.io/lesserafim',
+            'creator': 'LE SSERAFIM',
+            'timestamp': 1659353400,
+            'upload_date': '20220801',
+            'release_timestamp': 1659353400,
+            'release_date': '20220801',
+            'duration': 3006,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'was_live',
+            'subtitles': {
+                'id_ID': 'count:2',
+                'en_US': 'count:2',
+                'es_ES': 'count:2',
+                'vi_VN': 'count:2',
+                'th_TH': 'count:2',
+                'zh_CN': 'count:2',
+                'zh_TW': 'count:2',
+                'ja_JP': 'count:2',
+                'ko_KR': 'count:2',
+            },
+        },
+    }, {
+        'url': 'https://weverse.io/treasure/live/2-117230416',
+        'info_dict': {
+            'id': '2-117230416',
+            'ext': 'mp4',
+            'title': r're:스껄도려님 첫 스무살 생파🦋',
+            'description': '',
+            'uploader': 'TREASURE',
+            'uploader_id': '77eabbc449ca37f7970054a136f60082',
+            'channel': 'treasure',
+            'channel_id': '20',
+            'channel_url': 'https://weverse.io/treasure',
+            'creator': 'TREASURE',
+            'timestamp': 1680667651,
+            'upload_date': '20230405',
+            'release_timestamp': 1680667639,
+            'release_date': '20230405',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'is_live',
+        },
+        'skip': 'Livestream has ended',
+    }]
+
+    def _real_extract(self, url):
+        channel, video_id = self._match_valid_url(url).group('artist', 'id')
+        post = self._call_post_api(video_id)
+        api_video_id = post['extension']['video']['videoId']
+        availability = self._extract_availability(post)
+        live_status = self._extract_live_status(post)
+        video_info, formats = {}, []
+
+        if live_status == 'is_upcoming':
+            self.raise_no_formats('Livestream has not yet started', expected=True)
+
+        elif live_status == 'is_live':
+            video_info = self._call_api(
+                f'/video/v1.0/lives/{api_video_id}/playInfo?preview.format=json&preview.version=v2',
+                video_id, note='Downloading live JSON')
+            playback = self._parse_json(video_info['lipPlayback'], video_id)
+            m3u8_url = traverse_obj(playback, (
+                'media', lambda _, v: v['protocol'] == 'HLS', 'path', {url_or_none}), get_all=False)
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls', live=True)
+
+        elif live_status == 'post_live':
+            if availability in ('premium_only', 'subscriber_only'):
+                self.report_drm(video_id)
+            self.raise_no_formats(
+                'Livestream has ended and downloadable VOD is not available', expected=True)
+
+        else:
+            infra_video_id = post['extension']['video']['infraVideoId']
+            in_key = self._call_api(
+                f'/video/v1.0/vod/{api_video_id}/inKey?preview=false', video_id,
+                data=b'{}', note='Downloading VOD API key')['inKey']
+
+            video_info = self._download_json(
+                f'https://global.apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{infra_video_id}',
+                video_id, note='Downloading VOD JSON', query={
+                    'key': in_key,
+                    'sid': traverse_obj(post, ('extension', 'video', 'serviceId')) or '2070',
+                    'pid': str(uuid.uuid4()),
+                    'nonce': int(time.time() * 1000),
+                    'devt': 'html5_pc',
+                    'prv': 'Y' if post.get('membershipOnly') else 'N',
+                    'aup': 'N',
+                    'stpb': 'N',
+                    'cpl': 'en',
+                    'env': 'prod',
+                    'lc': 'en',
+                    'adi': '[{"adSystem":"null"}]',
+                    'adu': '/',
+                })
+
+            formats = self._get_formats(video_info, video_id)
+            has_drm = traverse_obj(video_info, ('meta', 'provider', 'name', {str.lower})) == 'drm'
+            if has_drm and formats:
+                self.report_warning(
+                    'Requested content is DRM-protected, only a 30-second preview is available', video_id)
+            elif has_drm and not formats:
+                self.report_drm(video_id)
+
+        return {
+            'id': video_id,
+            'channel': channel,
+            'channel_url': f'https://weverse.io/{channel}',
+            'formats': formats,
+            'availability': availability,
+            'live_status': live_status,
+            **self._parse_post_meta(post),
+            **NaverBaseIE.process_subtitles(video_info, self._get_subs),
+        }
+
+
+class WeverseMediaIE(WeverseBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/media/(?P<id>[\d-]+)'
+    _TESTS = [{
+        'url': 'https://weverse.io/billlie/media/4-116372884',
+        'md5': '8efc9cfd61b2f25209eb1a5326314d28',
+        'info_dict': {
+            'id': 'e-C9wLSQs6o',
+            'ext': 'mp4',
+            'title': 'Billlie | \'EUNOIA\' Performance Video (heartbeat ver.)',
+            'description': 'md5:6181caaf2a2397bca913ffe368c104e5',
+            'channel': 'Billlie',
+            'channel_id': 'UCyc9sUCxELTDK9vELO5Fzeg',
+            'channel_url': 'https://www.youtube.com/channel/UCyc9sUCxELTDK9vELO5Fzeg',
+            'uploader': 'Billlie',
+            'uploader_id': '@Billlie',
+            'uploader_url': 'http://www.youtube.com/@Billlie',
+            'upload_date': '20230403',
+            'duration': 211,
+            'age_limit': 0,
+            'playable_in_embed': True,
+            'live_status': 'not_live',
+            'availability': 'public',
+            'view_count': int,
+            'comment_count': int,
+            'like_count': int,
+            'channel_follower_count': int,
+            'thumbnail': 'https://i.ytimg.com/vi/e-C9wLSQs6o/maxresdefault.jpg',
+            'categories': ['Entertainment'],
+            'tags': 'count:7',
+        },
+    }, {
+        'url': 'https://weverse.io/billlie/media/3-102914520',
+        'md5': '031551fcbd716bc4f080cb6174a43d8a',
+        'info_dict': {
+            'id': '3-102914520',
+            'ext': 'mp4',
+            'title': 'From. SUHYEON🌸',
+            'description': 'Billlie 멤버별 독점 영상 공개💙💜',
+            'uploader': 'Billlie_official',
+            'uploader_id': 'f569c6e92f7eaffef0a395037dcaa54f',
+            'channel': 'billlie',
+            'channel_id': '72',
+            'channel_url': 'https://weverse.io/billlie',
+            'creator': 'Billlie',
+            'timestamp': 1662174000,
+            'upload_date': '20220903',
+            'release_timestamp': 1662174000,
+            'release_date': '20220903',
+            'duration': 17.0,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'not_live',
+        },
+    }]
+
+    def _real_extract(self, url):
+        channel, video_id = self._match_valid_url(url).group('artist', 'id')
+        post = self._call_post_api(video_id)
+        media_type = traverse_obj(post, ('extension', 'mediaInfo', 'mediaType', {str.lower}))
+        youtube_id = traverse_obj(post, ('extension', 'youtube', 'youtubeVideoId', {str}))
+
+        if media_type == 'vod':
+            return self.url_result(f'https://weverse.io/{channel}/live/{video_id}', WeverseIE)
+        elif media_type == 'youtube' and youtube_id:
+            return self.url_result(youtube_id, YoutubeIE)
+        elif media_type == 'image':
+            self.raise_no_formats('No video content found in webpage', expected=True)
+        elif media_type:
+            raise ExtractorError(f'Unsupported media type "{media_type}"')
+
+        self.raise_no_formats('No video content found in webpage')
+
+
+class WeverseMomentIE(WeverseBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/moment/(?P<uid>[\da-f]+)/post/(?P<id>[\d-]+)'
+    _TESTS = [{
+        'url': 'https://weverse.io/secretnumber/moment/66a07e164b56a696ee71c99315ffe27b/post/1-117229444',
+        'md5': '87733ac19a54081b7dfc2442036d282b',
+        'info_dict': {
+            'id': '1-117229444',
+            'ext': 'mp4',
+            'title': '今日もめっちゃいい天気☀️🌤️',
+            'uploader': '레아',
+            'uploader_id': '66a07e164b56a696ee71c99315ffe27b',
+            'channel': 'secretnumber',
+            'channel_id': '56',
+            'creator': 'SECRET NUMBER',
+            'duration': 10,
+            'upload_date': '20230405',
+            'timestamp': 1680653968,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+        },
+        'skip': 'Moment has expired',
+    }]
+
+    def _real_extract(self, url):
+        channel, uploader_id, video_id = self._match_valid_url(url).group('artist', 'uid', 'id')
+        post = self._call_post_api(video_id)
+        api_video_id = post['extension']['moment']['video']['videoId']
+        video_info = self._call_api(
+            f'/cvideo/v1.0/cvideo-{api_video_id}/playInfo?videoId={api_video_id}', video_id,
+            note='Downloading moment JSON')['playInfo']
+
+        return {
+            'id': video_id,
+            'channel': channel,
+            'uploader_id': uploader_id,
+            'formats': self._get_formats(video_info, video_id),
+            'availability': self._extract_availability(post),
+            **traverse_obj(post, {
+                'title': ((('extension', 'moment', 'body'), 'body'), {str}),
+                'uploader': ('author', 'profileName', {str}),
+                'creator': (('community', 'author'), 'communityName', {str}),
+                'channel_id': (('community', 'author'), 'communityId', {str_or_none}),
+                'duration': ('extension', 'moment', 'video', 'uploadInfo', 'playTime', {float_or_none}),
+                'timestamp': ('publishedAt', {lambda x: int_or_none(x, 1000)}),
+                'thumbnail': ('extension', 'moment', 'video', 'uploadInfo', 'imageUrl', {url_or_none}),
+                'like_count': ('emotionCount', {int_or_none}),
+                'comment_count': ('commentCount', {int_or_none}),
+            }, get_all=False),
+            **NaverBaseIE.process_subtitles(video_info, self._get_subs),
+        }
+
+
+class WeverseTabBaseIE(WeverseBaseIE):
+    _ENDPOINT = None
+    _PATH = None
+    _QUERY = {}
+    _RESULT_IE = None
+
+    def _entries(self, channel_id, channel, first_page):
+        query = self._QUERY.copy()
+
+        for page in itertools.count(1):
+            posts = first_page if page == 1 else self._call_api(
+                update_url_query(self._ENDPOINT % channel_id, query), channel,
+                note=f'Downloading {self._PATH} tab page {page}')
+
+            for post in traverse_obj(posts, ('data', lambda _, v: v['postId'])):
+                yield self.url_result(
+                    f'https://weverse.io/{channel}/{self._PATH}/{post["postId"]}',
+                    self._RESULT_IE, post['postId'], **self._parse_post_meta(post),
+                    channel=channel, channel_url=f'https://weverse.io/{channel}',
+                    availability=self._extract_availability(post),
+                    live_status=self._extract_live_status(post))
+
+            query['after'] = traverse_obj(posts, ('paging', 'nextParams', 'after', {str}))
+            if not query['after']:
+                break
+
+    def _real_extract(self, url):
+        channel = self._match_id(url)
+        channel_id = self._get_community_id(channel)
+
+        first_page = self._call_api(
+            update_url_query(self._ENDPOINT % channel_id, self._QUERY), channel,
+            note=f'Downloading {self._PATH} tab page 1')
+
+        return self.playlist_result(
+            self._entries(channel_id, channel, first_page), f'{channel}-{self._PATH}',
+            **traverse_obj(first_page, ('data', ..., {
+                'playlist_title': ('community', 'communityName', {str}),
+                'thumbnail': ('author', 'profileImageUrl', {url_or_none}),
+            }), get_all=False))
+
+
+class WeverseLiveTabIE(WeverseTabBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/live/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://weverse.io/billlie/live/',
+        'playlist_mincount': 55,
+        'info_dict': {
+            'id': 'billlie-live',
+            'title': 'Billlie',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+        },
+    }]
+
+    _ENDPOINT = '/post/v1.0/community-%s/liveTabPosts'
+    _PATH = 'live'
+    _QUERY = {'fieldSet': 'postsV1'}
+    _RESULT_IE = WeverseIE
+
+
+class WeverseMediaTabIE(WeverseTabBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/media(?:/|/all|/new)?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://weverse.io/billlie/media/',
+        'playlist_mincount': 231,
+        'info_dict': {
+            'id': 'billlie-media',
+            'title': 'Billlie',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+        },
+    }, {
+        'url': 'https://weverse.io/lesserafim/media/all',
+        'only_matching': True,
+    }, {
+        'url': 'https://weverse.io/lesserafim/media/new',
+        'only_matching': True,
+    }]
+
+    _ENDPOINT = '/media/v1.0/community-%s/more'
+    _PATH = 'media'
+    _QUERY = {'fieldSet': 'postsV1', 'filterType': 'RECENT'}
+    _RESULT_IE = WeverseMediaIE
+
+
+class WeverseLiveIE(WeverseBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://weverse.io/purplekiss',
+        'info_dict': {
+            'id': '3-116560493',
+            'ext': 'mp4',
+            'title': r're:모하냥🫶🏻',
+            'description': '내일은 금요일~><',
+            'uploader': '채인',
+            'uploader_id': '1ffb1d9d904d6b3db2783f876eb9229d',
+            'channel': 'purplekiss',
+            'channel_id': '35',
+            'channel_url': 'https://weverse.io/purplekiss',
+            'creator': 'PURPLE KISS',
+            'timestamp': 1680780892,
+            'upload_date': '20230406',
+            'release_timestamp': 1680780883,
+            'release_date': '20230406',
+            'thumbnail': 'https://weverse-live.pstatic.net/v1.0/live/62044/thumb',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'is_live',
+        },
+        'skip': 'Livestream has ended',
+    }, {
+        'url': 'https://weverse.io/billlie/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        channel = self._match_id(url)
+        channel_id = self._get_community_id(channel)
+
+        video_id = traverse_obj(
+            self._call_api(update_url_query(f'/post/v1.0/community-{channel_id}/liveTab', {
+                'debugMessage': 'true',
+                'fields': 'onAirLivePosts.fieldSet(postsV1).limit(10),reservedLivePosts.fieldSet(postsV1).limit(10)',
+            }), channel, note='Downloading live JSON'), (
+                ('onAirLivePosts', 'reservedLivePosts'), 'data',
+                lambda _, v: self._extract_live_status(v) in ('is_live', 'is_upcoming'), 'postId', {str}),
+            get_all=False)
+
+        if not video_id:
+            raise UserNotLive(video_id=channel)
+
+        return self.url_result(f'https://weverse.io/{channel}/live/{video_id}', WeverseIE)

From 66468bbf49562ff82670cbbd456c5e8448a6df34 Mon Sep 17 00:00:00 2001
From: sqrtNOT <77981959+sqrtNOT@users.noreply.github.com>
Date: Fri, 26 May 2023 13:03:19 +0000
Subject: [PATCH 281/871] [extractor/comedycentral] Add support for movies
 (#7108)

Closes #1926
Authored by: sqrtNOT
---
 yt_dlp/extractor/comedycentral.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/comedycentral.py b/yt_dlp/extractor/comedycentral.py
index 05fc9f2b5..27d295bb3 100644
--- a/yt_dlp/extractor/comedycentral.py
+++ b/yt_dlp/extractor/comedycentral.py
@@ -2,7 +2,7 @@
 
 
 class ComedyCentralIE(MTVServicesInfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cc\.com/(?:episodes|video(?:-clips)?|collection-playlist)/(?P<id>[0-9a-z]{6})'
+    _VALID_URL = r'https?://(?:www\.)?cc\.com/(?:episodes|video(?:-clips)?|collection-playlist|movies)/(?P<id>[0-9a-z]{6})'
     _FEED_URL = 'http://comedycentral.com/feeds/mrss/'
 
     _TESTS = [{
@@ -25,6 +25,9 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
     }, {
         'url': 'https://www.cc.com/collection-playlist/cosnej/stand-up-specials/t6vtjb',
         'only_matching': True,
+    }, {
+        'url': 'https://www.cc.com/movies/tkp406/a-cluesterfuenke-christmas',
+        'only_matching': True,
     }]
 
 

From 08916a49c777cb6e000eec092881eb93ec22076c Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 27 May 2023 19:06:13 +1200
Subject: [PATCH 282/871] [core] Improve HTTP redirect handling (#7094)

Aligns HTTP redirect handling with what browsers commonly do and RFC standards.

Fixes issues https://github.com/yt-dlp/yt-dlp/commit/afac4caa7db30804bebac33e53c3cb0237958224 missed.

Authored by: coletdjnz
---
 test/test_YoutubeDL.py |   6 -
 test/test_http.py      | 288 +++++++++++++++++++++++++++++++++++++----
 yt_dlp/utils/_utils.py |  59 +++------
 3 files changed, 281 insertions(+), 72 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 477fd220e..ee6c52713 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -10,7 +10,6 @@
 
 import copy
 import json
-import urllib.error
 
 from test.helper import FakeYDL, assertRegexpMatches
 from yt_dlp import YoutubeDL
@@ -1097,11 +1096,6 @@ def test_selection(params, expected_ids, evaluate_all=False):
         test_selection({'playlist_items': '-15::2'}, INDICES[1::2], True)
         test_selection({'playlist_items': '-15::15'}, [], True)
 
-    def test_urlopen_no_file_protocol(self):
-        # see https://github.com/ytdl-org/youtube-dl/issues/8227
-        ydl = YDL()
-        self.assertRaises(urllib.error.URLError, ydl.urlopen, 'file:///etc/passwd')
-
     def test_do_not_override_ie_key_in_url_transparent(self):
         ydl = YDL()
 
diff --git a/test/test_http.py b/test/test_http.py
index 5ca0d7a47..d684905da 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -7,40 +7,163 @@
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
+import gzip
+import http.cookiejar
 import http.server
+import io
+import pathlib
 import ssl
+import tempfile
 import threading
+import urllib.error
 import urllib.request
 
 from test.helper import http_server_port
 from yt_dlp import YoutubeDL
+from yt_dlp.utils import sanitized_Request, urlencode_postdata
+
+from .helper import FakeYDL
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
 class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
+    protocol_version = 'HTTP/1.1'
+
     def log_message(self, format, *args):
         pass
 
+    def _headers(self):
+        payload = str(self.headers).encode('utf-8')
+        self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.send_header('Content-Length', str(len(payload)))
+        self.end_headers()
+        self.wfile.write(payload)
+
+    def _redirect(self):
+        self.send_response(int(self.path[len('/redirect_'):]))
+        self.send_header('Location', '/method')
+        self.send_header('Content-Length', '0')
+        self.end_headers()
+
+    def _method(self, method, payload=None):
+        self.send_response(200)
+        self.send_header('Content-Length', str(len(payload or '')))
+        self.send_header('Method', method)
+        self.end_headers()
+        if payload:
+            self.wfile.write(payload)
+
+    def _status(self, status):
+        payload = f'<html>{status} NOT FOUND</html>'.encode()
+        self.send_response(int(status))
+        self.send_header('Content-Type', 'text/html; charset=utf-8')
+        self.send_header('Content-Length', str(len(payload)))
+        self.end_headers()
+        self.wfile.write(payload)
+
+    def _read_data(self):
+        if 'Content-Length' in self.headers:
+            return self.rfile.read(int(self.headers['Content-Length']))
+
+    def do_POST(self):
+        data = self._read_data()
+        if self.path.startswith('/redirect_'):
+            self._redirect()
+        elif self.path.startswith('/method'):
+            self._method('POST', data)
+        elif self.path.startswith('/headers'):
+            self._headers()
+        else:
+            self._status(404)
+
+    def do_HEAD(self):
+        if self.path.startswith('/redirect_'):
+            self._redirect()
+        elif self.path.startswith('/method'):
+            self._method('HEAD')
+        else:
+            self._status(404)
+
+    def do_PUT(self):
+        data = self._read_data()
+        if self.path.startswith('/redirect_'):
+            self._redirect()
+        elif self.path.startswith('/method'):
+            self._method('PUT', data)
+        else:
+            self._status(404)
+
     def do_GET(self):
         if self.path == '/video.html':
+            payload = b'<html><video src="/vid.mp4" /></html>'
             self.send_response(200)
             self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.send_header('Content-Length', str(len(payload)))  # required for persistent connections
             self.end_headers()
-            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
+            self.wfile.write(payload)
         elif self.path == '/vid.mp4':
+            payload = b'\x00\x00\x00\x00\x20\x66\x74[video]'
             self.send_response(200)
             self.send_header('Content-Type', 'video/mp4')
+            self.send_header('Content-Length', str(len(payload)))
             self.end_headers()
-            self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]')
+            self.wfile.write(payload)
         elif self.path == '/%E4%B8%AD%E6%96%87.html':
+            payload = b'<html><video src="/vid.mp4" /></html>'
             self.send_response(200)
             self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.send_header('Content-Length', str(len(payload)))
+            self.end_headers()
+            self.wfile.write(payload)
+        elif self.path == '/%c7%9f':
+            payload = b'<html><video src="/vid.mp4" /></html>'
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.send_header('Content-Length', str(len(payload)))
+            self.end_headers()
+            self.wfile.write(payload)
+        elif self.path.startswith('/redirect_'):
+            self._redirect()
+        elif self.path.startswith('/method'):
+            self._method('GET')
+        elif self.path.startswith('/headers'):
+            self._headers()
+        elif self.path == '/trailing_garbage':
+            payload = b'<html><video src="/vid.mp4" /></html>'
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.send_header('Content-Encoding', 'gzip')
+            buf = io.BytesIO()
+            with gzip.GzipFile(fileobj=buf, mode='wb') as f:
+                f.write(payload)
+            compressed = buf.getvalue() + b'trailing garbage'
+            self.send_header('Content-Length', str(len(compressed)))
+            self.end_headers()
+            self.wfile.write(compressed)
+        elif self.path == '/302-non-ascii-redirect':
+            new_url = f'http://127.0.0.1:{http_server_port(self.server)}/中文.html'
+            self.send_response(301)
+            self.send_header('Location', new_url)
+            self.send_header('Content-Length', '0')
             self.end_headers()
-            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
         else:
-            assert False
+            self._status(404)
+
+    def send_header(self, keyword, value):
+        """
+        Forcibly allow HTTP server to send non percent-encoded non-ASCII characters in headers.
+        This is against what is defined in RFC 3986, however we need to test we support this
+        since some sites incorrectly do this.
+        """
+        if keyword.lower() == 'connection':
+            return super().send_header(keyword, value)
+
+        if not hasattr(self, '_headers_buffer'):
+            self._headers_buffer = []
+
+        self._headers_buffer.append(f'{keyword}: {value}\r\n'.encode())
 
 
 class FakeLogger:
@@ -56,36 +179,128 @@ def error(self, msg):
 
 class TestHTTP(unittest.TestCase):
     def setUp(self):
-        self.httpd = http.server.HTTPServer(
+        # HTTP server
+        self.http_httpd = http.server.ThreadingHTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
-        self.port = http_server_port(self.httpd)
-        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
-        self.server_thread.daemon = True
-        self.server_thread.start()
+        self.http_port = http_server_port(self.http_httpd)
+        self.http_server_thread = threading.Thread(target=self.http_httpd.serve_forever)
+        # FIXME: we should probably stop the http server thread after each test
+        # See: https://github.com/yt-dlp/yt-dlp/pull/7094#discussion_r1199746041
+        self.http_server_thread.daemon = True
+        self.http_server_thread.start()
 
-
-class TestHTTPS(unittest.TestCase):
-    def setUp(self):
+        # HTTPS server
         certfn = os.path.join(TEST_DIR, 'testcert.pem')
-        self.httpd = http.server.HTTPServer(
+        self.https_httpd = http.server.ThreadingHTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
         sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         sslctx.load_cert_chain(certfn, None)
-        self.httpd.socket = sslctx.wrap_socket(self.httpd.socket, server_side=True)
-        self.port = http_server_port(self.httpd)
-        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
-        self.server_thread.daemon = True
-        self.server_thread.start()
+        self.https_httpd.socket = sslctx.wrap_socket(self.https_httpd.socket, server_side=True)
+        self.https_port = http_server_port(self.https_httpd)
+        self.https_server_thread = threading.Thread(target=self.https_httpd.serve_forever)
+        self.https_server_thread.daemon = True
+        self.https_server_thread.start()
 
     def test_nocheckcertificate(self):
-        ydl = YoutubeDL({'logger': FakeLogger()})
-        self.assertRaises(
-            Exception,
-            ydl.extract_info, 'https://127.0.0.1:%d/video.html' % self.port)
+        with FakeYDL({'logger': FakeLogger()}) as ydl:
+            with self.assertRaises(urllib.error.URLError):
+                ydl.urlopen(sanitized_Request(f'https://127.0.0.1:{self.https_port}/headers'))
 
-        ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
-        r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
-        self.assertEqual(r['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
+        with FakeYDL({'logger': FakeLogger(), 'nocheckcertificate': True}) as ydl:
+            r = ydl.urlopen(sanitized_Request(f'https://127.0.0.1:{self.https_port}/headers'))
+            self.assertEqual(r.status, 200)
+            r.close()
+
+    def test_percent_encode(self):
+        with FakeYDL() as ydl:
+            # Unicode characters should be encoded with uppercase percent-encoding
+            res = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/中文.html'))
+            self.assertEqual(res.status, 200)
+            res.close()
+            # don't normalize existing percent encodings
+            res = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/%c7%9f'))
+            self.assertEqual(res.status, 200)
+            res.close()
+
+    def test_unicode_path_redirection(self):
+        with FakeYDL() as ydl:
+            r = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/302-non-ascii-redirect'))
+            self.assertEqual(r.url, f'http://127.0.0.1:{self.http_port}/%E4%B8%AD%E6%96%87.html')
+            r.close()
+
+    def test_redirect(self):
+        with FakeYDL() as ydl:
+            def do_req(redirect_status, method):
+                data = b'testdata' if method in ('POST', 'PUT') else None
+                res = ydl.urlopen(sanitized_Request(
+                    f'http://127.0.0.1:{self.http_port}/redirect_{redirect_status}', method=method, data=data))
+                return res.read().decode('utf-8'), res.headers.get('method', '')
+
+            # A 303 must either use GET or HEAD for subsequent request
+            self.assertEqual(do_req(303, 'POST'), ('', 'GET'))
+            self.assertEqual(do_req(303, 'HEAD'), ('', 'HEAD'))
+
+            self.assertEqual(do_req(303, 'PUT'), ('', 'GET'))
+
+            # 301 and 302 turn POST only into a GET
+            self.assertEqual(do_req(301, 'POST'), ('', 'GET'))
+            self.assertEqual(do_req(301, 'HEAD'), ('', 'HEAD'))
+            self.assertEqual(do_req(302, 'POST'), ('', 'GET'))
+            self.assertEqual(do_req(302, 'HEAD'), ('', 'HEAD'))
+
+            self.assertEqual(do_req(301, 'PUT'), ('testdata', 'PUT'))
+            self.assertEqual(do_req(302, 'PUT'), ('testdata', 'PUT'))
+
+            # 307 and 308 should not change method
+            for m in ('POST', 'PUT'):
+                self.assertEqual(do_req(307, m), ('testdata', m))
+                self.assertEqual(do_req(308, m), ('testdata', m))
+
+            self.assertEqual(do_req(307, 'HEAD'), ('', 'HEAD'))
+            self.assertEqual(do_req(308, 'HEAD'), ('', 'HEAD'))
+
+            # These should not redirect and instead raise an HTTPError
+            for code in (300, 304, 305, 306):
+                with self.assertRaises(urllib.error.HTTPError):
+                    do_req(code, 'GET')
+
+    def test_content_type(self):
+        # https://github.com/yt-dlp/yt-dlp/commit/379a4f161d4ad3e40932dcf5aca6e6fb9715ab28
+        with FakeYDL({'nocheckcertificate': True}) as ydl:
+            # method should be auto-detected as POST
+            r = sanitized_Request(f'https://localhost:{self.https_port}/headers', data=urlencode_postdata({'test': 'test'}))
+
+            headers = ydl.urlopen(r).read().decode('utf-8')
+            self.assertIn('Content-Type: application/x-www-form-urlencoded', headers)
+
+            # test http
+            r = sanitized_Request(f'http://localhost:{self.http_port}/headers', data=urlencode_postdata({'test': 'test'}))
+            headers = ydl.urlopen(r).read().decode('utf-8')
+            self.assertIn('Content-Type: application/x-www-form-urlencoded', headers)
+
+    def test_cookiejar(self):
+        with FakeYDL() as ydl:
+            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(
+                0, 'test', 'ytdlp', None, False, '127.0.0.1', True,
+                False, '/headers', True, False, None, False, None, None, {}))
+            data = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/headers')).read()
+            self.assertIn(b'Cookie: test=ytdlp', data)
+
+    def test_no_compression_compat_header(self):
+        with FakeYDL() as ydl:
+            data = ydl.urlopen(
+                sanitized_Request(
+                    f'http://127.0.0.1:{self.http_port}/headers',
+                    headers={'Youtubedl-no-compression': True})).read()
+            self.assertIn(b'Accept-Encoding: identity', data)
+            self.assertNotIn(b'youtubedl-no-compression', data.lower())
+
+    def test_gzip_trailing_garbage(self):
+        # https://github.com/ytdl-org/youtube-dl/commit/aa3e950764337ef9800c936f4de89b31c00dfcf5
+        # https://github.com/ytdl-org/youtube-dl/commit/6f2ec15cee79d35dba065677cad9da7491ec6e6f
+        with FakeYDL() as ydl:
+            data = ydl.urlopen(sanitized_Request(f'http://localhost:{self.http_port}/trailing_garbage')).read().decode('utf-8')
+            self.assertEqual(data, '<html><video src="/vid.mp4" /></html>')
 
 
 class TestClientCert(unittest.TestCase):
@@ -112,8 +327,8 @@ def _run_test(self, **params):
             'nocheckcertificate': True,
             **params,
         })
-        r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
-        self.assertEqual(r['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
+        r = ydl.extract_info(f'https://127.0.0.1:{self.port}/video.html')
+        self.assertEqual(r['url'], f'https://127.0.0.1:{self.port}/vid.mp4')
 
     def test_certificate_combined_nopass(self):
         self._run_test(client_certificate=os.path.join(self.certdir, 'clientwithkey.crt'))
@@ -188,5 +403,22 @@ def test_proxy_with_idn(self):
         self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')
 
 
+class TestFileURL(unittest.TestCase):
+    # See https://github.com/ytdl-org/youtube-dl/issues/8227
+    def test_file_urls(self):
+        tf = tempfile.NamedTemporaryFile(delete=False)
+        tf.write(b'foobar')
+        tf.close()
+        url = pathlib.Path(tf.name).as_uri()
+        with FakeYDL() as ydl:
+            self.assertRaisesRegex(
+                urllib.error.URLError, 'file:// URLs are explicitly disabled in yt-dlp for security reasons', ydl.urlopen, url)
+        with FakeYDL({'enable_file_urls': True}) as ydl:
+            res = ydl.urlopen(url)
+            self.assertEqual(res.read(), b'foobar')
+            res.close()
+        os.unlink(tf.name)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 238b0fe69..d78022295 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -1664,61 +1664,44 @@ class YoutubeDLRedirectHandler(urllib.request.HTTPRedirectHandler):
 
     The code is based on HTTPRedirectHandler implementation from CPython [1].
 
-    This redirect handler solves two issues:
-     - ensures redirect URL is always unicode under python 2
-     - introduces support for experimental HTTP response status code
-       308 Permanent Redirect [2] used by some sites [3]
+    This redirect handler fixes and improves the logic to better align with RFC7261
+     and what browsers tend to do [2][3]
 
     1. https://github.com/python/cpython/blob/master/Lib/urllib/request.py
-    2. https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/308
-    3. https://github.com/ytdl-org/youtube-dl/issues/28768
+    2. https://datatracker.ietf.org/doc/html/rfc7231
+    3. https://github.com/python/cpython/issues/91306
     """
 
     http_error_301 = http_error_303 = http_error_307 = http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
 
     def redirect_request(self, req, fp, code, msg, headers, newurl):
-        """Return a Request or None in response to a redirect.
-
-        This is called by the http_error_30x methods when a
-        redirection response is received.  If a redirection should
-        take place, return a new Request to allow http_error_30x to
-        perform the redirect.  Otherwise, raise HTTPError if no-one
-        else should try to handle this url.  Return None if you can't
-        but another Handler might.
-        """
-        m = req.get_method()
-        if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
-                 or code in (301, 302, 303) and m == "POST")):
+        if code not in (301, 302, 303, 307, 308):
             raise urllib.error.HTTPError(req.full_url, code, msg, headers, fp)
-        # Strictly (according to RFC 2616), 301 or 302 in response to
-        # a POST MUST NOT cause a redirection without confirmation
-        # from the user (of urllib.request, in this case).  In practice,
-        # essentially all clients do redirect in this case, so we do
-        # the same.
-
-        # Be conciliant with URIs containing a space.  This is mainly
-        # redundant with the more complete encoding done in http_error_302(),
-        # but it is kept for compatibility with other callers.
-        newurl = newurl.replace(' ', '%20')
-
-        CONTENT_HEADERS = ("content-length", "content-type")
-        # NB: don't use dict comprehension for python 2.6 compatibility
-        newheaders = {k: v for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS}
 
+        new_method = req.get_method()
+        new_data = req.data
+        remove_headers = []
         # A 303 must either use GET or HEAD for subsequent request
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.4
-        if code == 303 and m != 'HEAD':
-            m = 'GET'
+        if code == 303 and req.get_method() != 'HEAD':
+            new_method = 'GET'
         # 301 and 302 redirects are commonly turned into a GET from a POST
         # for subsequent requests by browsers, so we'll do the same.
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.2
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.3
-        if code in (301, 302) and m == 'POST':
-            m = 'GET'
+        elif code in (301, 302) and req.get_method() == 'POST':
+            new_method = 'GET'
+
+        # only remove payload if method changed (e.g. POST to GET)
+        if new_method != req.get_method():
+            new_data = None
+            remove_headers.extend(['Content-Length', 'Content-Type'])
+
+        new_headers = {k: v for k, v in req.headers.items() if k.lower() not in remove_headers}
 
         return urllib.request.Request(
-            newurl, headers=newheaders, origin_req_host=req.origin_req_host,
-            unverifiable=True, method=m)
+            newurl, headers=new_headers, origin_req_host=req.origin_req_host,
+            unverifiable=True, method=new_method, data=new_data)
 
 
 def extract_timezone(date_str):

From b87e01c123fd560b6a674ce00f45a9459d82d98a Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 27 May 2023 19:08:19 +1200
Subject: [PATCH 283/871] [cookies] Move `YoutubeDLCookieJar` to cookies module
 (#7091)

Authored by: coletdjnz
---
 test/test_YoutubeDLCookieJar.py |   8 +-
 yt_dlp/YoutubeDL.py             |   7 +-
 yt_dlp/cookies.py               | 144 +++++++++++++++++++++++++++++++-
 yt_dlp/extractor/common.py      |   2 +-
 yt_dlp/utils/_legacy.py         |   3 +
 yt_dlp/utils/_utils.py          | 130 ----------------------------
 6 files changed, 157 insertions(+), 137 deletions(-)

diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py
index 0d4e7dc97..2c73d7d85 100644
--- a/test/test_YoutubeDLCookieJar.py
+++ b/test/test_YoutubeDLCookieJar.py
@@ -11,7 +11,7 @@
 import re
 import tempfile
 
-from yt_dlp.utils import YoutubeDLCookieJar
+from yt_dlp.cookies import YoutubeDLCookieJar
 
 
 class TestYoutubeDLCookieJar(unittest.TestCase):
@@ -47,6 +47,12 @@ def test_malformed_cookies(self):
         # will be ignored
         self.assertFalse(cookiejar._cookies)
 
+    def test_get_cookie_header(self):
+        cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt')
+        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        header = cookiejar.get_cookie_header('https://www.foobar.foobar')
+        self.assertIn('HTTPONLY_COOKIE', header)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index e1e558836..f69bc98c5 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2404,7 +2404,7 @@ def _calc_headers(self, info_dict):
         if 'Youtubedl-No-Compression' in res:  # deprecated
             res.pop('Youtubedl-No-Compression', None)
             res['Accept-Encoding'] = 'identity'
-        cookies = self._calc_cookies(info_dict['url'])
+        cookies = self.cookiejar.get_cookie_header(info_dict['url'])
         if cookies:
             res['Cookie'] = cookies
 
@@ -2416,9 +2416,8 @@ def _calc_headers(self, info_dict):
         return res
 
     def _calc_cookies(self, url):
-        pr = sanitized_Request(url)
-        self.cookiejar.add_cookie_header(pr)
-        return pr.get_header('Cookie')
+        self.deprecation_warning('"YoutubeDL._calc_cookies" is deprecated and may be removed in a future version')
+        return self.cookiejar.get_cookie_header(url)
 
     def _sort_thumbnails(self, thumbnails):
         thumbnails.sort(key=lambda t: (
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index 4cafb522e..eb6a2656b 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -1,7 +1,9 @@
 import base64
+import collections
 import contextlib
 import http.cookiejar
 import http.cookies
+import io
 import json
 import os
 import re
@@ -11,6 +13,7 @@
 import sys
 import tempfile
 import time
+import urllib.request
 from datetime import datetime, timedelta, timezone
 from enum import Enum, auto
 from hashlib import pbkdf2_hmac
@@ -29,11 +32,14 @@
 from .minicurses import MultilinePrinter, QuietMultilinePrinter
 from .utils import (
     Popen,
-    YoutubeDLCookieJar,
     error_to_str,
+    escape_url,
     expand_path,
     is_path_like,
+    sanitize_url,
+    str_or_none,
     try_call,
+    write_string,
 )
 
 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
@@ -1091,3 +1097,139 @@ def load(self, data):
 
             else:
                 morsel = None
+
+
+class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
+    """
+    See [1] for cookie file format.
+
+    1. https://curl.haxx.se/docs/http-cookies.html
+    """
+    _HTTPONLY_PREFIX = '#HttpOnly_'
+    _ENTRY_LEN = 7
+    _HEADER = '''# Netscape HTTP Cookie File
+# This file is generated by yt-dlp.  Do not edit.
+
+'''
+    _CookieFileEntry = collections.namedtuple(
+        'CookieFileEntry',
+        ('domain_name', 'include_subdomains', 'path', 'https_only', 'expires_at', 'name', 'value'))
+
+    def __init__(self, filename=None, *args, **kwargs):
+        super().__init__(None, *args, **kwargs)
+        if is_path_like(filename):
+            filename = os.fspath(filename)
+        self.filename = filename
+
+    @staticmethod
+    def _true_or_false(cndn):
+        return 'TRUE' if cndn else 'FALSE'
+
+    @contextlib.contextmanager
+    def open(self, file, *, write=False):
+        if is_path_like(file):
+            with open(file, 'w' if write else 'r', encoding='utf-8') as f:
+                yield f
+        else:
+            if write:
+                file.truncate(0)
+            yield file
+
+    def _really_save(self, f, ignore_discard=False, ignore_expires=False):
+        now = time.time()
+        for cookie in self:
+            if (not ignore_discard and cookie.discard
+                    or not ignore_expires and cookie.is_expired(now)):
+                continue
+            name, value = cookie.name, cookie.value
+            if value is None:
+                # cookies.txt regards 'Set-Cookie: foo' as a cookie
+                # with no name, whereas http.cookiejar regards it as a
+                # cookie with no value.
+                name, value = '', name
+            f.write('%s\n' % '\t'.join((
+                cookie.domain,
+                self._true_or_false(cookie.domain.startswith('.')),
+                cookie.path,
+                self._true_or_false(cookie.secure),
+                str_or_none(cookie.expires, default=''),
+                name, value
+            )))
+
+    def save(self, filename=None, *args, **kwargs):
+        """
+        Save cookies to a file.
+        Code is taken from CPython 3.6
+        https://github.com/python/cpython/blob/8d999cbf4adea053be6dbb612b9844635c4dfb8e/Lib/http/cookiejar.py#L2091-L2117 """
+
+        if filename is None:
+            if self.filename is not None:
+                filename = self.filename
+            else:
+                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
+
+        # Store session cookies with `expires` set to 0 instead of an empty string
+        for cookie in self:
+            if cookie.expires is None:
+                cookie.expires = 0
+
+        with self.open(filename, write=True) as f:
+            f.write(self._HEADER)
+            self._really_save(f, *args, **kwargs)
+
+    def load(self, filename=None, ignore_discard=False, ignore_expires=False):
+        """Load cookies from a file."""
+        if filename is None:
+            if self.filename is not None:
+                filename = self.filename
+            else:
+                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
+
+        def prepare_line(line):
+            if line.startswith(self._HTTPONLY_PREFIX):
+                line = line[len(self._HTTPONLY_PREFIX):]
+            # comments and empty lines are fine
+            if line.startswith('#') or not line.strip():
+                return line
+            cookie_list = line.split('\t')
+            if len(cookie_list) != self._ENTRY_LEN:
+                raise http.cookiejar.LoadError('invalid length %d' % len(cookie_list))
+            cookie = self._CookieFileEntry(*cookie_list)
+            if cookie.expires_at and not cookie.expires_at.isdigit():
+                raise http.cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
+            return line
+
+        cf = io.StringIO()
+        with self.open(filename) as f:
+            for line in f:
+                try:
+                    cf.write(prepare_line(line))
+                except http.cookiejar.LoadError as e:
+                    if f'{line.strip()} '[0] in '[{"':
+                        raise http.cookiejar.LoadError(
+                            'Cookies file must be Netscape formatted, not JSON. See  '
+                            'https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp')
+                    write_string(f'WARNING: skipping cookie file entry due to {e}: {line!r}\n')
+                    continue
+        cf.seek(0)
+        self._really_load(cf, filename, ignore_discard, ignore_expires)
+        # Session cookies are denoted by either `expires` field set to
+        # an empty string or 0. MozillaCookieJar only recognizes the former
+        # (see [1]). So we need force the latter to be recognized as session
+        # cookies on our own.
+        # Session cookies may be important for cookies-based authentication,
+        # e.g. usually, when user does not check 'Remember me' check box while
+        # logging in on a site, some important cookies are stored as session
+        # cookies so that not recognizing them will result in failed login.
+        # 1. https://bugs.python.org/issue17164
+        for cookie in self:
+            # Treat `expires=0` cookies as session cookies
+            if cookie.expires == 0:
+                cookie.expires = None
+                cookie.discard = True
+
+    def get_cookie_header(self, url):
+        """Generate a Cookie HTTP header for a given url"""
+        cookie_req = urllib.request.Request(escape_url(sanitize_url(url)))
+        self.add_cookie_header(cookie_req)
+        return cookie_req.get_header('Cookie')
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 1b1dd560f..306911a6c 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -3444,7 +3444,7 @@ def _set_cookie(self, domain, name, value, expire_time=None, port=None,
 
     def _get_cookies(self, url):
         """ Return a http.cookies.SimpleCookie with the cookies for the url """
-        return LenientSimpleCookie(self._downloader._calc_cookies(url))
+        return LenientSimpleCookie(self._downloader.cookiejar.get_cookie_header(url))
 
     def _apply_first_set_cookie_header(self, url_handle, cookie):
         """
diff --git a/yt_dlp/utils/_legacy.py b/yt_dlp/utils/_legacy.py
index b0578a1d6..1097778f0 100644
--- a/yt_dlp/utils/_legacy.py
+++ b/yt_dlp/utils/_legacy.py
@@ -10,6 +10,9 @@
 from .traversal import traverse_obj
 from ..dependencies import certifi, websockets
 
+# isort: split
+from ..cookies import YoutubeDLCookieJar  # noqa: F401
+
 has_certifi = bool(certifi)
 has_websockets = bool(websockets)
 
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index d78022295..6f4f22bb3 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -1518,136 +1518,6 @@ def is_path_like(f):
     return isinstance(f, (str, bytes, os.PathLike))
 
 
-class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
-    """
-    See [1] for cookie file format.
-
-    1. https://curl.haxx.se/docs/http-cookies.html
-    """
-    _HTTPONLY_PREFIX = '#HttpOnly_'
-    _ENTRY_LEN = 7
-    _HEADER = '''# Netscape HTTP Cookie File
-# This file is generated by yt-dlp.  Do not edit.
-
-'''
-    _CookieFileEntry = collections.namedtuple(
-        'CookieFileEntry',
-        ('domain_name', 'include_subdomains', 'path', 'https_only', 'expires_at', 'name', 'value'))
-
-    def __init__(self, filename=None, *args, **kwargs):
-        super().__init__(None, *args, **kwargs)
-        if is_path_like(filename):
-            filename = os.fspath(filename)
-        self.filename = filename
-
-    @staticmethod
-    def _true_or_false(cndn):
-        return 'TRUE' if cndn else 'FALSE'
-
-    @contextlib.contextmanager
-    def open(self, file, *, write=False):
-        if is_path_like(file):
-            with open(file, 'w' if write else 'r', encoding='utf-8') as f:
-                yield f
-        else:
-            if write:
-                file.truncate(0)
-            yield file
-
-    def _really_save(self, f, ignore_discard=False, ignore_expires=False):
-        now = time.time()
-        for cookie in self:
-            if (not ignore_discard and cookie.discard
-                    or not ignore_expires and cookie.is_expired(now)):
-                continue
-            name, value = cookie.name, cookie.value
-            if value is None:
-                # cookies.txt regards 'Set-Cookie: foo' as a cookie
-                # with no name, whereas http.cookiejar regards it as a
-                # cookie with no value.
-                name, value = '', name
-            f.write('%s\n' % '\t'.join((
-                cookie.domain,
-                self._true_or_false(cookie.domain.startswith('.')),
-                cookie.path,
-                self._true_or_false(cookie.secure),
-                str_or_none(cookie.expires, default=''),
-                name, value
-            )))
-
-    def save(self, filename=None, *args, **kwargs):
-        """
-        Save cookies to a file.
-        Code is taken from CPython 3.6
-        https://github.com/python/cpython/blob/8d999cbf4adea053be6dbb612b9844635c4dfb8e/Lib/http/cookiejar.py#L2091-L2117 """
-
-        if filename is None:
-            if self.filename is not None:
-                filename = self.filename
-            else:
-                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
-
-        # Store session cookies with `expires` set to 0 instead of an empty string
-        for cookie in self:
-            if cookie.expires is None:
-                cookie.expires = 0
-
-        with self.open(filename, write=True) as f:
-            f.write(self._HEADER)
-            self._really_save(f, *args, **kwargs)
-
-    def load(self, filename=None, ignore_discard=False, ignore_expires=False):
-        """Load cookies from a file."""
-        if filename is None:
-            if self.filename is not None:
-                filename = self.filename
-            else:
-                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
-
-        def prepare_line(line):
-            if line.startswith(self._HTTPONLY_PREFIX):
-                line = line[len(self._HTTPONLY_PREFIX):]
-            # comments and empty lines are fine
-            if line.startswith('#') or not line.strip():
-                return line
-            cookie_list = line.split('\t')
-            if len(cookie_list) != self._ENTRY_LEN:
-                raise http.cookiejar.LoadError('invalid length %d' % len(cookie_list))
-            cookie = self._CookieFileEntry(*cookie_list)
-            if cookie.expires_at and not cookie.expires_at.isdigit():
-                raise http.cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
-            return line
-
-        cf = io.StringIO()
-        with self.open(filename) as f:
-            for line in f:
-                try:
-                    cf.write(prepare_line(line))
-                except http.cookiejar.LoadError as e:
-                    if f'{line.strip()} '[0] in '[{"':
-                        raise http.cookiejar.LoadError(
-                            'Cookies file must be Netscape formatted, not JSON. See  '
-                            'https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp')
-                    write_string(f'WARNING: skipping cookie file entry due to {e}: {line!r}\n')
-                    continue
-        cf.seek(0)
-        self._really_load(cf, filename, ignore_discard, ignore_expires)
-        # Session cookies are denoted by either `expires` field set to
-        # an empty string or 0. MozillaCookieJar only recognizes the former
-        # (see [1]). So we need force the latter to be recognized as session
-        # cookies on our own.
-        # Session cookies may be important for cookies-based authentication,
-        # e.g. usually, when user does not check 'Remember me' check box while
-        # logging in on a site, some important cookies are stored as session
-        # cookies so that not recognizing them will result in failed login.
-        # 1. https://bugs.python.org/issue17164
-        for cookie in self:
-            # Treat `expires=0` cookies as session cookies
-            if cookie.expires == 0:
-                cookie.expires = None
-                cookie.discard = True
-
-
 class YoutubeDLCookieProcessor(urllib.request.HTTPCookieProcessor):
     def __init__(self, cookiejar=None):
         urllib.request.HTTPCookieProcessor.__init__(self, cookiejar)

From 3f66b6fe50f8d5b545712f8b19d5ae62f5373980 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 27 May 2023 19:17:27 +1200
Subject: [PATCH 284/871] [core] Workaround erroneous urllib Windows proxy
 parsing (#7092)

Convert proxies extracted from windows registry to http for older Python versions.
See: https://github.com/python/cpython/issues/86793

Authored by: coletdjnz
---
 Makefile                         |  2 +-
 yt_dlp/YoutubeDL.py              |  2 +-
 yt_dlp/compat/urllib/__init__.py |  7 ++++++
 yt_dlp/compat/urllib/request.py  | 40 ++++++++++++++++++++++++++++++++
 4 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 yt_dlp/compat/urllib/__init__.py
 create mode 100644 yt_dlp/compat/urllib/request.py

diff --git a/Makefile b/Makefile
index f03fe2052..b1ac0e7d6 100644
--- a/Makefile
+++ b/Makefile
@@ -74,7 +74,7 @@ offlinetest: codetest
 	$(PYTHON) -m pytest -k "not download"
 
 # XXX: This is hard to maintain
-CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/utils yt_dlp/dependencies
+CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/compat/urllib yt_dlp/utils yt_dlp/dependencies
 yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
 	mkdir -p zip
 	for d in $(CODE_FOLDERS) ; do \
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index f69bc98c5..f49dbf07d 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -21,9 +21,9 @@
 import tokenize
 import traceback
 import unicodedata
-import urllib.request
 
 from .cache import Cache
+from .compat import urllib  # isort: split
 from .compat import compat_os_name, compat_shlex_quote
 from .cookies import load_cookies
 from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name
diff --git a/yt_dlp/compat/urllib/__init__.py b/yt_dlp/compat/urllib/__init__.py
new file mode 100644
index 000000000..6b6b8e103
--- /dev/null
+++ b/yt_dlp/compat/urllib/__init__.py
@@ -0,0 +1,7 @@
+# flake8: noqa: F405
+from urllib import *  # noqa: F403
+
+from ..compat_utils import passthrough_module
+
+passthrough_module(__name__, 'urllib')
+del passthrough_module
diff --git a/yt_dlp/compat/urllib/request.py b/yt_dlp/compat/urllib/request.py
new file mode 100644
index 000000000..ff63b2f0e
--- /dev/null
+++ b/yt_dlp/compat/urllib/request.py
@@ -0,0 +1,40 @@
+# flake8: noqa: F405
+from urllib.request import *  # noqa: F403
+
+from ..compat_utils import passthrough_module
+
+passthrough_module(__name__, 'urllib.request')
+del passthrough_module
+
+
+from .. import compat_os_name
+
+if compat_os_name == 'nt':
+    # On older python versions, proxies are extracted from Windows registry erroneously. [1]
+    # If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2]
+    # It is unlikely that the user has actually set it to be https, so we should be fine to safely downgrade
+    # it to http on these older python versions to avoid issues
+    # This also applies for ftp proxy type, as ftp:// proxy scheme is not supported.
+    # 1: https://github.com/python/cpython/issues/86793
+    # 2: https://github.com/python/cpython/blob/51f1ae5ceb0673316c4e4b0175384e892e33cc6e/Lib/urllib/request.py#L2683-L2698
+    import sys
+    from urllib.request import getproxies_environment, getproxies_registry
+
+    def getproxies_registry_patched():
+        proxies = getproxies_registry()
+        if (
+            sys.version_info >= (3, 10, 5)  # https://docs.python.org/3.10/whatsnew/changelog.html#python-3-10-5-final
+            or (3, 9, 13) <= sys.version_info < (3, 10)  # https://docs.python.org/3.9/whatsnew/changelog.html#python-3-9-13-final
+        ):
+            return proxies
+
+        for scheme in ('https', 'ftp'):
+            if scheme in proxies and proxies[scheme].startswith(f'{scheme}://'):
+                proxies[scheme] = 'http' + proxies[scheme][len(scheme):]
+
+        return proxies
+
+    def getproxies():
+        return getproxies_environment() or getproxies_registry_patched()
+
+del compat_os_name

From daafbf49b3482edae4d70dd37070be99742a926e Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 27 May 2023 22:40:05 +1200
Subject: [PATCH 285/871] [core] Support decoding multiple content encodings
 (#7142)

Authored by: coletdjnz
---
 test/test_http.py      | 76 ++++++++++++++++++++++++++++++++++++++++++
 yt_dlp/utils/_utils.py | 61 +++++++++++++++++----------------
 2 files changed, 108 insertions(+), 29 deletions(-)

diff --git a/test/test_http.py b/test/test_http.py
index d684905da..3941a6e77 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -17,9 +17,11 @@
 import threading
 import urllib.error
 import urllib.request
+import zlib
 
 from test.helper import http_server_port
 from yt_dlp import YoutubeDL
+from yt_dlp.dependencies import brotli
 from yt_dlp.utils import sanitized_Request, urlencode_postdata
 
 from .helper import FakeYDL
@@ -148,6 +150,31 @@ def do_GET(self):
             self.send_header('Location', new_url)
             self.send_header('Content-Length', '0')
             self.end_headers()
+        elif self.path == '/content-encoding':
+            encodings = self.headers.get('ytdl-encoding', '')
+            payload = b'<html><video src="/vid.mp4" /></html>'
+            for encoding in filter(None, (e.strip() for e in encodings.split(','))):
+                if encoding == 'br' and brotli:
+                    payload = brotli.compress(payload)
+                elif encoding == 'gzip':
+                    buf = io.BytesIO()
+                    with gzip.GzipFile(fileobj=buf, mode='wb') as f:
+                        f.write(payload)
+                    payload = buf.getvalue()
+                elif encoding == 'deflate':
+                    payload = zlib.compress(payload)
+                elif encoding == 'unsupported':
+                    payload = b'raw'
+                    break
+                else:
+                    self._status(415)
+                    return
+            self.send_response(200)
+            self.send_header('Content-Encoding', encodings)
+            self.send_header('Content-Length', str(len(payload)))
+            self.end_headers()
+            self.wfile.write(payload)
+
         else:
             self._status(404)
 
@@ -302,6 +329,55 @@ def test_gzip_trailing_garbage(self):
             data = ydl.urlopen(sanitized_Request(f'http://localhost:{self.http_port}/trailing_garbage')).read().decode('utf-8')
             self.assertEqual(data, '<html><video src="/vid.mp4" /></html>')
 
+    @unittest.skipUnless(brotli, 'brotli support is not installed')
+    def test_brotli(self):
+        with FakeYDL() as ydl:
+            res = ydl.urlopen(
+                sanitized_Request(
+                    f'http://127.0.0.1:{self.http_port}/content-encoding',
+                    headers={'ytdl-encoding': 'br'}))
+            self.assertEqual(res.headers.get('Content-Encoding'), 'br')
+            self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+
+    def test_deflate(self):
+        with FakeYDL() as ydl:
+            res = ydl.urlopen(
+                sanitized_Request(
+                    f'http://127.0.0.1:{self.http_port}/content-encoding',
+                    headers={'ytdl-encoding': 'deflate'}))
+            self.assertEqual(res.headers.get('Content-Encoding'), 'deflate')
+            self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+
+    def test_gzip(self):
+        with FakeYDL() as ydl:
+            res = ydl.urlopen(
+                sanitized_Request(
+                    f'http://127.0.0.1:{self.http_port}/content-encoding',
+                    headers={'ytdl-encoding': 'gzip'}))
+            self.assertEqual(res.headers.get('Content-Encoding'), 'gzip')
+            self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+
+    def test_multiple_encodings(self):
+        # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.4
+        with FakeYDL() as ydl:
+            for pair in ('gzip,deflate', 'deflate, gzip', 'gzip, gzip', 'deflate, deflate'):
+                res = ydl.urlopen(
+                    sanitized_Request(
+                        f'http://127.0.0.1:{self.http_port}/content-encoding',
+                        headers={'ytdl-encoding': pair}))
+                self.assertEqual(res.headers.get('Content-Encoding'), pair)
+                self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+
+    def test_unsupported_encoding(self):
+        # it should return the raw content
+        with FakeYDL() as ydl:
+            res = ydl.urlopen(
+                sanitized_Request(
+                    f'http://127.0.0.1:{self.http_port}/content-encoding',
+                    headers={'ytdl-encoding': 'unsupported'}))
+            self.assertEqual(res.headers.get('Content-Encoding'), 'unsupported')
+            self.assertEqual(res.read(), b'raw')
+
 
 class TestClientCert(unittest.TestCase):
     def setUp(self):
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 6f4f22bb3..7c91faff8 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -1361,6 +1361,23 @@ def brotli(data):
             return data
         return brotli.decompress(data)
 
+    @staticmethod
+    def gz(data):
+        gz = gzip.GzipFile(fileobj=io.BytesIO(data), mode='rb')
+        try:
+            return gz.read()
+        except OSError as original_oserror:
+            # There may be junk add the end of the file
+            # See http://stackoverflow.com/q/4928560/35070 for details
+            for i in range(1, 1024):
+                try:
+                    gz = gzip.GzipFile(fileobj=io.BytesIO(data[:-i]), mode='rb')
+                    return gz.read()
+                except OSError:
+                    continue
+            else:
+                raise original_oserror
+
     def http_request(self, req):
         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
         # always respected by websites, some tend to give out URLs with non percent-encoded
@@ -1394,35 +1411,21 @@ def http_request(self, req):
 
     def http_response(self, req, resp):
         old_resp = resp
-        # gzip
-        if resp.headers.get('Content-encoding', '') == 'gzip':
-            content = resp.read()
-            gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
-            try:
-                uncompressed = io.BytesIO(gz.read())
-            except OSError as original_ioerror:
-                # There may be junk add the end of the file
-                # See http://stackoverflow.com/q/4928560/35070 for details
-                for i in range(1, 1024):
-                    try:
-                        gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
-                        uncompressed = io.BytesIO(gz.read())
-                    except OSError:
-                        continue
-                    break
-                else:
-                    raise original_ioerror
-            resp = urllib.request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
-            resp.msg = old_resp.msg
-        # deflate
-        if resp.headers.get('Content-encoding', '') == 'deflate':
-            gz = io.BytesIO(self.deflate(resp.read()))
-            resp = urllib.request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
-            resp.msg = old_resp.msg
-        # brotli
-        if resp.headers.get('Content-encoding', '') == 'br':
-            resp = urllib.request.addinfourl(
-                io.BytesIO(self.brotli(resp.read())), old_resp.headers, old_resp.url, old_resp.code)
+
+        # Content-Encoding header lists the encodings in order that they were applied [1].
+        # To decompress, we simply do the reverse.
+        # [1]: https://datatracker.ietf.org/doc/html/rfc9110#name-content-encoding
+        decoded_response = None
+        for encoding in (e.strip() for e in reversed(resp.headers.get('Content-encoding', '').split(','))):
+            if encoding == 'gzip':
+                decoded_response = self.gz(decoded_response or resp.read())
+            elif encoding == 'deflate':
+                decoded_response = self.deflate(decoded_response or resp.read())
+            elif encoding == 'br' and brotli:
+                decoded_response = self.brotli(decoded_response or resp.read())
+
+        if decoded_response is not None:
+            resp = urllib.request.addinfourl(io.BytesIO(decoded_response), old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
         # https://github.com/ytdl-org/youtube-dl/issues/6457).

From 6dc00acf0f1f1107a626c21befd1691403e6aeeb Mon Sep 17 00:00:00 2001
From: Mohamed Al Mehairbi <62325490+ItzMaxTV@users.noreply.github.com>
Date: Sat, 27 May 2023 22:32:39 +0400
Subject: [PATCH 286/871] [extractor/weyyak] Add extractor (#7124)

Closes #7118
Authored by: ItzMaxTV
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/weyyak.py      | 86 +++++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 yt_dlp/extractor/weyyak.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 49dd9aecd..c288dca19 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2329,6 +2329,7 @@
     WeverseLiveIE,
 )
 from .wevidi import WeVidiIE
+from .weyyak import WeyyakIE
 from .whyp import WhypIE
 from .wikimedia import WikimediaIE
 from .willow import WillowIE
diff --git a/yt_dlp/extractor/weyyak.py b/yt_dlp/extractor/weyyak.py
new file mode 100644
index 000000000..ef12be871
--- /dev/null
+++ b/yt_dlp/extractor/weyyak.py
@@ -0,0 +1,86 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    parse_age_limit,
+    traverse_obj,
+    unified_timestamp,
+    url_or_none,
+)
+
+
+class WeyyakIE(InfoExtractor):
+    _VALID_URL = r'https?://weyyak\.com/(?P<lang>\w+)/(?:player/)?(?P<type>episode|movie)/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'https://weyyak.com/en/player/episode/1341952/Ribat-Al-Hob-Episode49',
+            'md5': '0caf55c1a615531c8fe60f146ae46849',
+            'info_dict': {
+                'id': '1341952',
+                'ext': 'mp4',
+                'title': 'Ribat Al Hob',
+                'duration': 2771,
+                'alt_title': 'رباط الحب',
+                'season': 'Season 1',
+                'season_number': 1,
+                'episode': 'Episode 49',
+                'episode_number': 49,
+                'timestamp': 1485907200,
+                'upload_date': '20170201',
+                'thumbnail': r're:^https://content\.weyyak\.com/.+/poster-image',
+                'categories': ['Drama', 'Thrillers', 'Romance'],
+                'tags': 'count:8',
+            },
+        },
+        {
+            'url': 'https://weyyak.com/en/movie/233255/8-Seconds',
+            'md5': 'fe740ae0f63e4d1c8a7fc147a410c564',
+            'info_dict': {
+                'id': '233255',
+                'ext': 'mp4',
+                'title': '8 Seconds',
+                'duration': 6490,
+                'alt_title': '8 ثواني',
+                'description': 'md5:45b83a155c30b49950624c7e99600b9d',
+                'age_limit': 15,
+                'release_year': 2015,
+                'timestamp': 1683106031,
+                'upload_date': '20230503',
+                'thumbnail': r're:^https://content\.weyyak\.com/.+/poster-image',
+                'categories': ['Drama', 'Social'],
+                'cast': ['Ceylin Adiyaman', 'Esra Inal'],
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id, lang, type_ = self._match_valid_url(url).group('id', 'lang', 'type')
+
+        path = 'episode/' if type_ == 'episode' else 'contents/moviedetails?contentkey='
+        data = self._download_json(
+            f'https://msapifo-prod-me.weyyak.z5.com/v1/{lang}/{path}{video_id}', video_id)['data']
+        m3u8_url = self._download_json(
+            f'https://api-weyyak.akamaized.net/get_info/{data["video_id"]}',
+            video_id, 'Extracting video details')['url_video']
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'alt_title': ('translated_title', {str}),
+                'description': ('synopsis', {str}),
+                'duration': ('length', {float_or_none}),
+                'age_limit': ('age_rating', {parse_age_limit}),
+                'season_number': ('season_number', {int_or_none}),
+                'episode_number': ('episode_number', {int_or_none}),
+                'thumbnail': ('imagery', 'thumbnail', {url_or_none}),
+                'categories': ('genres', ..., {str}),
+                'tags': ('tags', ..., {str}),
+                'cast': (('main_actor', 'main_actress'), {str}),
+                'timestamp': ('insertedAt', {unified_timestamp}),
+                'release_year': ('production_year', {int_or_none}),
+            }),
+        }

From 93e12ed76ef49252dc6869b59d21d0777e5e11af Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sun, 28 May 2023 11:31:45 +1200
Subject: [PATCH 287/871] [extractor/youtube] Extract uploader metadata for
 feed/playlist items

Fixes https://github.com/yt-dlp/yt-dlp/issues/7104

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 80edcd77d..3f0a4cd20 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -1038,6 +1038,13 @@ def _extract_video(self, renderer):
                       else self._get_count({'simpleText': view_count_text}))
         view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
 
+        channel = (self._get_text(renderer, 'ownerText', 'shortBylineText')
+                   or self._get_text(reel_header_renderer, 'channelTitleText'))
+
+        channel_handle = traverse_obj(renderer, (
+            'shortBylineText', 'runs', ..., 'navigationEndpoint',
+            (('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl'))),
+            expected_type=self.handle_from_url, get_all=False)
         return {
             '_type': 'url',
             'ie_key': YoutubeIE.ie_key(),
@@ -1047,9 +1054,11 @@ def _extract_video(self, renderer):
             'description': description,
             'duration': duration,
             'channel_id': channel_id,
-            'channel': (self._get_text(renderer, 'ownerText', 'shortBylineText')
-                        or self._get_text(reel_header_renderer, 'channelTitleText')),
+            'channel': channel,
             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
+            'uploader': channel,
+            'uploader_id': channel_handle,
+            'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
             'timestamp': (self._parse_time_text(time_text)
                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
@@ -5851,7 +5860,25 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader_id': '@colethedj1894',
             'uploader': 'colethedj',
         },
+        'playlist': [{
+            'info_dict': {
+                'title': 'youtube-dl test video "\'/\\ä↭𝕐',
+                'id': 'BaW_jenozKc',
+                '_type': 'url',
+                'ie_key': 'Youtube',
+                'duration': 10,
+                'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
+                'channel_url': 'https://www.youtube.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
+                'view_count': int,
+                'url': 'https://www.youtube.com/watch?v=BaW_jenozKc',
+                'channel': 'Philipp Hagemeister',
+                'uploader_id': '@PhilippHagemeister',
+                'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
+                'uploader': 'Philipp Hagemeister',
+            }
+        }],
         'playlist_count': 1,
+        'params': {'extract_flat': True},
     }, {
         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
         'url': 'https://www.youtube.com/feed/recommended',
@@ -6152,6 +6179,9 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
                 'channel_url': str,
                 'concurrent_view_count': int,
                 'channel': str,
+                'uploader': str,
+                'uploader_url': str,
+                'uploader_id': str
             }
         }],
         'params': {'extract_flat': True, 'playlist_items': '1'},

From 738c90a463257634455ada3e5c18b714c531dede Mon Sep 17 00:00:00 2001
From: "lauren n. liberda" <lauren@selfisekai.rocks>
Date: Mon, 29 May 2023 05:22:38 +0200
Subject: [PATCH 288/871] [extractor/polskieradio] Improve extractors (#5948)

Authored by: selfisekai
---
 yt_dlp/extractor/_extractors.py  |   1 -
 yt_dlp/extractor/polskieradio.py | 208 ++++++++++++++++++++-----------
 2 files changed, 137 insertions(+), 72 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index c288dca19..ba55ccbaf 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1479,7 +1479,6 @@
     PolskieRadioPlayerIE,
     PolskieRadioPodcastIE,
     PolskieRadioPodcastListIE,
-    PolskieRadioRadioKierowcowIE,
 )
 from .popcorntimes import PopcorntimesIE
 from .popcorntv import PopcornTVIE
diff --git a/yt_dlp/extractor/polskieradio.py b/yt_dlp/extractor/polskieradio.py
index 68c4a2afd..5bf92b9b5 100644
--- a/yt_dlp/extractor/polskieradio.py
+++ b/yt_dlp/extractor/polskieradio.py
@@ -2,26 +2,24 @@
 import json
 import math
 import re
+import urllib.parse
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse_unquote,
-    compat_urlparse
-)
+from ..compat import compat_str
 from ..utils import (
-    determine_ext,
-    extract_attributes,
     ExtractorError,
     InAdvancePagedList,
+    determine_ext,
+    extract_attributes,
     int_or_none,
     js_to_json,
     parse_iso8601,
     strip_or_none,
     traverse_obj,
-    unified_timestamp,
     unescapeHTML,
+    unified_timestamp,
     url_or_none,
+    urljoin,
 )
 
 
@@ -44,7 +42,7 @@ def _extract_webpage_player_entries(self, webpage, playlist_id, base_data):
                 'duration': int_or_none(media.get('length')),
                 'vcodec': 'none' if media.get('provider') == 'audio' else None,
             })
-            entry_title = compat_urllib_parse_unquote(media['desc'])
+            entry_title = urllib.parse.unquote(media['desc'])
             if entry_title:
                 entry['title'] = entry_title
             yield entry
@@ -130,10 +128,11 @@ def _real_extract(self, url):
         return self.playlist_result(entries, playlist_id, title, description)
 
 
-class PolskieRadioIE(InfoExtractor):
-    # new next.js sites, excluding radiokierowcow.pl
-    _VALID_URL = r'https?://(?:[^/]+\.)?polskieradio(?:24)?\.pl/artykul/(?P<id>\d+)'
+class PolskieRadioIE(PolskieRadioBaseExtractor):
+    # new next.js sites
+    _VALID_URL = r'https?://(?:[^/]+\.)?(?:polskieradio(?:24)?|radiokierowcow)\.pl/artykul/(?P<id>\d+)'
     _TESTS = [{
+        # articleData, attachments
         'url': 'https://jedynka.polskieradio.pl/artykul/1587943',
         'info_dict': {
             'id': '1587943',
@@ -148,6 +147,31 @@ class PolskieRadioIE(InfoExtractor):
                 'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
             },
         }],
+    }, {
+        # post, legacy html players
+        'url': 'https://trojka.polskieradio.pl/artykul/2589163,Czy-wciaz-otrzymujemy-zdjecia-z-sond-Voyager',
+        'info_dict': {
+            'id': '2589163',
+            'title': 'Czy wciąż otrzymujemy zdjęcia z sond Voyager?',
+            'description': 'md5:cf1a7f348d63a2db9c0d7a63d1669473',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '2577880',
+                'ext': 'mp3',
+                'title': 'md5:a57d10a0c02abd34dd675cb33707ad5a',
+                'duration': 321,
+            },
+        }],
+    }, {
+        # data, legacy
+        'url': 'https://radiokierowcow.pl/artykul/2694529',
+        'info_dict': {
+            'id': '2694529',
+            'title': 'Zielona fala reliktem przeszłości?',
+            'description': 'md5:f20a9a7ed9cb58916c54add94eae3bc0',
+        },
+        'playlist_count': 3,
     }, {
         'url': 'https://trojka.polskieradio.pl/artykul/1632955',
         'only_matching': True,
@@ -166,7 +190,8 @@ def _real_extract(self, url):
         webpage = self._download_webpage(url, playlist_id)
 
         article_data = traverse_obj(
-            self._search_nextjs_data(webpage, playlist_id), ('props', 'pageProps', 'data', 'articleData'))
+            self._search_nextjs_data(webpage, playlist_id), (
+                'props', 'pageProps', (('data', 'articleData'), 'post', 'data')), get_all=False)
 
         title = strip_or_none(article_data['title'])
 
@@ -178,7 +203,13 @@ def _real_extract(self, url):
             'id': self._search_regex(
                 r'([a-f\d]{8}-(?:[a-f\d]{4}-){3}[a-f\d]{12})', entry['file'], 'entry id'),
             'title': strip_or_none(entry.get('description')) or title,
-        } for entry in article_data.get('attachments') or () if entry['fileType'] in ('Audio', )]
+        } for entry in article_data.get('attachments') or () if entry.get('fileType') in ('Audio', )]
+
+        if not entries:
+            # some legacy articles have no json attachments, but players in body
+            entries = self._extract_webpage_player_entries(article_data['content'], playlist_id, {
+                'title': title,
+            })
 
         return self.playlist_result(entries, playlist_id, title, description)
 
@@ -214,6 +245,15 @@ class PolskieRadioAuditionIE(InfoExtractor):
             'thumbnail': r're:https://static\.prsa\.pl/images/.+',
         },
         'playlist_mincount': 722,
+    }, {
+        # some articles were "promoted to main page" and thus link to old frontend
+        'url': 'https://trojka.polskieradio.pl/audycja/305',
+        'info_dict': {
+            'id': '305',
+            'title': 'Co w mowie piszczy?',
+            'thumbnail': r're:https://static\.prsa\.pl/images/.+',
+        },
+        'playlist_count': 1523,
     }]
 
     def _call_lp3(self, path, query, video_id, note):
@@ -254,7 +294,6 @@ def _entries(self, playlist_id, has_episodes, has_articles):
             for article in page['data']:
                 yield {
                     '_type': 'url_transparent',
-                    'ie_key': PolskieRadioIE.ie_key(),
                     'id': str(article['id']),
                     'url': article['url'],
                     'title': article.get('shortTitle'),
@@ -282,11 +321,8 @@ def _real_extract(self, url):
 class PolskieRadioCategoryIE(InfoExtractor):
     # legacy sites
     IE_NAME = 'polskieradio:category'
-    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(?:,[^/]+)?/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/(?:\d+(?:,[^/]+)?/|[^/]+/Tag)(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://www.polskieradio.pl/7/129,Sygnaly-dnia?ref=source',
-        'only_matching': True
-    }, {
         'url': 'http://www.polskieradio.pl/37,RedakcjaKatolicka/4143,Kierunek-Krakow',
         'info_dict': {
             'id': '4143',
@@ -300,6 +336,36 @@ class PolskieRadioCategoryIE(InfoExtractor):
             'title': 'Muzyka',
         },
         'playlist_mincount': 61
+    }, {
+        # billennium tabs
+        'url': 'https://www.polskieradio.pl/8/2385',
+        'info_dict': {
+            'id': '2385',
+            'title': 'Droga przez mąkę',
+        },
+        'playlist_mincount': 111,
+    }, {
+        'url': 'https://www.polskieradio.pl/10/4930',
+        'info_dict': {
+            'id': '4930',
+            'title': 'Teraz K-pop!',
+        },
+        'playlist_mincount': 392,
+    }, {
+        # post back pages, audio content directly without articles
+        'url': 'https://www.polskieradio.pl/8,dwojka/7376,nowa-mowa',
+        'info_dict': {
+            'id': '7376',
+            'title': 'Nowa mowa',
+        },
+        'playlist_mincount': 244,
+    }, {
+        'url': 'https://www.polskieradio.pl/Krzysztof-Dziuba/Tag175458',
+        'info_dict': {
+            'id': '175458',
+            'title': 'Krzysztof Dziuba',
+        },
+        'playlist_mincount': 420,
     }, {
         'url': 'http://www.polskieradio.pl/8,Dwojka/196,Publicystyka',
         'only_matching': True,
@@ -311,25 +377,61 @@ def suitable(cls, url):
 
     def _entries(self, url, page, category_id):
         content = page
+        is_billennium_tabs = 'onclick="TB_LoadTab(' in page
+        is_post_back = 'onclick="__doPostBack(' in page
+        pagination = page if is_billennium_tabs else None
         for page_num in itertools.count(2):
             for a_entry, entry_id in re.findall(
-                    r'(?s)<article[^>]+>.*?(<a[^>]+href=["\']/\d+/\d+/Artykul/(\d+)[^>]+>).*?</article>',
+                    r'(?s)<article[^>]+>.*?(<a[^>]+href=["\'](?:(?:https?)?://[^/]+)?/\d+/\d+/Artykul/(\d+)[^>]+>).*?</article>',
                     content):
                 entry = extract_attributes(a_entry)
-                href = entry.get('href')
-                if not href:
-                    continue
-                yield self.url_result(
-                    compat_urlparse.urljoin(url, href), PolskieRadioLegacyIE,
-                    entry_id, entry.get('title'))
-            mobj = re.search(
-                r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1',
-                content)
-            if not mobj:
-                break
-            next_url = compat_urlparse.urljoin(url, mobj.group('url'))
-            content = self._download_webpage(
-                next_url, category_id, 'Downloading page %s' % page_num)
+                if entry.get('href'):
+                    yield self.url_result(
+                        urljoin(url, entry['href']), PolskieRadioLegacyIE, entry_id, entry.get('title'))
+            for a_entry in re.findall(r'<span data-media=({[^ ]+})', content):
+                yield traverse_obj(self._parse_json(a_entry, category_id), {
+                    'url': 'file',
+                    'id': 'uid',
+                    'duration': 'length',
+                    'title': ('title', {urllib.parse.unquote}),
+                    'description': ('desc', {urllib.parse.unquote}),
+                })
+            if is_billennium_tabs:
+                params = self._search_json(
+                    r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+onclick=["\']TB_LoadTab\(',
+                    pagination, 'next page params', category_id, default=None, close_objects=1,
+                    contains_pattern='.+', transform_source=lambda x: '[%s' % js_to_json(unescapeHTML(x)))
+                if not params:
+                    break
+                tab_content = self._download_json(
+                    'https://www.polskieradio.pl/CMS/TemplateBoxesManagement/TemplateBoxTabContent.aspx/GetTabContent',
+                    category_id, f'Downloading page {page_num}', headers={'content-type': 'application/json'},
+                    data=json.dumps(dict(zip((
+                        'boxInstanceId', 'tabId', 'categoryType', 'sectionId', 'categoryId', 'pagerMode',
+                        'subjectIds', 'tagIndexId', 'queryString', 'name', 'openArticlesInParentTemplate',
+                        'idSectionFromUrl', 'maxDocumentAge', 'showCategoryForArticle', 'pageNumber'
+                    ), params))).encode())['d']
+                content, pagination = tab_content['Content'], tab_content.get('PagerContent')
+            elif is_post_back:
+                target = self._search_regex(
+                    r'onclick=(?:["\'])__doPostBack\((?P<q1>["\'])(?P<target>[\w$]+)(?P=q1)\s*,\s*(?P<q2>["\'])Next(?P=q2)',
+                    content, 'pagination postback target', group='target', default=None)
+                if not target:
+                    break
+                content = self._download_webpage(
+                    url, category_id, f'Downloading page {page_num}',
+                    data=urllib.parse.urlencode({
+                        **self._hidden_inputs(content),
+                        '__EVENTTARGET': target,
+                        '__EVENTARGUMENT': 'Next',
+                    }).encode())
+            else:
+                next_url = urljoin(url, self._search_regex(
+                    r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1',
+                    content, 'next page url', group='url', default=None))
+                if not next_url:
+                    break
+                content = self._download_webpage(next_url, category_id, f'Downloading page {page_num}')
 
     def _real_extract(self, url):
         category_id = self._match_id(url)
@@ -337,7 +439,7 @@ def _real_extract(self, url):
         if PolskieRadioAuditionIE.suitable(urlh.url):
             return self.url_result(urlh.url, PolskieRadioAuditionIE, category_id)
         title = self._html_search_regex(
-            r'<title>([^<]+) - [^<]+ - [^<]+</title>',
+            r'<title>([^<]+)(?: - [^<]+ - [^<]+| w [Pp]olskie[Rr]adio\.pl\s*)</title>',
             webpage, 'title', fatal=False)
         return self.playlist_result(
             self._entries(url, webpage, category_id),
@@ -506,39 +608,3 @@ def _real_extract(self, url):
                 'Content-Type': 'application/json',
             })
         return self._parse_episode(data[0])
-
-
-class PolskieRadioRadioKierowcowIE(PolskieRadioBaseExtractor):
-    _VALID_URL = r'https?://(?:www\.)?radiokierowcow\.pl/artykul/(?P<id>[0-9]+)'
-    IE_NAME = 'polskieradio:kierowcow'
-
-    _TESTS = [{
-        'url': 'https://radiokierowcow.pl/artykul/2694529',
-        'info_dict': {
-            'id': '2694529',
-            'title': 'Zielona fala reliktem przeszłości?',
-            'description': 'md5:343950a8717c9818fdfd4bd2b8ca9ff2',
-        },
-        'playlist_count': 3,
-    }]
-
-    def _real_extract(self, url):
-        media_id = self._match_id(url)
-        webpage = self._download_webpage(url, media_id)
-        nextjs_build = self._search_nextjs_data(webpage, media_id)['buildId']
-        article = self._download_json(
-            f'https://radiokierowcow.pl/_next/data/{nextjs_build}/artykul/{media_id}.json?articleId={media_id}',
-            media_id)
-        data = article['pageProps']['data']
-        title = data['title']
-        entries = self._extract_webpage_player_entries(data['content'], media_id, {
-            'title': title,
-        })
-
-        return {
-            '_type': 'playlist',
-            'id': media_id,
-            'entries': entries,
-            'title': title,
-            'description': data.get('lead'),
-        }

From fc5a7f9b27d2a89b1f3ca7d33a95301c21d832cd Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Sun, 28 May 2023 23:31:26 -0400
Subject: [PATCH 289/871] [extractor/daftsex] Update domain and embed player
 url (#5966)

Closes #5881
Authored by: JChris246
---
 yt_dlp/extractor/daftsex.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/yt_dlp/extractor/daftsex.py b/yt_dlp/extractor/daftsex.py
index 551d5e3ab..92510c767 100644
--- a/yt_dlp/extractor/daftsex.py
+++ b/yt_dlp/extractor/daftsex.py
@@ -1,6 +1,7 @@
 from .common import InfoExtractor
 from ..compat import compat_b64decode
 from ..utils import (
+    ExtractorError,
     int_or_none,
     js_to_json,
     parse_count,
@@ -12,21 +13,24 @@
 
 
 class DaftsexIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?daftsex\.com/watch/(?P<id>-?\d+_\d+)'
+    _VALID_URL = r'https?://(?:www\.)?daft\.sex/watch/(?P<id>-?\d+_\d+)'
     _TESTS = [{
-        'url': 'https://daftsex.com/watch/-35370899_456246186',
-        'md5': 'd95135e6cea2d905bea20dbe82cda64a',
+        'url': 'https://daft.sex/watch/-35370899_456246186',
+        'md5': '64c04ef7b4c7b04b308f3b0c78efe7cd',
         'info_dict': {
             'id': '-35370899_456246186',
             'ext': 'mp4',
             'title': 'just relaxing',
-            'description': 'just relaxing - Watch video Watch video in high quality',
+            'description': 'just relaxing – Watch video Watch video in high quality',
             'upload_date': '20201113',
             'timestamp': 1605261911,
-            'thumbnail': r're:https://[^/]+/impf/-43BuMDIawmBGr3GLcZ93CYwWf2PBv_tVWoS1A/dnu41DnARU4\.jpg\?size=800x450&quality=96&keep_aspect_ratio=1&background=000000&sign=6af2c26ff4a45e55334189301c867384&type=video_thumb',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'age_limit': 18,
+            'duration': 15.0,
+            'view_count': int
         },
     }, {
-        'url': 'https://daftsex.com/watch/-156601359_456242791',
+        'url': 'https://daft.sex/watch/-156601359_456242791',
         'info_dict': {
             'id': '-156601359_456242791',
             'ext': 'mp4',
@@ -36,6 +40,7 @@ class DaftsexIE(InfoExtractor):
             'timestamp': 1600250735,
             'thumbnail': 'https://psv153-1.crazycloud.ru/videos/-156601359/456242791/thumb.jpg?extra=i3D32KaBbBFf9TqDRMAVmQ',
         },
+        'skip': 'deleted / private'
     }]
 
     def _real_extract(self, url):
@@ -60,7 +65,7 @@ def _real_extract(self, url):
             webpage, 'player color', fatal=False) or ''
 
         embed_page = self._download_webpage(
-            'https://daxab.com/player/%s?color=%s' % (player_hash, player_color),
+            'https://dxb.to/player/%s?color=%s' % (player_hash, player_color),
             video_id, headers={'Referer': url})
         video_params = self._parse_json(
             self._search_regex(
@@ -94,15 +99,19 @@ def _real_extract(self, url):
                 'age_limit': 18,
             }
 
-        item = self._download_json(
+        items = self._download_json(
             f'{server_domain}/method/video.get/{video_id}', video_id,
             headers={'Referer': url}, query={
                 'token': video_params['video']['access_token'],
                 'videos': video_id,
                 'ckey': video_params['c_key'],
                 'credentials': video_params['video']['credentials'],
-            })['response']['items'][0]
+            })['response']['items']
 
+        if not items:
+            raise ExtractorError('Video is not available', video_id=video_id, expected=True)
+
+        item = items[0]
         formats = []
         for f_id, f_url in item.get('files', {}).items():
             if f_id == 'external':

From aed945e1b9b7d3af2a907e1a12e6508cc81d6a20 Mon Sep 17 00:00:00 2001
From: "lauren n. liberda" <lauren@selfisekai.rocks>
Date: Mon, 29 May 2023 06:07:45 +0200
Subject: [PATCH 290/871] [extractor/wykop] Add extractors (#6140)

Authored by: selfisekai
---
 yt_dlp/extractor/_extractors.py |   6 +
 yt_dlp/extractor/wykop.py       | 268 ++++++++++++++++++++++++++++++++
 2 files changed, 274 insertions(+)
 create mode 100644 yt_dlp/extractor/wykop.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ba55ccbaf..bf041ae61 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2357,6 +2357,12 @@
     WSJArticleIE,
 )
 from .wwe import WWEIE
+from .wykop import (
+    WykopDigIE,
+    WykopDigCommentIE,
+    WykopPostIE,
+    WykopPostCommentIE,
+)
 from .xanimu import XanimuIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
diff --git a/yt_dlp/extractor/wykop.py b/yt_dlp/extractor/wykop.py
new file mode 100644
index 000000000..0fa6d524d
--- /dev/null
+++ b/yt_dlp/extractor/wykop.py
@@ -0,0 +1,268 @@
+import json
+import urllib.error
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    format_field,
+    parse_iso8601,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class WykopBaseExtractor(InfoExtractor):
+    def _get_token(self, force_refresh=False):
+        if not force_refresh:
+            maybe_cached = self.cache.load('wykop', 'bearer')
+            if maybe_cached:
+                return maybe_cached
+
+        new_token = traverse_obj(
+            self._do_call_api('auth', None, 'Downloading anonymous auth token', data={
+                # hardcoded in frontend
+                'key': 'w53947240748',
+                'secret': 'd537d9e0a7adc1510842059ae5316419',
+            }), ('data', 'token'))
+
+        self.cache.store('wykop', 'bearer', new_token)
+        return new_token
+
+    def _do_call_api(self, path, video_id, note='Downloading JSON metadata', data=None, headers={}):
+        if data:
+            data = json.dumps({'data': data}).encode()
+            headers['Content-Type'] = 'application/json'
+
+        return self._download_json(
+            f'https://wykop.pl/api/v3/{path}', video_id,
+            note=note, data=data, headers=headers)
+
+    def _call_api(self, path, video_id, note='Downloading JSON metadata'):
+        token = self._get_token()
+        for retrying in range(2):
+            try:
+                return self._do_call_api(path, video_id, note, headers={'Authorization': f'Bearer {token}'})
+            except ExtractorError as e:
+                if not retrying and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                    token = self._get_token(True)
+                    continue
+                raise
+
+    def _common_data_extract(self, data):
+        author = traverse_obj(data, ('author', 'username'), expected_type=str)
+
+        return {
+            '_type': 'url_transparent',
+            'display_id': data.get('slug'),
+            'url': traverse_obj(data,
+                                ('media', 'embed', 'url'),  # what gets an iframe embed
+                                ('source', 'url'),  # clickable url (dig only)
+                                expected_type=url_or_none),
+            'thumbnail': traverse_obj(
+                data, ('media', 'photo', 'url'), ('media', 'embed', 'thumbnail'), expected_type=url_or_none),
+            'uploader': author,
+            'uploader_id': author,
+            'uploader_url': format_field(author, None, 'https://wykop.pl/ludzie/%s'),
+            'timestamp': parse_iso8601(data.get('created_at'), delimiter=' '),  # time it got submitted
+            'like_count': traverse_obj(data, ('votes', 'up'), expected_type=int),
+            'dislike_count': traverse_obj(data, ('votes', 'down'), expected_type=int),
+            'comment_count': traverse_obj(data, ('comments', 'count'), expected_type=int),
+            'age_limit': 18 if data.get('adult') else 0,
+            'tags': data.get('tags'),
+        }
+
+
+class WykopDigIE(WykopBaseExtractor):
+    IE_NAME = 'wykop:dig'
+    _VALID_URL = r'https?://(?:www\.)?wykop\.pl/link/(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://wykop.pl/link/6912923/najbardziej-zrzedliwy-kot-na-swiecie-i-frozen-planet-ii-i-bbc-earth',
+        'info_dict': {
+            'id': 'rlSTBvViflc',
+            'ext': 'mp4',
+            'title': 'Najbardziej zrzędliwy kot na świecie I Frozen Planet II I BBC Earth',
+            'display_id': 'najbardziej-zrzedliwy-kot-na-swiecie-i-frozen-planet-ii-i-bbc-earth',
+            'description': 'md5:ac0f87dea1cdcb6b0c53f3612a095c87',
+            'tags': ['zwierzaczki', 'koty', 'smiesznykotek', 'humor', 'rozrywka', 'ciekawostki'],
+            'age_limit': 0,
+            'timestamp': 1669154480,
+            'release_timestamp': 1669194241,
+            'release_date': '20221123',
+            'uploader': 'starnak',
+            'uploader_id': 'starnak',
+            'uploader_url': 'https://wykop.pl/ludzie/starnak',
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'thumbnail': r're:https?://wykop\.pl/cdn/.+',
+            'view_count': int,
+            'channel': 'BBC Earth',
+            'channel_id': 'UCwmZiChSryoWQCZMIQezgTg',
+            'channel_url': 'https://www.youtube.com/channel/UCwmZiChSryoWQCZMIQezgTg',
+            'categories': ['Pets & Animals'],
+            'upload_date': '20220923',
+            'duration': 191,
+            'channel_follower_count': int,
+            'availability': 'public',
+            'live_status': 'not_live',
+            'playable_in_embed': True,
+        },
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return cls._match_valid_url(url) and not WykopDigCommentIE.suitable(url)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._call_api(f'links/{video_id}', video_id)['data']
+
+        return {
+            **self._common_data_extract(data),
+            'id': video_id,
+            'title': data['title'],
+            'description': data.get('description'),
+            # time it got "digged" to the homepage
+            'release_timestamp': parse_iso8601(data.get('published_at'), delimiter=' '),
+        }
+
+
+class WykopDigCommentIE(WykopBaseExtractor):
+    IE_NAME = 'wykop:dig:comment'
+    _VALID_URL = r'https?://(?:www\.)?wykop\.pl/link/(?P<dig_id>\d+)/[^/]+/komentarz/(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://wykop.pl/link/6992589/strollowal-oszusta-przez-ponad-24-minuty-udawal-naiwniaka-i-nagral-rozmowe/komentarz/114540527/podobna-sytuacja-ponizej-ciekawa-dyskusja-z-oszustem-na-sam-koniec-sam-bylem-w-biurze-swiadkiem-podobnej-rozmowy-niemal-zakonczonej-sukcesem-bandyty-g',
+        'info_dict': {
+            'id': 'u6tEi2FmKZY',
+            'ext': 'mp4',
+            'title': 'md5:e7c741c5baa7ed6478000caf72865577',
+            'display_id': 'md5:45b2d12bd0e262d09cc7cf7abc8412db',
+            'description': 'md5:bcec7983429f9c0630f9deb9d3d1ba5e',
+            'timestamp': 1674476945,
+            'uploader': 'Bartholomew',
+            'uploader_id': 'Bartholomew',
+            'uploader_url': 'https://wykop.pl/ludzie/Bartholomew',
+            'thumbnail': r're:https?://wykop\.pl/cdn/.+',
+            'tags': [],
+            'availability': 'public',
+            'duration': 1838,
+            'upload_date': '20230117',
+            'categories': ['Entertainment'],
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'channel_follower_count': int,
+            'playable_in_embed': True,
+            'live_status': 'not_live',
+            'age_limit': 0,
+            'chapters': 'count:3',
+            'channel': 'Poszukiwacze Okazji',
+            'channel_id': 'UCzzvJDZThwv06dR4xmzrZBw',
+            'channel_url': 'https://www.youtube.com/channel/UCzzvJDZThwv06dR4xmzrZBw',
+        },
+    }]
+
+    def _real_extract(self, url):
+        dig_id, comment_id = self._search_regex(self._VALID_URL, url, 'dig and comment ids', group=('dig_id', 'id'))
+        data = self._call_api(f'links/{dig_id}/comments/{comment_id}', comment_id)['data']
+
+        return {
+            **self._common_data_extract(data),
+            'id': comment_id,
+            'title': f"{traverse_obj(data, ('author', 'username'))} - {data.get('content') or ''}",
+            'description': data.get('content'),
+        }
+
+
+class WykopPostIE(WykopBaseExtractor):
+    IE_NAME = 'wykop:post'
+    _VALID_URL = r'https?://(?:www\.)?wykop\.pl/wpis/(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://wykop.pl/wpis/68893343/kot-koty-smiesznykotek',
+        'info_dict': {
+            'id': 'PL8JMjiUPHUhwc9ZlKa_5IFeBwBV8Xe7jI',
+            'title': 'PawelW124 - #kot #koty #smiesznykotek',
+            'description': '#kot #koty #smiesznykotek',
+            'display_id': 'kot-koty-smiesznykotek',
+            'tags': ['kot', 'koty', 'smiesznykotek'],
+            'uploader': 'PawelW124',
+            'uploader_id': 'PawelW124',
+            'uploader_url': 'https://wykop.pl/ludzie/PawelW124',
+            'timestamp': 1668938142,
+            'age_limit': 0,
+            'like_count': int,
+            'dislike_count': int,
+            'thumbnail': r're:https?://wykop\.pl/cdn/.+',
+            'comment_count': int,
+            'channel': 'Revan',
+            'channel_id': 'UCW9T_-uZoiI7ROARQdTDyOw',
+            'channel_url': 'https://www.youtube.com/channel/UCW9T_-uZoiI7ROARQdTDyOw',
+            'upload_date': '20221120',
+            'modified_date': '20220814',
+            'availability': 'public',
+            'view_count': int,
+        },
+        'playlist_mincount': 15,
+        'params': {
+            'flat_playlist': True,
+        }
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return cls._match_valid_url(url) and not WykopPostCommentIE.suitable(url)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._call_api(f'entries/{video_id}', video_id)['data']
+
+        return {
+            **self._common_data_extract(data),
+            'id': video_id,
+            'title': f"{traverse_obj(data, ('author', 'username'))} - {data.get('content') or ''}",
+            'description': data.get('content'),
+        }
+
+
+class WykopPostCommentIE(WykopBaseExtractor):
+    IE_NAME = 'wykop:post:comment'
+    _VALID_URL = r'https?://(?:www\.)?wykop\.pl/wpis/(?P<post_id>\d+)/[^/#]+#(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://wykop.pl/wpis/70084873/test-test-test#249303979',
+        'info_dict': {
+            'id': 'confusedquickarmyant',
+            'ext': 'mp4',
+            'title': 'tpap - treść komentarza',
+            'display_id': 'tresc-komentarza',
+            'description': 'treść komentarza',
+            'uploader': 'tpap',
+            'uploader_id': 'tpap',
+            'uploader_url': 'https://wykop.pl/ludzie/tpap',
+            'timestamp': 1675349470,
+            'upload_date': '20230202',
+            'tags': [],
+            'duration': 2.12,
+            'age_limit': 0,
+            'categories': [],
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'thumbnail': r're:https?://wykop\.pl/cdn/.+',
+        },
+    }]
+
+    def _real_extract(self, url):
+        post_id, comment_id = self._search_regex(self._VALID_URL, url, 'post and comment ids', group=('post_id', 'id'))
+        data = self._call_api(f'entries/{post_id}/comments/{comment_id}', comment_id)['data']
+
+        return {
+            **self._common_data_extract(data),
+            'id': comment_id,
+            'title': f"{traverse_obj(data, ('author', 'username'))} - {data.get('content') or ''}",
+            'description': data.get('content'),
+        }

From c6d3f81a4077aaf9cffc6aa2d0dec92f38e74bb0 Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Mon, 29 May 2023 06:20:03 +0200
Subject: [PATCH 291/871] [extractor/rai] Rewrite extractors (#5940)

Authored by: nixxo, danog
Closes #5672, closes #6341

Co-authored-by: Daniil Gentili <daniil@daniil.it>
---
 yt_dlp/extractor/_extractors.py |   3 +-
 yt_dlp/extractor/rai.py         | 575 +++++++++++++++-----------------
 2 files changed, 271 insertions(+), 307 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index bf041ae61..d9028a831 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1548,6 +1548,8 @@
     RadLiveSeasonIE,
 )
 from .rai import (
+    RaiIE,
+    RaiCulturaIE,
     RaiPlayIE,
     RaiPlayLiveIE,
     RaiPlayPlaylistIE,
@@ -1556,7 +1558,6 @@
     RaiPlaySoundPlaylistIE,
     RaiNewsIE,
     RaiSudtirolIE,
-    RaiIE,
 )
 from .raywenderlich import (
     RayWenderlichIE,
diff --git a/yt_dlp/extractor/rai.py b/yt_dlp/extractor/rai.py
index cab12cc21..df4102a40 100644
--- a/yt_dlp/extractor/rai.py
+++ b/yt_dlp/extractor/rai.py
@@ -1,19 +1,12 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urlparse,
-)
 from ..utils import (
     clean_html,
     determine_ext,
     ExtractorError,
     filter_dict,
-    find_xpath_attr,
-    fix_xml_ampersands,
     GeoRestrictedError,
-    HEADRequest,
     int_or_none,
     join_nonempty,
     parse_duration,
@@ -35,82 +28,70 @@ class RaiBaseIE(InfoExtractor):
     _GEO_BYPASS = False
 
     def _extract_relinker_info(self, relinker_url, video_id, audio_only=False):
+        def fix_cdata(s):
+            # remove \r\n\t before and after <![CDATA[ ]]> to avoid
+            # polluted text with xpath_text
+            s = re.sub(r'(\]\]>)[\r\n\t]+(</)', '\\1\\2', s)
+            return re.sub(r'(>)[\r\n\t]+(<!\[CDATA\[)', '\\1\\2', s)
+
         if not re.match(r'https?://', relinker_url):
             return {'formats': [{'url': relinker_url}]}
 
+        # set User-Agent to generic 'Rai' to avoid quality filtering from
+        # the media server and get the maximum qualities available
+        relinker = self._download_xml(
+            relinker_url, video_id, note='Downloading XML metadata',
+            transform_source=fix_cdata, query={'output': 64},
+            headers={**self.geo_verification_headers(), 'User-Agent': 'Rai'})
+
+        if xpath_text(relinker, './license_url', default='{}') != '{}':
+            self.report_drm(video_id)
+
+        is_live = xpath_text(relinker, './is_live', default='N') == 'Y'
+        duration = parse_duration(xpath_text(relinker, './duration', default=None))
+        media_url = xpath_text(relinker, './url[@type="content"]', default=None)
+
+        if not media_url:
+            self.raise_no_formats('The relinker returned no media url')
+
+        # geo flag is a bit unreliable and not properly set all the time
+        geoprotection = xpath_text(relinker, './geoprotection', default='N') == 'Y'
+
+        ext = determine_ext(media_url)
         formats = []
-        geoprotection = None
-        is_live = None
-        duration = None
 
-        for platform in ('mon', 'flash', 'native'):
-            relinker = self._download_xml(
-                relinker_url, video_id,
-                note=f'Downloading XML metadata for platform {platform}',
-                transform_source=fix_xml_ampersands,
-                query={'output': 45, 'pl': platform},
-                headers=self.geo_verification_headers())
+        if ext == 'mp3':
+            formats.append({
+                'url': media_url,
+                'vcodec': 'none',
+                'acodec': 'mp3',
+                'format_id': 'https-mp3',
+            })
+        elif ext == 'm3u8' or 'format=m3u8' in media_url:
+            formats.extend(self._extract_m3u8_formats(
+                media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+        elif ext == 'f4m':
+            # very likely no longer needed. Cannot find any url that uses it.
+            manifest_url = update_url_query(
+                media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
+                {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
+            formats.extend(self._extract_f4m_formats(
+                manifest_url, video_id, f4m_id='hds', fatal=False))
+        elif ext == 'mp4':
+            bitrate = int_or_none(xpath_text(relinker, './bitrate'))
+            formats.append({
+                'url': media_url,
+                'tbr': bitrate if bitrate > 0 else None,
+                'format_id': join_nonempty('https', bitrate, delim='-'),
+            })
+        else:
+            raise ExtractorError('Unrecognized media file found')
 
-            if xpath_text(relinker, './license_url', default='{}') != '{}':
-                self.report_drm(video_id)
-
-            if not geoprotection:
-                geoprotection = xpath_text(
-                    relinker, './geoprotection', default=None) == 'Y'
-
-            if not is_live:
-                is_live = xpath_text(
-                    relinker, './is_live', default=None) == 'Y'
-            if not duration:
-                duration = parse_duration(xpath_text(
-                    relinker, './duration', default=None))
-
-            url_elem = find_xpath_attr(relinker, './url', 'type', 'content')
-            if url_elem is None:
-                continue
-
-            media_url = url_elem.text
-
-            # This does not imply geo restriction (e.g.
-            # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html)
-            if '/video_no_available.mp4' in media_url:
-                continue
-
-            ext = determine_ext(media_url)
-            if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
-                continue
-
-            if ext == 'mp3':
-                formats.append({
-                    'url': media_url,
-                    'vcodec': 'none',
-                    'acodec': 'mp3',
-                    'format_id': 'http-mp3',
-                })
-                break
-            elif ext == 'm3u8' or 'format=m3u8' in media_url or platform == 'mon':
-                formats.extend(self._extract_m3u8_formats(
-                    media_url, video_id, 'mp4', 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-            elif ext == 'f4m' or platform == 'flash':
-                manifest_url = update_url_query(
-                    media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
-                    {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
-                formats.extend(self._extract_f4m_formats(
-                    manifest_url, video_id, f4m_id='hds', fatal=False))
-            else:
-                bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
-                formats.append({
-                    'url': media_url,
-                    'tbr': bitrate if bitrate > 0 else None,
-                    'format_id': f'http-{bitrate if bitrate > 0 else "http"}',
-                })
-
-        if not formats and geoprotection is True:
+        if (not formats and geoprotection is True) or '/video_no_available.mp4' in media_url:
             self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
 
-        if not audio_only:
-            formats.extend(self._create_http_urls(relinker_url, formats))
+        if not audio_only and not is_live:
+            formats.extend(self._create_http_urls(media_url, relinker_url, formats))
 
         return filter_dict({
             'is_live': is_live,
@@ -118,38 +99,31 @@ def _extract_relinker_info(self, relinker_url, video_id, audio_only=False):
             'formats': formats,
         })
 
-    def _create_http_urls(self, relinker_url, fmts):
-        _RELINKER_REG = r'https?://(?P<host>[^/]+?)/(?:i/)?(?P<extra>[^/]+?)/(?P<path>.+?)/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4|/playlist\.m3u8).+?'
+    def _create_http_urls(self, manifest_url, relinker_url, fmts):
+        _MANIFEST_REG = r'/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4)?(?:\.csmil)?/playlist\.m3u8'
         _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
         _QUALITY = {
             # tbr: w, h
-            '250': [352, 198],
-            '400': [512, 288],
-            '700': [512, 288],
-            '800': [700, 394],
-            '1200': [736, 414],
-            '1800': [1024, 576],
-            '2400': [1280, 720],
-            '3200': [1440, 810],
-            '3600': [1440, 810],
-            '5000': [1920, 1080],
-            '10000': [1920, 1080],
+            250: [352, 198],
+            400: [512, 288],
+            600: [512, 288],
+            700: [512, 288],
+            800: [700, 394],
+            1200: [736, 414],
+            1500: [920, 518],
+            1800: [1024, 576],
+            2400: [1280, 720],
+            3200: [1440, 810],
+            3600: [1440, 810],
+            5000: [1920, 1080],
+            10000: [1920, 1080],
         }
 
-        def test_url(url):
-            resp = self._request_webpage(
-                HEADRequest(url), None, headers={'User-Agent': 'Rai'},
-                fatal=False, errnote=False, note=False)
-
-            if resp is False:
+        def percentage(number, target, pc=20, roof=125):
+            '''check if the target is in the range of number +/- percent'''
+            if not number or number < 0:
                 return False
-
-            if resp.code == 200:
-                return False if resp.url == url else resp.url
-            return None
-
-        # filter out audio-only formats
-        fmts = [f for f in fmts if not f.get('vcodec') == 'none']
+            return abs(target - number) < min(float(number) * float(pc) / 100.0, roof)
 
         def get_format_info(tbr):
             import math
@@ -157,67 +131,78 @@ def get_format_info(tbr):
             if len(fmts) == 1 and not br:
                 br = fmts[0].get('tbr')
             if br and br > 300:
-                tbr = compat_str(math.floor(br / 100) * 100)
+                tbr = math.floor(br / 100) * 100
             else:
-                tbr = '250'
+                tbr = 250
 
             # try extracting info from available m3u8 formats
-            format_copy = None
+            format_copy = [None, None]
             for f in fmts:
                 if f.get('tbr'):
-                    br_limit = math.floor(br / 100)
-                    if br_limit - 1 <= math.floor(f['tbr'] / 100) <= br_limit + 1:
-                        format_copy = f.copy()
+                    if percentage(tbr, f['tbr']):
+                        format_copy[0] = f.copy()
+                if [f.get('width'), f.get('height')] == _QUALITY.get(tbr):
+                    format_copy[1] = f.copy()
+                    format_copy[1]['tbr'] = tbr
+
+            # prefer format with similar bitrate because there might be
+            # multiple video with the same resolution but different bitrate
+            format_copy = format_copy[0] or format_copy[1] or {}
             return {
+                'format_id': f'https-{tbr}',
                 'width': format_copy.get('width'),
                 'height': format_copy.get('height'),
                 'tbr': format_copy.get('tbr'),
                 'vcodec': format_copy.get('vcodec'),
                 'acodec': format_copy.get('acodec'),
                 'fps': format_copy.get('fps'),
-                'format_id': f'https-{tbr}',
             } if format_copy else {
+                'format_id': f'https-{tbr}',
                 'width': _QUALITY[tbr][0],
                 'height': _QUALITY[tbr][1],
-                'format_id': f'https-{tbr}',
-                'tbr': int(tbr),
+                'tbr': tbr,
+                'vcodec': 'avc1',
+                'acodec': 'mp4a',
+                'fps': 25,
             }
 
-        loc = test_url(_MP4_TMPL % (relinker_url, '*'))
-        if not isinstance(loc, compat_str):
-            return []
+        # filter out single-stream formats
+        fmts = [f for f in fmts
+                if not f.get('vcodec') == 'none' and not f.get('acodec') == 'none']
 
-        mobj = re.match(
-            _RELINKER_REG,
-            test_url(relinker_url) or '')
+        mobj = re.search(_MANIFEST_REG, manifest_url)
         if not mobj:
             return []
-
         available_qualities = mobj.group('quality').split(',') if mobj.group('quality') else ['*']
-        available_qualities = [i for i in available_qualities if i]
 
         formats = []
-        for q in available_qualities:
-            fmt = {
+        for q in filter(None, available_qualities):
+            self.write_debug(f'Creating https format for quality {q}')
+            formats.append({
                 'url': _MP4_TMPL % (relinker_url, q),
                 'protocol': 'https',
                 'ext': 'mp4',
                 **get_format_info(q)
-            }
-            formats.append(fmt)
+            })
         return formats
 
+    @staticmethod
+    def _get_thumbnails_list(thumbs, url):
+        return [{
+            'url': urljoin(url, thumb_url),
+        } for thumb_url in (thumbs or {}).values() if thumb_url]
+
     @staticmethod
     def _extract_subtitles(url, video_data):
         STL_EXT = 'stl'
         SRT_EXT = 'srt'
         subtitles = {}
-        subtitles_array = video_data.get('subtitlesArray') or []
+        subtitles_array = video_data.get('subtitlesArray') or video_data.get('subtitleList') or []
         for k in ('subtitles', 'subtitlesUrl'):
             subtitles_array.append({'url': video_data.get(k)})
         for subtitle in subtitles_array:
             sub_url = subtitle.get('url')
-            if sub_url and isinstance(sub_url, compat_str):
+            if sub_url and isinstance(sub_url, str):
                 sub_lang = subtitle.get('language') or 'it'
                 sub_url = urljoin(url, sub_url)
                 sub_ext = determine_ext(sub_url, SRT_EXT)
@@ -236,7 +221,7 @@ def _extract_subtitles(url, video_data):
 class RaiPlayIE(RaiBaseIE):
     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
     _TESTS = [{
-        'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
+        'url': 'https://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
         'info_dict': {
             'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
@@ -244,22 +229,20 @@ class RaiPlayIE(RaiBaseIE):
             'title': 'Report del 07/04/2014',
             'alt_title': 'St 2013/14 - Report - Espresso nel caffè - 07/04/2014',
             'description': 'md5:d730c168a58f4bb35600fc2f881ec04e',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'Rai Gulp',
+            'thumbnail': r're:^https?://www\.raiplay\.it/.+\.jpg',
+            'uploader': 'Rai 3',
+            'creator': 'Rai 3',
             'duration': 6160,
             'series': 'Report',
             'season': '2013/14',
-            'subtitles': {
-                'it': 'count:4',
-            },
+            'subtitles': {'it': 'count:4'},
             'release_year': 2022,
             'episode': 'Espresso nel caffè - 07/04/2014',
             'timestamp': 1396919880,
             'upload_date': '20140408',
+            'formats': 'count:4',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'params': {'skip_download': True},
     }, {
         # 1080p direct mp4 url
         'url': 'https://www.raiplay.it/video/2021/11/Blanca-S1E1-Senza-occhi-b1255a4a-8e72-4a2f-b9f3-fc1308e00736.html',
@@ -270,8 +253,9 @@ class RaiPlayIE(RaiBaseIE):
             'title': 'Blanca - S1E1 - Senza occhi',
             'alt_title': 'St 1 Ep 1 - Blanca - Senza occhi',
             'description': 'md5:75f95d5c030ec8bac263b1212322e28c',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'Rai 1',
+            'thumbnail': r're:^https://www\.raiplay\.it/dl/img/.+\.jpg',
+            'uploader': 'Rai Premium',
+            'creator': 'Rai Fiction',
             'duration': 6493,
             'series': 'Blanca',
             'season': 'Season 1',
@@ -281,6 +265,30 @@ class RaiPlayIE(RaiBaseIE):
             'episode': 'Senza occhi',
             'timestamp': 1637318940,
             'upload_date': '20211119',
+            'formats': 'count:12',
+        },
+        'params': {'skip_download': True},
+        'expected_warnings': ['Video not available. Likely due to geo-restriction.']
+    }, {
+        # 1500 quality
+        'url': 'https://www.raiplay.it/video/2012/09/S1E11---Tutto-cio-che-luccica-0cab3323-732e-45d6-8e86-7704acab6598.html',
+        'md5': 'a634d20e8ab2d43724c273563f6bf87a',
+        'info_dict': {
+            'id': '0cab3323-732e-45d6-8e86-7704acab6598',
+            'ext': 'mp4',
+            'title': 'Mia and Me - S1E11 - Tutto ciò che luccica',
+            'alt_title': 'St 1 Ep 11 - Mia and Me - Tutto ciò che luccica',
+            'description': 'md5:4969e594184b1920c4c1f2b704da9dea',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'Rai Gulp',
+            'series': 'Mia and Me',
+            'season': 'Season 1',
+            'episode_number': 11,
+            'release_year': 2015,
+            'season_number': 1,
+            'episode': 'Tutto ciò che luccica',
+            'timestamp': 1348495020,
+            'upload_date': '20120924',
         },
     }, {
         'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
@@ -299,57 +307,40 @@ def _real_extract(self, url):
         base, video_id = self._match_valid_url(url).groups()
 
         media = self._download_json(
-            base + '.json', video_id, 'Downloading video JSON')
+            f'{base}.json', video_id, 'Downloading video JSON')
 
         if not self.get_param('allow_unplayable_formats'):
-            if try_get(
-                    media,
-                    (lambda x: x['rights_management']['rights']['drm'],
-                     lambda x: x['program_info']['rights_management']['rights']['drm']),
-                    dict):
+            if traverse_obj(media, (('program_info', None), 'rights_management', 'rights', 'drm')):
                 self.report_drm(video_id)
 
-        title = media['name']
         video = media['video']
-
         relinker_info = self._extract_relinker_info(video['content_url'], video_id)
-
-        thumbnails = []
-        for _, value in media.get('images', {}).items():
-            if value:
-                thumbnails.append({
-                    'url': urljoin(url, value),
-                })
-
-        date_published = media.get('date_published')
-        time_published = media.get('time_published')
-        if date_published and time_published:
-            date_published += ' ' + time_published
-
-        subtitles = self._extract_subtitles(url, video)
-
-        program_info = media.get('program_info') or {}
+        date_published = join_nonempty(
+            media.get('date_published'), media.get('time_published'), delim=' ')
         season = media.get('season')
-
         alt_title = join_nonempty(media.get('subtitle'), media.get('toptitle'), delim=' - ')
 
         return {
             'id': remove_start(media.get('id'), 'ContentItem-') or video_id,
             'display_id': video_id,
-            'title': title,
+            'title': media.get('name'),
             'alt_title': strip_or_none(alt_title or None),
             'description': media.get('description'),
-            'uploader': strip_or_none(media.get('channel') or None),
-            'creator': strip_or_none(media.get('editor') or None),
+            'uploader': strip_or_none(
+                traverse_obj(media, ('program_info', 'channel'))
+                or media.get('channel') or None),
+            'creator': strip_or_none(
+                traverse_obj(media, ('program_info', 'editor'))
+                or media.get('editor') or None),
             'duration': parse_duration(video.get('duration')),
             'timestamp': unified_timestamp(date_published),
-            'thumbnails': thumbnails,
-            'series': program_info.get('name'),
+            'thumbnails': self._get_thumbnails_list(media.get('images'), url),
+            'series': traverse_obj(media, ('program_info', 'name')),
             'season_number': int_or_none(season),
             'season': season if (season and not season.isdigit()) else None,
             'episode': media.get('episode_title'),
             'episode_number': int_or_none(media.get('episode')),
-            'subtitles': subtitles,
+            'subtitles': self._extract_subtitles(url, video),
             'release_year': int_or_none(traverse_obj(media, ('track_info', 'edit_year'))),
             **relinker_info
         }
@@ -371,38 +362,39 @@ class RaiPlayLiveIE(RaiPlayIE):  # XXX: Do not subclass from concrete IE
             'live_status': 'is_live',
             'upload_date': '20090502',
             'timestamp': 1241276220,
+            'formats': 'count:3',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'params': {'skip_download': True},
     }]
 
 
 class RaiPlayPlaylistIE(InfoExtractor):
     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
     _TESTS = [{
+        # entire series episodes + extras...
         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/',
         'info_dict': {
             'id': 'nondirloalmiocapo',
             'title': 'Non dirlo al mio capo',
             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
         },
-        'playlist_mincount': 12,
+        'playlist_mincount': 30,
     }, {
+        # single season
         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/episodi/stagione-2/',
         'info_dict': {
             'id': 'nondirloalmiocapo',
             'title': 'Non dirlo al mio capo - Stagione 2',
             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
         },
-        'playlist_mincount': 12,
+        'playlist_count': 12,
     }]
 
     def _real_extract(self, url):
         base, playlist_id, extra_id = self._match_valid_url(url).groups()
 
         program = self._download_json(
-            base + '.json', playlist_id, 'Downloading program JSON')
+            f'{base}.json', playlist_id, 'Downloading program JSON')
 
         if extra_id:
             extra_id = extra_id.upper().rstrip('/')
@@ -450,7 +442,7 @@ class RaiPlaySoundIE(RaiBaseIE):
             'title': 'Il Ruggito del Coniglio del 10/12/2021',
             'alt_title': 'md5:0e6476cd57858bb0f3fcc835d305b455',
             'description': 'md5:2a17d2107e59a4a8faa0e18334139ee2',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.+\.jpg$',
             'uploader': 'rai radio 2',
             'duration': 5685,
             'series': 'Il Ruggito del Coniglio',
@@ -459,9 +451,7 @@ class RaiPlaySoundIE(RaiBaseIE):
             'timestamp': 1638346620,
             'upload_date': '20211201',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'params': {'skip_download': True},
     }]
 
     def _real_extract(self, url):
@@ -480,9 +470,6 @@ def _real_extract(self, url):
                                          lambda x: x['live']['create_date']))
 
         podcast_info = traverse_obj(media, 'podcast_info', ('live', 'cards', 0)) or {}
-        thumbnails = [{
-            'url': urljoin(url, thumb_url),
-        } for thumb_url in (podcast_info.get('images') or {}).values() if thumb_url]
 
         return {
             **info,
@@ -494,7 +481,7 @@ def _real_extract(self, url):
             'uploader': traverse_obj(media, ('track_info', 'channel'), expected_type=strip_or_none),
             'creator': traverse_obj(media, ('track_info', 'editor'), expected_type=strip_or_none),
             'timestamp': unified_timestamp(date_published),
-            'thumbnails': thumbnails,
+            'thumbnails': self._get_thumbnails_list(podcast_info.get('images'), url),
             'series': podcast_info.get('title'),
             'season_number': int_or_none(media.get('season')),
             'episode': media.get('episode_title'),
@@ -512,30 +499,30 @@ class RaiPlaySoundLiveIE(RaiPlaySoundIE):  # XXX: Do not subclass from concrete
             'display_id': 'radio2',
             'ext': 'mp4',
             'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+',
-            'thumbnail': r're:https://www.raiplaysound.it/dl/img/.+?png',
+            'thumbnail': r're:^https://www\.raiplaysound\.it/dl/img/.+\.png',
             'uploader': 'rai radio 2',
             'series': 'Rai Radio 2',
             'creator': 'raiplaysound',
             'is_live': True,
             'live_status': 'is_live',
         },
-        'params': {
-            'skip_download': 'live',
-        },
+        'params': {'skip_download': True},
     }]
 
 
 class RaiPlaySoundPlaylistIE(InfoExtractor):
     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?:programmi|playlist|audiolibri)/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
     _TESTS = [{
+        # entire show
         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio',
         'info_dict': {
             'id': 'ilruggitodelconiglio',
             'title': 'Il Ruggito del Coniglio',
-            'description': 'md5:1bbaf631245a7ab1ec4d9fbb3c7aa8f3',
+            'description': 'md5:48cff6972435964284614d70474132e6',
         },
         'playlist_mincount': 65,
     }, {
+        # single season
         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio/puntate/prima-stagione-1995',
         'info_dict': {
             'id': 'ilruggitodelconiglio_puntate_prima-stagione-1995',
@@ -568,22 +555,19 @@ def _real_extract(self, url):
 class RaiIE(RaiBaseIE):
     _VALID_URL = rf'https?://[^/]+\.(?:rai\.(?:it|tv))/.+?-(?P<id>{RaiBaseIE._UUID_RE})(?:-.+?)?\.html'
     _TESTS = [{
-        # var uniquename = "ContentItem-..."
-        # data-id="ContentItem-..."
         'url': 'https://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
         'info_dict': {
             'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
             'ext': 'mp4',
             'title': 'TG PRIMO TEMPO',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg',
             'duration': 1758,
             'upload_date': '20140612',
         },
-        'skip': 'This content is available only in Italy',
+        'params': {'skip_download': True},
+        'expected_warnings': ['Video not available. Likely due to geo-restriction.']
     }, {
-        # with ContentItem in og:url
         'url': 'https://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
-        'md5': '06345bd97c932f19ffb129973d07a020',
         'info_dict': {
             'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
             'ext': 'mp4',
@@ -592,123 +576,51 @@ class RaiIE(RaiBaseIE):
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 2214,
             'upload_date': '20161103'
-        }
+        },
+        'params': {'skip_download': True},
     }, {
-        # Direct MMS URL
+        # Direct MMS: Media URL no longer works.
         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html',
         'only_matching': True,
     }]
 
-    def _extract_from_content_id(self, content_id, url):
+    def _real_extract(self, url):
+        content_id = self._match_id(url)
         media = self._download_json(
             f'https://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-{content_id}.html?json',
-            content_id, 'Downloading video JSON')
+            content_id, 'Downloading video JSON', fatal=False, expected_status=404)
 
-        title = media['name'].strip()
+        if media is None:
+            return None
 
-        media_type = media['type']
-        if 'Audio' in media_type:
+        if 'Audio' in media['type']:
             relinker_info = {
                 'formats': [{
-                    'format_id': media.get('formatoAudio'),
+                    'format_id': join_nonempty('https', media.get('formatoAudio'), delim='-'),
                     'url': media['audioUrl'],
                     'ext': media.get('formatoAudio'),
+                    'vcodec': 'none',
+                    'acodec': media.get('formatoAudio'),
                 }]
             }
-        elif 'Video' in media_type:
+        elif 'Video' in media['type']:
             relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)
         else:
             raise ExtractorError('not a media file')
 
-        thumbnails = []
-        for image_type in ('image', 'image_medium', 'image_300'):
-            thumbnail_url = media.get(image_type)
-            if thumbnail_url:
-                thumbnails.append({
-                    'url': compat_urlparse.urljoin(url, thumbnail_url),
-                })
-
-        subtitles = self._extract_subtitles(url, media)
+        thumbnails = self._get_thumbnails_list(
+            {image_type: media.get(image_type) for image_type in (
+                'image', 'image_medium', 'image_300')}, url)
 
         return {
             'id': content_id,
-            'title': title,
-            'description': strip_or_none(media.get('desc') or None),
+            'title': strip_or_none(media.get('name') or media.get('title')),
+            'description': strip_or_none(media.get('desc')) or None,
             'thumbnails': thumbnails,
-            'uploader': strip_or_none(media.get('author') or None),
+            'uploader': strip_or_none(media.get('author')) or None,
             'upload_date': unified_strdate(media.get('date')),
             'duration': parse_duration(media.get('length')),
-            'subtitles': subtitles,
-            **relinker_info
-        }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        content_item_id = None
-
-        content_item_url = self._html_search_meta(
-            ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url',
-             'twitter:player', 'jsonlink'), webpage, default=None)
-        if content_item_url:
-            content_item_id = self._search_regex(
-                rf'ContentItem-({self._UUID_RE})', content_item_url,
-                'content item id', default=None)
-
-        if not content_item_id:
-            content_item_id = self._search_regex(
-                rf'''(?x)
-                    (?:
-                        (?:initEdizione|drawMediaRaiTV)\(|
-                        <(?:[^>]+\bdata-id|var\s+uniquename)=|
-                        <iframe[^>]+\bsrc=
-                    )
-                    (["\'])
-                    (?:(?!\1).)*\bContentItem-(?P<id>{self._UUID_RE})
-                ''',
-                webpage, 'content item id', default=None, group='id')
-
-        content_item_ids = set()
-        if content_item_id:
-            content_item_ids.add(content_item_id)
-        if video_id not in content_item_ids:
-            content_item_ids.add(video_id)
-
-        for content_item_id in content_item_ids:
-            try:
-                return self._extract_from_content_id(content_item_id, url)
-            except GeoRestrictedError:
-                raise
-            except ExtractorError:
-                pass
-
-        relinker_url = self._proto_relative_url(self._search_regex(
-            r'''(?x)
-                (?:
-                    var\s+videoURL|
-                    mediaInfo\.mediaUri
-                )\s*=\s*
-                ([\'"])
-                (?P<url>
-                    (?:https?:)?
-                    //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\?
-                    (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1
-            ''',
-            webpage, 'relinker URL', group='url'))
-
-        relinker_info = self._extract_relinker_info(
-            urljoin(url, relinker_url), video_id)
-
-        title = self._search_regex(
-            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
-            webpage, 'title', group='title',
-            default=None) or self._og_search_title(webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
+            'subtitles': self._extract_subtitles(url, media),
             **relinker_info
         }
 
@@ -726,7 +638,8 @@ class RaiNewsIE(RaiIE):  # XXX: Do not subclass from concrete IE
             'duration': 1589,
             'upload_date': '20220529',
             'uploader': 'rainews',
-        }
+        },
+        'params': {'skip_download': True},
     }, {
         # old content with fallback method to extract media urls
         'url': 'https://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
@@ -739,12 +652,14 @@ class RaiNewsIE(RaiIE):  # XXX: Do not subclass from concrete IE
             'duration': 833,
             'upload_date': '20161103'
         },
+        'params': {'skip_download': True},
         'expected_warnings': ['unable to extract player_data'],
     }, {
         # iframe + drm
         'url': 'https://www.rainews.it/iframe/video/2022/07/euro2022-europei-calcio-femminile-italia-belgio-gol-0-1-video-4de06a69-de75-4e32-a657-02f0885f8118.html',
         'only_matching': True,
     }]
+    _PLAYER_TAG = 'news'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -752,8 +667,8 @@ def _real_extract(self, url):
         webpage = self._download_webpage(url, video_id)
 
         player_data = self._search_json(
-            r'<rainews-player\s*data=\'', webpage, 'player_data', video_id,
-            transform_source=clean_html, fatal=False)
+            rf'<rai{self._PLAYER_TAG}-player\s*data=\'', webpage, 'player_data', video_id,
+            transform_source=clean_html, default={})
         track_info = player_data.get('track_info')
         relinker_url = traverse_obj(player_data, 'mediapolis', 'content_url')
 
@@ -770,16 +685,36 @@ def _real_extract(self, url):
 
         return {
             'id': video_id,
-            'title': track_info.get('title') or self._og_search_title(webpage),
+            'title': player_data.get('title') or track_info.get('title') or self._og_search_title(webpage),
             'upload_date': unified_strdate(track_info.get('date')),
             'uploader': strip_or_none(track_info.get('editor') or None),
             **relinker_info
         }
 
 
-class RaiSudtirolIE(RaiBaseIE):
-    _VALID_URL = r'https?://raisudtirol\.rai\.it/.+?media=(?P<id>[TP]tv\d+)'
+class RaiCulturaIE(RaiNewsIE):  # XXX: Do not subclass from concrete IE
+    _VALID_URL = rf'https?://(www\.)?raicultura\.it/(?!articoli)[^?#]+-(?P<id>{RaiBaseIE._UUID_RE})(?:-[^/?#]+)?\.html'
+    _EMBED_REGEX = [rf'<iframe[^>]+data-src="(?P<url>/iframe/[^?#]+?{RaiBaseIE._UUID_RE}\.html)']
     _TESTS = [{
+        'url': 'https://www.raicultura.it/letteratura/articoli/2018/12/Alberto-Asor-Rosa-Letteratura-e-potere-05ba8775-82b5-45c5-a89d-dd955fbde1fb.html',
+        'info_dict': {
+            'id': '05ba8775-82b5-45c5-a89d-dd955fbde1fb',
+            'ext': 'mp4',
+            'title': 'Alberto Asor Rosa: Letteratura e potere',
+            'duration': 1756,
+            'upload_date': '20181206',
+            'uploader': 'raicultura',
+            'formats': 'count:2',
+        },
+        'params': {'skip_download': True},
+    }]
+    _PLAYER_TAG = 'cultura'
+
+
+class RaiSudtirolIE(RaiBaseIE):
+    _VALID_URL = r'https?://raisudtirol\.rai\.it/.+media=(?P<id>\w+)'
+    _TESTS = [{
+        # mp4 file
         'url': 'https://raisudtirol.rai.it/la/index.php?media=Ptv1619729460',
         'info_dict': {
             'id': 'Ptv1619729460',
@@ -787,34 +722,62 @@ class RaiSudtirolIE(RaiBaseIE):
             'title': 'Euro: trasmisciun d\'economia - 29-04-2021 20:51',
             'series': 'Euro: trasmisciun d\'economia',
             'upload_date': '20210429',
-            'thumbnail': r're:https://raisudtirol\.rai\.it/img/.+?\.jpg',
+            'thumbnail': r're:https://raisudtirol\.rai\.it/img/.+\.jpg',
             'uploader': 'raisudtirol',
-        }
+            'formats': 'count:1',
+        },
+        'params': {'skip_download': True},
+    }, {
+        # m3u manifest
+        'url': 'https://raisudtirol.rai.it/it/kidsplayer.php?lang=it&media=GUGGUG_P1.smil',
+        'info_dict': {
+            'id': 'GUGGUG_P1',
+            'ext': 'mp4',
+            'title': 'GUGGUG! La Prospettiva - Die Perspektive',
+            'uploader': 'raisudtirol',
+            'formats': 'count:6',
+        },
+        'params': {'skip_download': True},
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_date = self._html_search_regex(r'<span class="med_data">(.+?)</span>', webpage, 'video_date', fatal=False)
-        video_title = self._html_search_regex(r'<span class="med_title">(.+?)</span>', webpage, 'video_title', fatal=False)
-        video_url = self._html_search_regex(r'sources:\s*\[\{file:\s*"(.+?)"\}\]', webpage, 'video_url')
-        video_thumb = self._html_search_regex(r'image: \'(.+?)\'', webpage, 'video_thumb', fatal=False)
+        video_date = self._html_search_regex(
+            r'<span class="med_data">(.+?)</span>', webpage, 'video_date', default=None)
+        video_title = self._html_search_regex([
+            r'<span class="med_title">(.+?)</span>', r'title: \'(.+?)\','],
+            webpage, 'video_title', default=None)
+        video_url = self._html_search_regex([
+            r'sources:\s*\[\{file:\s*"(.+?)"\}\]',
+            r'<source\s+src="(.+?)"\s+type="application/x-mpegURL"'],
+            webpage, 'video_url', default=None)
 
-        return {
-            'id': video_id,
-            'title': join_nonempty(video_title, video_date, delim=' - '),
-            'series': video_title,
-            'upload_date': unified_strdate(video_date),
-            'thumbnail': urljoin('https://raisudtirol.rai.it/', video_thumb),
-            'uploader': 'raisudtirol',
-            'formats': [{
+        ext = determine_ext(video_url)
+        if ext == 'm3u8':
+            formats = self._extract_m3u8_formats(video_url, video_id)
+        elif ext == 'mp4':
+            formats = [{
                 'format_id': 'https-mp4',
                 'url': self._proto_relative_url(video_url),
                 'width': 1024,
                 'height': 576,
                 'fps': 25,
-                'vcodec': 'h264',
-                'acodec': 'aac',
-            }],
+                'vcodec': 'avc1',
+                'acodec': 'mp4a',
+            }]
+        else:
+            formats = []
+            self.raise_no_formats(f'Unrecognized media file: {video_url}')
+
+        return {
+            'id': video_id,
+            'title': join_nonempty(video_title, video_date, delim=' - '),
+            'series': video_title if video_date else None,
+            'upload_date': unified_strdate(video_date),
+            'thumbnail': urljoin('https://raisudtirol.rai.it/', self._html_search_regex(
+                r'image: \'(.+?)\'', webpage, 'video_thumb', default=None)),
+            'uploader': 'raisudtirol',
+            'formats': formats,
         }

From bfdf144c7e5d7a93fbfa9d8e65598c72bf2b542a Mon Sep 17 00:00:00 2001
From: Mohit Tokas <mohittokas@live.com>
Date: Mon, 29 May 2023 10:16:32 +0530
Subject: [PATCH 292/871] [extractor/livestream] Support videos with account id
 (#6324)

Authored by: theperfectpunk
Closes #2225
---
 yt_dlp/extractor/livestream.py | 96 +++++++++++++++++++++-------------
 1 file changed, 60 insertions(+), 36 deletions(-)

diff --git a/yt_dlp/extractor/livestream.py b/yt_dlp/extractor/livestream.py
index d883eafcf..692d6ab3a 100644
--- a/yt_dlp/extractor/livestream.py
+++ b/yt_dlp/extractor/livestream.py
@@ -1,33 +1,36 @@
-import re
 import itertools
+import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urlparse,
-)
+from ..compat import compat_str, compat_urlparse
 from ..utils import (
-    find_xpath_attr,
-    xpath_attr,
-    xpath_with_ns,
-    xpath_text,
-    orderedSet,
-    update_url_query,
-    int_or_none,
-    float_or_none,
-    parse_iso8601,
     determine_ext,
+    find_xpath_attr,
+    float_or_none,
+    int_or_none,
+    orderedSet,
+    parse_iso8601,
+    traverse_obj,
+    update_url_query,
+    xpath_attr,
+    xpath_text,
+    xpath_with_ns,
 )
 
 
 class LivestreamIE(InfoExtractor):
     IE_NAME = 'livestream'
-    _VALID_URL = r'https?://(?:new\.)?livestream\.com/(?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))/(?:events/(?P<event_id>\d+)|(?P<event_name>[^/]+))(?:/videos/(?P<id>\d+))?'
+    _VALID_URL = r'''(?x)
+        https?://(?:new\.)?livestream\.com/
+        (?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))
+        (?:/events/(?P<event_id>\d+)|/(?P<event_name>[^/]+))?
+        (?:/videos/(?P<id>\d+))?
+    '''
     _EMBED_REGEX = [r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"']
 
     _TESTS = [{
         'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
-        'md5': '53274c76ba7754fb0e8d072716f2292b',
+        'md5': '7876c5f5dc3e711b6b73acce4aac1527',
         'info_dict': {
             'id': '4719370',
             'ext': 'mp4',
@@ -37,22 +40,37 @@ class LivestreamIE(InfoExtractor):
             'duration': 5968.0,
             'like_count': int,
             'view_count': int,
+            'comment_count': int,
             'thumbnail': r're:^http://.*\.jpg$'
         }
     }, {
-        'url': 'http://new.livestream.com/tedx/cityenglish',
+        'url': 'https://livestream.com/coheedandcambria/websterhall',
         'info_dict': {
-            'title': 'TEDCity2.0 (English)',
-            'id': '2245590',
+            'id': '1585861',
+            'title': 'Live From Webster Hall'
+        },
+        'playlist_mincount': 1,
+    }, {
+        'url': 'https://livestream.com/dayananda/events/7954027',
+        'info_dict': {
+            'title': 'Live from Mevo',
+            'id': '7954027',
         },
         'playlist_mincount': 4,
     }, {
-        'url': 'http://new.livestream.com/chess24/tatasteelchess',
+        'url': 'https://livestream.com/accounts/82',
         'info_dict': {
-            'title': 'Tata Steel Chess',
-            'id': '3705884',
-        },
-        'playlist_mincount': 60,
+            'id': '253978',
+            'view_count': int,
+            'title': 'trsr',
+            'comment_count': int,
+            'like_count': int,
+            'upload_date': '20120306',
+            'timestamp': 1331042383,
+            'thumbnail': 'http://img.new.livestream.com/videos/0000000000000372/cacbeed6-fb68-4b5e-ad9c-e148124e68a9_640x427.jpg',
+            'duration': 15.332,
+            'ext': 'mp4'
+        }
     }, {
         'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
         'only_matching': True,
@@ -179,7 +197,7 @@ def _extract_stream_info(self, stream_info):
             'is_live': is_live,
         }
 
-    def _extract_event(self, event_data):
+    def _generate_event_playlist(self, event_data):
         event_id = compat_str(event_data['id'])
         account_id = compat_str(event_data['owner_account_id'])
         feed_root_url = self._API_URL_TEMPLATE % (account_id, event_id) + '/feed.json'
@@ -189,7 +207,6 @@ def _extract_event(self, event_data):
             return self._extract_stream_info(stream_info)
 
         last_video = None
-        entries = []
         for i in itertools.count(1):
             if last_video is None:
                 info_url = feed_root_url
@@ -197,31 +214,38 @@ def _extract_event(self, event_data):
                 info_url = '{root}?&id={id}&newer=-1&type=video'.format(
                     root=feed_root_url, id=last_video)
             videos_info = self._download_json(
-                info_url, event_id, 'Downloading page {0}'.format(i))['data']
+                info_url, event_id, f'Downloading page {i}')['data']
             videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
             if not videos_info:
                 break
             for v in videos_info:
                 v_id = compat_str(v['id'])
-                entries.append(self.url_result(
-                    'http://livestream.com/accounts/%s/events/%s/videos/%s' % (account_id, event_id, v_id),
-                    'Livestream', v_id, v.get('caption')))
+                yield self.url_result(
+                    f'http://livestream.com/accounts/{account_id}/events/{event_id}/videos/{v_id}',
+                    LivestreamIE, v_id, v.get('caption'))
             last_video = videos_info[-1]['id']
-        return self.playlist_result(entries, event_id, event_data['full_name'])
 
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         video_id = mobj.group('id')
         event = mobj.group('event_id') or mobj.group('event_name')
         account = mobj.group('account_id') or mobj.group('account_name')
-        api_url = self._API_URL_TEMPLATE % (account, event)
+        api_url = f'http://livestream.com/api/accounts/{account}'
+
         if video_id:
             video_data = self._download_json(
-                api_url + '/videos/%s' % video_id, video_id)
+                f'{api_url}/events/{event}/videos/{video_id}', video_id)
             return self._extract_video_info(video_data)
-        else:
-            event_data = self._download_json(api_url, video_id)
-            return self._extract_event(event_data)
+        elif event:
+            event_data = self._download_json(f'{api_url}/events/{event}', None)
+            return self.playlist_result(
+                self._generate_event_playlist(event_data), str(event_data['id']), event_data['full_name'])
+
+        account_data = self._download_json(api_url, None)
+        items = traverse_obj(account_data, (('upcoming_events', 'past_events'), 'data', ...))
+        return self.playlist_result(
+            itertools.chain.from_iterable(map(self._generate_event_playlist, items)),
+            account_data.get('id'), account_data.get('full_name'))
 
 
 # The original version of Livestream uses a different system

From 17d7ca84ea723c20668bd9bfa938be7ea0e64f6b Mon Sep 17 00:00:00 2001
From: Ha Tien Loi <loiht.b17vt220@stu.ptit.edu.vn>
Date: Mon, 29 May 2023 12:02:16 +0700
Subject: [PATCH 293/871] [extractor/zingmp3] Fix and improve extractors
 (#6367)

Authored by: hatienl0i261299
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/zingmp3.py     | 101 ++++++++++++++++++++++----------
 2 files changed, 72 insertions(+), 30 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index d9028a831..6066b809b 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2483,6 +2483,7 @@
     ZingMp3WeekChartIE,
     ZingMp3ChartMusicVideoIE,
     ZingMp3UserIE,
+    ZingMp3HubIE,
 )
 from .zoom import ZoomIE
 from .zype import ZypeIE
diff --git a/yt_dlp/extractor/zingmp3.py b/yt_dlp/extractor/zingmp3.py
index a818c9fa9..007658c65 100644
--- a/yt_dlp/extractor/zingmp3.py
+++ b/yt_dlp/extractor/zingmp3.py
@@ -1,16 +1,11 @@
-import functools
 import hashlib
 import hmac
+import itertools
 import json
 import urllib.parse
 
 from .common import InfoExtractor
-from ..utils import (
-    OnDemandPagedList,
-    int_or_none,
-    traverse_obj,
-    urljoin,
-)
+from ..utils import int_or_none, traverse_obj, try_call, urljoin
 
 
 class ZingMp3BaseIE(InfoExtractor):
@@ -37,6 +32,7 @@ class ZingMp3BaseIE(InfoExtractor):
         'info-artist': '/api/v2/page/get/artist',
         'user-list-song': '/api/v2/song/get/list',
         'user-list-video': '/api/v2/video/get/list',
+        'hub': '/api/v2/page/get/hub-detail',
     }
 
     def _api_url(self, url_type, params):
@@ -46,9 +42,9 @@ def _api_url(self, url_type, params):
             ''.join(f'{k}={v}' for k, v in sorted(params.items())).encode()).hexdigest()
         data = {
             **params,
-            'apiKey': '88265e23d4284f25963e6eedac8fbfa3',
-            'sig': hmac.new(
-                b'2aa2d1c561e809b267f3638c4a307aab', f'{api_slug}{sha256}'.encode(), hashlib.sha512).hexdigest(),
+            'apiKey': 'X5BM3w8N7MKozC0B85o4KMlzLZKhV00y',
+            'sig': hmac.new(b'acOrvUS15XRW2o9JksiK1KgQ6Vbds8ZW',
+                            f'{api_slug}{sha256}'.encode(), hashlib.sha512).hexdigest(),
         }
         return f'{self._DOMAIN}{api_slug}?{urllib.parse.urlencode(data)}'
 
@@ -67,6 +63,19 @@ def _parse_items(self, items):
         for url in traverse_obj(items, (..., 'link')) or []:
             yield self.url_result(urljoin(self._DOMAIN, url))
 
+    def _fetch_page(self, id_, url_type, page):
+        raise NotImplementedError('This method must be implemented by subclasses')
+
+    def _paged_list(self, _id, url_type):
+        count = 0
+        for page in itertools.count(1):
+            data = self._fetch_page(_id, url_type, page)
+            entries = list(self._parse_items(data.get('items')))
+            count += len(entries)
+            yield from entries
+            if not data.get('hasMore') or try_call(lambda: count > data['total']):
+                break
+
 
 class ZingMp3IE(ZingMp3BaseIE):
     _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'bai-hat|video-clip|embed'
@@ -166,8 +175,11 @@ def _real_extract(self, url):
                     'height': int_or_none(res),
                 })
 
-        if not formats and item.get('msg') == 'Sorry, this content is not available in your country.':
-            self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
+        if not formats:
+            if item.get('msg') == 'Sorry, this content is not available in your country.':
+                self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
+            else:
+                self.raise_no_formats('The song is only for VIP accounts.')
 
         lyric = item.get('lyric') or self._call_api('lyric', {'id': item_id}, fatal=False).get('file')
 
@@ -200,7 +212,7 @@ class ZingMp3AlbumIE(ZingMp3BaseIE):
             'id': 'ZWZAEZZD',
             'title': 'Những Bài Hát Hay Nhất Của Mr. Siro',
         },
-        'playlist_mincount': 49,
+        'playlist_mincount': 20,
     }, {
         'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
         'only_matching': True,
@@ -305,22 +317,20 @@ class ZingMp3ChartMusicVideoIE(ZingMp3BaseIE):
             'id': 'IWZ9Z086',
             'title': 'the-loai-video_Khong-Loi',
         },
-        'playlist_mincount': 10,
+        'playlist_mincount': 1,
     }]
 
     def _fetch_page(self, song_id, url_type, page):
-        return self._parse_items(self._call_api(url_type, {
+        return self._call_api(url_type, {
             'id': song_id,
             'type': 'genre',
-            'page': page + 1,
+            'page': page,
             'count': self._PER_PAGE
-        }).get('items'))
+        })
 
     def _real_extract(self, url):
         song_id, regions, url_type = self._match_valid_url(url).group('id', 'regions', 'type')
-        return self.playlist_result(
-            OnDemandPagedList(functools.partial(self._fetch_page, song_id, url_type), self._PER_PAGE),
-            song_id, f'{url_type}_{regions}')
+        return self.playlist_result(self._paged_list(song_id, url_type), song_id, f'{url_type}_{regions}')
 
 
 class ZingMp3UserIE(ZingMp3BaseIE):
@@ -331,7 +341,7 @@ class ZingMp3UserIE(ZingMp3BaseIE):
         'info_dict': {
             'id': 'IWZ98609',
             'title': 'Mr. Siro - bai-hat',
-            'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
+            'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
         },
         'playlist_mincount': 91,
     }, {
@@ -339,7 +349,7 @@ class ZingMp3UserIE(ZingMp3BaseIE):
         'info_dict': {
             'id': 'IWZ98609',
             'title': 'Mr. Siro - album',
-            'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
+            'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
         },
         'playlist_mincount': 3,
     }, {
@@ -347,7 +357,7 @@ class ZingMp3UserIE(ZingMp3BaseIE):
         'info_dict': {
             'id': 'IWZ98609',
             'title': 'Mr. Siro - single',
-            'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
+            'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
         },
         'playlist_mincount': 20,
     }, {
@@ -355,19 +365,19 @@ class ZingMp3UserIE(ZingMp3BaseIE):
         'info_dict': {
             'id': 'IWZ98609',
             'title': 'Mr. Siro - video',
-            'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
+            'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
         },
         'playlist_mincount': 15,
     }]
 
     def _fetch_page(self, user_id, url_type, page):
         url_type = 'user-list-song' if url_type == 'bai-hat' else 'user-list-video'
-        return self._parse_items(self._call_api(url_type, {
+        return self._call_api(url_type, {
             'id': user_id,
             'type': 'artist',
-            'page': page + 1,
+            'page': page,
             'count': self._PER_PAGE
-        }, query={'sort': 'new', 'sectionId': 'aSong'}).get('items'))
+        })
 
     def _real_extract(self, url):
         user_alias, url_type = self._match_valid_url(url).group('user', 'type')
@@ -376,10 +386,41 @@ def _real_extract(self, url):
 
         user_info = self._call_api('info-artist', {}, user_alias, query={'alias': user_alias})
         if url_type in ('bai-hat', 'video'):
-            entries = OnDemandPagedList(
-                functools.partial(self._fetch_page, user_info['id'], url_type), self._PER_PAGE)
+            entries = self._paged_list(user_info['id'], url_type)
         else:
             entries = self._parse_items(traverse_obj(user_info, (
-                'sections', lambda _, v: v['link'] == f'/{user_alias}/{url_type}', 'items', ...)))
+                'sections',
+                lambda _, v: v['sectionId'] == 'aAlbum' if url_type == 'album' else v['sectionId'] == 'aSingle',
+                'items', ...)))
         return self.playlist_result(
             entries, user_info['id'], f'{user_info.get("name")} - {url_type}', user_info.get('biography'))
+
+
+class ZingMp3HubIE(ZingMp3BaseIE):
+    IE_NAME = 'zingmp3:hub'
+    _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<type>hub)/(?P<regions>[^/]+)/(?P<id>[^\.]+)'
+    _TESTS = [{
+        'url': 'https://zingmp3.vn/hub/Nhac-Moi/IWZ9Z0CA.html',
+        'info_dict': {
+            'id': 'IWZ9Z0CA',
+            'title': 'Nhạc Mới',
+            'description': 'md5:1cc31b68a6f746427b07b2756c22a558',
+        },
+        'playlist_mincount': 20,
+    }, {
+        'url': 'https://zingmp3.vn/hub/Nhac-Viet/IWZ9Z087.html',
+        'info_dict': {
+            'id': 'IWZ9Z087',
+            'title': 'Nhạc Việt',
+            'description': 'md5:acc976c8bdde64d5c6ee4a92c39f7a77',
+        },
+        'playlist_mincount': 30,
+    }]
+
+    def _real_extract(self, url):
+        song_id, regions, url_type = self._match_valid_url(url).group('id', 'regions', 'type')
+        hub_detail = self._call_api(url_type, {'id': song_id})
+        entries = self._parse_items(traverse_obj(hub_detail, (
+            'sections', lambda _, v: v['sectionId'] == 'hub', 'items', ...)))
+        return self.playlist_result(
+            entries, song_id, hub_detail.get('title'), hub_detail.get('description'))

From c6d4b82a8b8bce59b1c9ce5e6d349ea428dac0a7 Mon Sep 17 00:00:00 2001
From: Daniel Vogt <c0d3d3v@mag-keinen-spam.de>
Date: Mon, 29 May 2023 07:21:26 +0200
Subject: [PATCH 294/871] [extractor/owncloud] Add extractor (#6533)

Authored by: C0D3D3V
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/owncloud.py    | 80 +++++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 yt_dlp/extractor/owncloud.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 6066b809b..b02244284 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1378,6 +1378,7 @@
     ORFIPTVIE,
 )
 from .outsidetv import OutsideTVIE
+from .owncloud import OwnCloudIE
 from .packtpub import (
     PacktPubIE,
     PacktPubCourseIE,
diff --git a/yt_dlp/extractor/owncloud.py b/yt_dlp/extractor/owncloud.py
new file mode 100644
index 000000000..e1d5682f8
--- /dev/null
+++ b/yt_dlp/extractor/owncloud.py
@@ -0,0 +1,80 @@
+import re
+import urllib.parse
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    url_or_none,
+    urlencode_postdata,
+)
+
+
+class OwnCloudIE(InfoExtractor):
+    _INSTANCES_RE = '|'.join((
+        r'(?:[^\.]+\.)?sciebo\.de',
+        r'cloud\.uni-koblenz-landau\.de',
+    ))
+    _VALID_URL = rf'https?://(?:{_INSTANCES_RE})/s/(?P<id>[\w.-]+)'
+
+    _TESTS = [
+        {
+            'url': 'https://ruhr-uni-bochum.sciebo.de/s/wWhqZzh9jTumVFN',
+            'info_dict': {
+                'id': 'wWhqZzh9jTumVFN',
+                'ext': 'mp4',
+                'title': 'CmvpJST.mp4',
+            },
+        },
+        {
+            'url': 'https://ruhr-uni-bochum.sciebo.de/s/WNDuFu0XuFtmm3f',
+            'info_dict': {
+                'id': 'WNDuFu0XuFtmm3f',
+                'ext': 'mp4',
+                'title': 'CmvpJST.mp4',
+            },
+            'params': {
+                'videopassword': '12345',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage, urlh = self._download_webpage_handle(url, video_id)
+
+        if re.search(r'<label[^>]+for="password"', webpage):
+            webpage = self._verify_video_password(webpage, urlh.geturl(), video_id)
+
+        hidden_inputs = self._hidden_inputs(webpage)
+        title = hidden_inputs.get('filename')
+        parsed_url = urllib.parse.urlparse(url)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url_or_none(hidden_inputs.get('downloadURL')) or parsed_url._replace(
+                path=urllib.parse.urljoin(parsed_url.path, 'download')).geturl(),
+            'ext': determine_ext(title),
+        }
+
+    def _verify_video_password(self, webpage, url, video_id):
+        password = self.get_param('videopassword')
+        if password is None:
+            raise ExtractorError(
+                'This video is protected by a password, use the --video-password option',
+                expected=True)
+
+        validation_response = self._download_webpage(
+            url, video_id, 'Validating Password', 'Wrong password?',
+            data=urlencode_postdata({
+                'requesttoken': self._hidden_inputs(webpage)['requesttoken'],
+                'password': password,
+            }))
+
+        if re.search(r'<label[^>]+for="password"', validation_response):
+            warning = self._search_regex(
+                r'<div[^>]+class="warning">([^<]*)</div>', validation_response,
+                'warning', default='The password is wrong')
+            raise ExtractorError(f'Opening the video failed, {self.IE_NAME} said: {warning!r}', expected=True)
+        return validation_response

From 94627c5dde12a72766bdba36e056916c29c40ed1 Mon Sep 17 00:00:00 2001
From: Stefan Borer <stefan.borer@gmail.com>
Date: Mon, 29 May 2023 07:26:49 +0200
Subject: [PATCH 295/871] [extractor/playsuisse] Support new url format (#6528)

Authored by: sbor23
---
 yt_dlp/extractor/playsuisse.py | 88 ++++++++++++++++++++++++----------
 1 file changed, 64 insertions(+), 24 deletions(-)

diff --git a/yt_dlp/extractor/playsuisse.py b/yt_dlp/extractor/playsuisse.py
index a635ac92f..76288c778 100644
--- a/yt_dlp/extractor/playsuisse.py
+++ b/yt_dlp/extractor/playsuisse.py
@@ -5,10 +5,16 @@
 
 
 class PlaySuisseIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?playsuisse\.ch/watch/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?playsuisse\.ch/(?:watch|detail)/(?:[^#]*[?&]episodeId=)?(?P<id>[0-9]+)'
     _TESTS = [
         {
+            # Old URL
             'url': 'https://www.playsuisse.ch/watch/763211/0',
+            'only_matching': True,
+        },
+        {
+            # episode in a series
+            'url': 'https://www.playsuisse.ch/watch/763182?episodeId=763211',
             'md5': '82df2a470b2dfa60c2d33772a8a60cf8',
             'info_dict': {
                 'id': '763211',
@@ -21,11 +27,11 @@ class PlaySuisseIE(InfoExtractor):
                 'season_number': 1,
                 'episode': 'Knochen',
                 'episode_number': 1,
-                'thumbnail': 'md5:9260abe0c0ec9b69914d0a10d54c5878'
+                'thumbnail': 're:https://playsuisse-img.akamaized.net/',
             }
-        },
-        {
-            'url': 'https://www.playsuisse.ch/watch/808675/0',
+        }, {
+            # film
+            'url': 'https://www.playsuisse.ch/watch/808675',
             'md5': '818b94c1d2d7c4beef953f12cb8f3e75',
             'info_dict': {
                 'id': '808675',
@@ -33,26 +39,60 @@ class PlaySuisseIE(InfoExtractor):
                 'title': 'Der Läufer',
                 'description': 'md5:9f61265c7e6dcc3e046137a792b275fd',
                 'duration': 5280,
-                'episode': 'Der Läufer',
-                'thumbnail': 'md5:44af7d65ee02bbba4576b131868bb783'
+                'thumbnail': 're:https://playsuisse-img.akamaized.net/',
             }
-        },
-        {
-            'url': 'https://www.playsuisse.ch/watch/817193/0',
-            'md5': '1d6c066f92cd7fffd8b28a53526d6b59',
+        }, {
+            # series (treated as a playlist)
+            'url': 'https://www.playsuisse.ch/detail/1115687',
             'info_dict': {
-                'id': '817193',
-                'ext': 'mp4',
-                'title': 'Die Einweihungsparty',
-                'description': 'md5:91ebf04d3a42cb3ab70666acf750a930',
-                'duration': 1380,
-                'series': 'Nr. 47',
-                'season': 'Season 1',
-                'season_number': 1,
-                'episode': 'Die Einweihungsparty',
-                'episode_number': 1,
-                'thumbnail': 'md5:637585fb106e3a4bcd991958924c7e44'
-            }
+                'description': 'md5:e4a2ae29a8895823045b5c3145a02aa3',
+                'id': '1115687',
+                'series': 'They all came out to Montreux',
+                'title': 'They all came out to Montreux',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'description': 'md5:f2462744834b959a31adc6292380cda2',
+                    'duration': 3180,
+                    'episode': 'Folge 1',
+                    'episode_number': 1,
+                    'id': '1112663',
+                    'season': 'Season 1',
+                    'season_number': 1,
+                    'series': 'They all came out to Montreux',
+                    'thumbnail': 're:https://playsuisse-img.akamaized.net/',
+                    'title': 'Folge 1',
+                    'ext': 'mp4'
+                },
+            }, {
+                'info_dict': {
+                    'description': 'md5:9dfd308699fe850d3bce12dc1bad9b27',
+                    'duration': 2935,
+                    'episode': 'Folge 2',
+                    'episode_number': 2,
+                    'id': '1112661',
+                    'season': 'Season 1',
+                    'season_number': 1,
+                    'series': 'They all came out to Montreux',
+                    'thumbnail': 're:https://playsuisse-img.akamaized.net/',
+                    'title': 'Folge 2',
+                    'ext': 'mp4'
+                },
+            }, {
+                'info_dict': {
+                    'description': 'md5:14a93a3356b2492a8f786ab2227ef602',
+                    'duration': 2994,
+                    'episode': 'Folge 3',
+                    'episode_number': 3,
+                    'id': '1112664',
+                    'season': 'Season 1',
+                    'season_number': 1,
+                    'series': 'They all came out to Montreux',
+                    'thumbnail': 're:https://playsuisse-img.akamaized.net/',
+                    'title': 'Folge 3',
+                    'ext': 'mp4'
+                }
+            }],
         }
     ]
 
@@ -142,6 +182,6 @@ def _extract_single(self, media_data):
             'subtitles': subtitles,
             'series': media_data.get('seriesName'),
             'season_number': int_or_none(media_data.get('seasonNumber')),
-            'episode': media_data.get('name'),
+            'episode': media_data.get('name') if media_data.get('episodeNumber') else None,
             'episode_number': int_or_none(media_data.get('episodeNumber')),
         }

From 02312c03cf53eb1da24c9ad022ee79af26060733 Mon Sep 17 00:00:00 2001
From: bepvte <8226605+bepvte@users.noreply.github.com>
Date: Sun, 28 May 2023 22:54:36 -0700
Subject: [PATCH 296/871] [extractor/twitch] Support mobile clips (#6699)

Authored by: bepvte
---
 yt_dlp/extractor/twitch.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index 9b333f6f6..d7a1cc531 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -1075,7 +1075,7 @@ class TwitchClipsIE(TwitchBaseIE):
                     https?://
                         (?:
                             clips\.twitch\.tv/(?:embed\?.*?\bclip=|(?:[^/]+/)*)|
-                            (?:(?:www|go|m)\.)?twitch\.tv/[^/]+/clip/
+                            (?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/)?clip/
                         )
                         (?P<id>[^/?#&]+)
                     '''
@@ -1111,6 +1111,9 @@ class TwitchClipsIE(TwitchBaseIE):
     }, {
         'url': 'https://go.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
         'only_matching': True,
+    }, {
+        'url': 'https://m.twitch.tv/clip/FaintLightGullWholeWheat',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 5c14b213679ed4401288bdc86ae696932e219222 Mon Sep 17 00:00:00 2001
From: ping <ping@users.noreply.github.com>
Date: Mon, 29 May 2023 14:01:42 +0800
Subject: [PATCH 297/871] [extractor/idolplus] Add extractor (#6732)

Authored by:  ping
Closes #6246
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/idolplus.py    | 115 ++++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)
 create mode 100644 yt_dlp/extractor/idolplus.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b02244284..999b11378 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -789,6 +789,7 @@
     IchinanaLiveIE,
     IchinanaLiveClipIE,
 )
+from .idolplus import IdolPlusIE
 from .ign import (
     IGNIE,
     IGNVideoIE,
diff --git a/yt_dlp/extractor/idolplus.py b/yt_dlp/extractor/idolplus.py
new file mode 100644
index 000000000..3c905b071
--- /dev/null
+++ b/yt_dlp/extractor/idolplus.py
@@ -0,0 +1,115 @@
+from .common import InfoExtractor
+from ..utils import traverse_obj, try_call, url_or_none
+
+
+class IdolPlusIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?idolplus\.com/z[us]/(?:concert/|contents/?\?(?:[^#]+&)?albumId=)(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://idolplus.com/zs/contents?albumId=M012077298PPV00',
+        'md5': '2ace3f4661c943a2f7e79f0b88cea1e7',
+        'info_dict': {
+            'id': 'M012077298PPV00',
+            'ext': 'mp4',
+            'title': '[MultiCam] Aegyo on Top of Aegyo (IZ*ONE EATING TRIP)',
+            'release_date': '20200707',
+            'formats': 'count:65',
+        },
+        'params': {'format': '532-KIM_MINJU'},
+    }, {
+        'url': 'https://idolplus.com/zs/contents?albumId=M01232H058PPV00&catId=E9TX5',
+        'info_dict': {
+            'id': 'M01232H058PPV00',
+            'ext': 'mp4',
+            'title': 'YENA (CIRCLE CHART MUSIC AWARDS 2022 RED CARPET)',
+            'release_date': '20230218',
+            'formats': 'count:5',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        # live stream
+        'url': 'https://idolplus.com/zu/contents?albumId=M012323174PPV00',
+        'info_dict': {
+            'id': 'M012323174PPV00',
+            'ext': 'mp4',
+            'title': 'Hanteo Music Awards 2022 DAY2',
+            'release_date': '20230211',
+            'formats': 'count:5',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://idolplus.com/zs/concert/M012323039PPV00',
+        'info_dict': {
+            'id': 'M012323039PPV00',
+            'ext': 'mp4',
+            'title': 'CIRCLE CHART MUSIC AWARDS 2022',
+            'release_date': '20230218',
+            'formats': 'count:5',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data_list = traverse_obj(self._download_json(
+            'https://idolplus.com/api/zs/viewdata/ruleset/build', video_id,
+            headers={'App_type': 'web', 'Country_Code': 'KR'}, query={
+                'rulesetId': 'contents',
+                'albumId': video_id,
+                'distribute': 'PRD',
+                'loggedIn': 'false',
+                'region': 'zs',
+                'countryGroup': '00010',
+                'lang': 'en',
+                'saId': '999999999998',
+            }), ('data', 'viewData', ...))
+
+        player_data = {}
+        while data_list:
+            player_data = data_list.pop()
+            if traverse_obj(player_data, 'type') == 'player':
+                break
+            elif traverse_obj(player_data, ('dataList', ...)):
+                data_list += player_data['dataList']
+
+        formats = self._extract_m3u8_formats(traverse_obj(player_data, (
+            'vodPlayerList', 'vodProfile', 0, 'vodServer', 0, 'video_url', {url_or_none})), video_id)
+
+        subtitles = {}
+        for caption in traverse_obj(player_data, ('vodPlayerList', 'caption')) or []:
+            subtitles.setdefault(caption.get('lang') or 'und', []).append({
+                'url': caption.get('smi_url'),
+                'ext': 'vtt',
+            })
+
+        # Add member multicams as alternative formats
+        if (traverse_obj(player_data, ('detail', 'has_cuesheet')) == 'Y'
+                and traverse_obj(player_data, ('detail', 'is_omni_member')) == 'Y'):
+            cuesheet = traverse_obj(self._download_json(
+                'https://idolplus.com/gapi/contents/v1.0/content/cuesheet', video_id,
+                'Downloading JSON metadata for member multicams',
+                headers={'App_type': 'web', 'Country_Code': 'KR'}, query={
+                    'ALBUM_ID': video_id,
+                    'COUNTRY_GRP': '00010',
+                    'LANG': 'en',
+                    'SA_ID': '999999999998',
+                    'COUNTRY_CODE': 'KR',
+                }), ('data', 'cuesheet_item', 0))
+
+            for member in traverse_obj(cuesheet, ('members', ...)):
+                index = try_call(lambda: int(member['omni_view_index']) - 1)
+                member_video_url = traverse_obj(cuesheet, ('omni_view', index, 'cdn_url', 0, 'url', {url_or_none}))
+                if not member_video_url:
+                    continue
+                member_formats = self._extract_m3u8_formats(
+                    member_video_url, video_id, note=f'Downloading m3u8 for multicam {member["name"]}')
+                for mf in member_formats:
+                    mf['format_id'] = f'{mf["format_id"]}-{member["name"].replace(" ", "_")}'
+                formats.extend(member_formats)
+
+        return {
+            'id': video_id,
+            'title': traverse_obj(player_data, ('detail', 'albumName')),
+            'formats': formats,
+            'subtitles': subtitles,
+            'release_date': traverse_obj(player_data, ('detail', 'broadcastDate')),
+        }

From 4afb208cf07b59291ae3b0c4efc83945ee5b8812 Mon Sep 17 00:00:00 2001
From: jo-nike <derter@gmail.com>
Date: Mon, 29 May 2023 02:04:08 -0400
Subject: [PATCH 298/871] [extractor/cbc] Ignore 426 from API (#6781)

Closes #6716
Authored by: jo-nike
---
 yt_dlp/extractor/cbc.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index e42f06246..41e092422 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -351,7 +351,9 @@ def _find_secret_formats(self, formats, video_id):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_info = self._download_json('https://services.radio-canada.ca/ott/cbc-api/v2/assets/' + video_id, video_id)
+        video_info = self._download_json(
+            f'https://services.radio-canada.ca/ott/cbc-api/v2/assets/{video_id}',
+            video_id, expected_status=426)
 
         email, password = self._get_login_info()
         if email and password:
@@ -426,7 +428,7 @@ def _real_extract(self, url):
         match = self._match_valid_url(url)
         season_id = match.group('id')
         show = match.group('show')
-        show_info = self._download_json(self._API_BASE + show, season_id)
+        show_info = self._download_json(self._API_BASE + show, season_id, expected_status=426)
         season = int(match.group('season'))
 
         season_info = next((s for s in show_info['seasons'] if s.get('season') == season), None)

From a58182b75a05fe0a10c5e94a536711d3ade19c20 Mon Sep 17 00:00:00 2001
From: Nam Vu <git@yuru.moe>
Date: Mon, 29 May 2023 15:05:51 +0900
Subject: [PATCH 299/871] [cookies] Support custom Safari cookies path (#6783)

Authored by: NextFire
---
 yt_dlp/cookies.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index eb6a2656b..ee2af0f70 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -495,18 +495,22 @@ def decrypt(self, encrypted_value):
 
 
 def _extract_safari_cookies(profile, logger):
-    if profile is not None:
-        logger.error('safari does not support profiles')
     if sys.platform != 'darwin':
         raise ValueError(f'unsupported platform: {sys.platform}')
 
-    cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
-
-    if not os.path.isfile(cookies_path):
-        logger.debug('Trying secondary cookie location')
-        cookies_path = os.path.expanduser('~/Library/Containers/com.apple.Safari/Data/Library/Cookies/Cookies.binarycookies')
+    if profile:
+        cookies_path = os.path.expanduser(profile)
         if not os.path.isfile(cookies_path):
-            raise FileNotFoundError('could not find safari cookies database')
+            raise FileNotFoundError('custom safari cookies database not found')
+
+    else:
+        cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
+
+        if not os.path.isfile(cookies_path):
+            logger.debug('Trying secondary cookie location')
+            cookies_path = os.path.expanduser('~/Library/Containers/com.apple.Safari/Data/Library/Cookies/Cookies.binarycookies')
+            if not os.path.isfile(cookies_path):
+                raise FileNotFoundError('could not find safari cookies database')
 
     with open(cookies_path, 'rb') as f:
         cookies_data = f.read()

From c25cac2f8e5fbac2737a426d7778fd2f0efc5381 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Mon, 29 May 2023 01:40:44 -0500
Subject: [PATCH 300/871] [extractor/dacast] Add extractors (#6896)

Closes #6163
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/dacast.py      | 158 ++++++++++++++++++++++++++++++++
 2 files changed, 162 insertions(+)
 create mode 100644 yt_dlp/extractor/dacast.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 999b11378..0f65f1cc7 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -428,6 +428,10 @@
     CybraryIE,
     CybraryCourseIE
 )
+from .dacast import (
+    DacastVODIE,
+    DacastPlaylistIE,
+)
 from .daftsex import DaftsexIE
 from .dailymail import DailyMailIE
 from .dailymotion import (
diff --git a/yt_dlp/extractor/dacast.py b/yt_dlp/extractor/dacast.py
new file mode 100644
index 000000000..cf683bad4
--- /dev/null
+++ b/yt_dlp/extractor/dacast.py
@@ -0,0 +1,158 @@
+import hashlib
+import re
+import time
+import urllib.error
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    classproperty,
+    float_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class DacastBaseIE(InfoExtractor):
+    _URL_TYPE = None
+
+    @classproperty
+    def _VALID_URL(cls):
+        return fr'https?://iframe\.dacast\.com/{cls._URL_TYPE}/(?P<user_id>[\w-]+)/(?P<id>[\w-]+)'
+
+    @classproperty
+    def _EMBED_REGEX(cls):
+        return [rf'<iframe[^>]+\bsrc=["\'](?P<url>{cls._VALID_URL})']
+
+    _API_INFO_URL = 'https://playback.dacast.com/content/info'
+
+    @classmethod
+    def _get_url_from_id(cls, content_id):
+        user_id, media_id = content_id.split(f'-{cls._URL_TYPE}-')
+        return f'https://iframe.dacast.com/{cls._URL_TYPE}/{user_id}/{media_id}'
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        yield from super()._extract_embed_urls(url, webpage)
+        for content_id in re.findall(
+                rf'<script[^>]+\bsrc=["\']https://player\.dacast\.com/js/player\.js\?contentId=([\w-]+-{cls._URL_TYPE}-[\w-]+)["\']', webpage):
+            yield cls._get_url_from_id(content_id)
+
+
+class DacastVODIE(DacastBaseIE):
+    _URL_TYPE = 'vod'
+    _TESTS = [{
+        'url': 'https://iframe.dacast.com/vod/acae82153ef4d7a7344ae4eaa86af534/1c6143e3-5a06-371d-8695-19b96ea49090',
+        'info_dict': {
+            'id': '1c6143e3-5a06-371d-8695-19b96ea49090',
+            'ext': 'mp4',
+            'uploader_id': 'acae82153ef4d7a7344ae4eaa86af534',
+            'title': '2_4||Adnexal mass characterisation: O-RADS US and MRI||N. Bharwani, London/UK',
+            'thumbnail': 'https://universe-files.dacast.com/26137208-5858-65c1-5e9a-9d6b6bd2b6c2',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://www.dacast.com/support/knowledgebase/how-can-i-embed-a-video-on-my-website/',
+        'info_dict': {
+            'id': 'b6674869-f08a-23c5-1d7b-81f5309e1a90',
+            'ext': 'mp4',
+            'title': '4-HowToEmbedVideo.mp4',
+            'uploader_id': '3b67c4a9-3886-4eb1-d0eb-39b23b14bef3',
+            'thumbnail': 'https://universe-files.dacast.com/d26ab48f-a52a-8783-c42e-a90290ba06b6.png',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://gist.githubusercontent.com/bashonly/4ad249ef2910346fbdf3809b220f11ee/raw/87349778d4af1a80b1fcc3beb9c88108de5858f5/dacast_embeds.html',
+        'info_dict': {
+            'id': 'e7df418e-a83b-7a7f-7b5e-1a667981e8fa',
+            'ext': 'mp4',
+            'title': 'Evening Service 2-5-23',
+            'uploader_id': '943bb1ab3c03695ba85330d92d6d226e',
+            'thumbnail': 'https://universe-files.dacast.com/337472b3-e92c-2ea4-7eb7-5700da477f67',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        user_id, video_id = self._match_valid_url(url).group('user_id', 'id')
+        query = {'contentId': f'{user_id}-vod-{video_id}', 'provider': 'universe'}
+        info = self._download_json(self._API_INFO_URL, video_id, query=query, fatal=False)
+        access = self._download_json(
+            'https://playback.dacast.com/content/access', video_id,
+            note='Downloading access JSON', query=query, expected_status=403)
+
+        error = access.get('error')
+        if error in ('Broadcaster has been blocked', 'Content is offline'):
+            raise ExtractorError(error, expected=True)
+        elif error:
+            raise ExtractorError(f'Dacast API says "{error}"')
+
+        hls_url = access['hls']
+        hls_aes = {}
+
+        if 'DRM_EXT' in hls_url:
+            self.report_drm(video_id)
+        elif '/uspaes/' in hls_url:
+            # From https://player.dacast.com/js/player.js
+            ts = int(time.time())
+            signature = hashlib.sha1(
+                f'{10413792000 - ts}{ts}YfaKtquEEpDeusCKbvYszIEZnWmBcSvw').digest().hex()
+            hls_aes['uri'] = f'https://keys.dacast.com/uspaes/{video_id}.key?s={signature}&ts={ts}'
+
+        for retry in self.RetryManager():
+            try:
+                formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls')
+            except ExtractorError as e:
+                # CDN will randomly respond with 403
+                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                    retry.error = e
+                    continue
+                raise
+
+        return {
+            'id': video_id,
+            'uploader_id': user_id,
+            'formats': formats,
+            'hls_aes': hls_aes or None,
+            **traverse_obj(info, ('contentInfo', {
+                'title': 'title',
+                'duration': ('duration', {float_or_none}),
+                'thumbnail': ('thumbnailUrl', {url_or_none}),
+            })),
+        }
+
+
+class DacastPlaylistIE(DacastBaseIE):
+    _URL_TYPE = 'playlist'
+    _TESTS = [{
+        'url': 'https://iframe.dacast.com/playlist/943bb1ab3c03695ba85330d92d6d226e/b632eb053cac17a9c9a02bcfc827f2d8',
+        'playlist_mincount': 28,
+        'info_dict': {
+            'id': 'b632eb053cac17a9c9a02bcfc827f2d8',
+            'title': 'Archive Sermons',
+        },
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://gist.githubusercontent.com/bashonly/7efb606f49f3c6e07ea0327de5a661d1/raw/05a16eac830245ea301fb0a585023bec71e6093c/dacast_playlist_embed.html',
+        'playlist_mincount': 28,
+        'info_dict': {
+            'id': 'b632eb053cac17a9c9a02bcfc827f2d8',
+            'title': 'Archive Sermons',
+        },
+    }]
+
+    def _real_extract(self, url):
+        user_id, playlist_id = self._match_valid_url(url).group('user_id', 'id')
+        info = self._download_json(
+            self._API_INFO_URL, playlist_id, note='Downloading playlist JSON', query={
+                'contentId': f'{user_id}-playlist-{playlist_id}',
+                'provider': 'universe',
+            })['contentInfo']
+
+        def entries(info):
+            for video in traverse_obj(info, ('features', 'playlist', 'contents', lambda _, v: v['id'])):
+                yield self.url_result(
+                    DacastVODIE._get_url_from_id(video['id']), DacastVODIE, video['id'], video.get('title'))
+
+        return self.playlist_result(entries(info), playlist_id, info.get('title'))

From 3459d3c5af3b2572ed51e8ecfda6c11022a838c6 Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Mon, 29 May 2023 18:33:37 +0900
Subject: [PATCH 301/871] [extractor/JStream] Add extractor (#6252)

Authored by: Lesmiscore
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/jstream.py     | 73 +++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 yt_dlp/extractor/jstream.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 0f65f1cc7..d560ed91c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -878,6 +878,7 @@
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
 from .joj import JojIE
+from .jstream import JStreamIE
 from .jwplatform import JWPlatformIE
 from .kakao import KakaoIE
 from .kaltura import KalturaIE
diff --git a/yt_dlp/extractor/jstream.py b/yt_dlp/extractor/jstream.py
new file mode 100644
index 000000000..3e2e62712
--- /dev/null
+++ b/yt_dlp/extractor/jstream.py
@@ -0,0 +1,73 @@
+import base64
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    js_to_json,
+    remove_start,
+)
+
+
+class JStreamIE(InfoExtractor):
+    # group "id" only exists for compliance, not directly used in requests
+    # also all components are mandatory
+    _VALID_URL = r'jstream:(?P<host>www\d+):(?P<id>(?P<publisher>[a-z0-9]+):(?P<mid>\d+))'
+
+    _TESTS = [{
+        'url': 'jstream:www50:eqd638pvwx:752',
+        'info_dict': {
+            'id': 'eqd638pvwx:752',
+            'ext': 'mp4',
+            'title': '阪神淡路大震災 激震の記録2020年版　解説動画',
+            'duration': 672,
+            'thumbnail': r're:https?://eqd638pvwx\.eq\.webcdn\.stream\.ne\.jp/.+\.jpg',
+        },
+    }]
+
+    def _parse_jsonp(self, callback, string, video_id):
+        return self._search_json(rf'\s*{re.escape(callback)}\s*\(', string, callback, video_id)
+
+    def _find_formats(self, video_id, movie_list_hls, host, publisher, subtitles):
+        for value in movie_list_hls:
+            text = value.get('text') or ''
+            if not text.startswith('auto'):
+                continue
+            m3u8_id = remove_start(remove_start(text, 'auto'), '_') or None
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                f'https://{publisher}.eq.webcdn.stream.ne.jp/{host}/{publisher}/jmc_pub/{value.get("url")}', video_id, 'mp4', m3u8_id=m3u8_id)
+            self._merge_subtitles(subs, target=subtitles)
+            yield from fmts
+
+    def _real_extract(self, url):
+        host, publisher, mid, video_id = self._match_valid_url(url).group('host', 'publisher', 'mid', 'id')
+        video_info_jsonp = self._download_webpage(
+            f'https://{publisher}.eq.webcdn.stream.ne.jp/{host}/{publisher}/jmc_pub/eq_meta/v1/{mid}.jsonp',
+            video_id, 'Requesting video info')
+        video_info = self._parse_jsonp('metaDataResult', video_info_jsonp, video_id)['movie']
+        subtitles = {}
+        formats = list(self._find_formats(video_id, video_info.get('movie_list_hls'), host, publisher, subtitles))
+        self._remove_duplicate_formats(formats)
+        return {
+            'id': video_id,
+            'title': video_info.get('title'),
+            'duration': float_or_none(video_info.get('duration')),
+            'thumbnail': video_info.get('thumbnail_url'),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        # check for eligiblity of webpage
+        # https://support.eq.stream.co.jp/hc/ja/articles/115008388147-%E3%83%97%E3%83%AC%E3%82%A4%E3%83%A4%E3%83%BCAPI%E3%81%AE%E3%82%B5%E3%83%B3%E3%83%97%E3%83%AB%E3%82%B3%E3%83%BC%E3%83%89
+        script_tag = re.search(r'<script\s*[^>]+?src="https://ssl-cache\.stream\.ne\.jp/(?P<host>www\d+)/(?P<publisher>[a-z0-9]+)/[^"]+?/if\.js"', webpage)
+        if not script_tag:
+            return
+        host, publisher = script_tag.groups()
+        for m in re.finditer(r'(?s)PlayerFactoryIF\.create\(\s*({[^\}]+?})\s*\)\s*;', webpage):
+            # TODO: using json.loads here as InfoExtractor._parse_json is not classmethod
+            info = json.loads(js_to_json(m.group(1)))
+            mid = base64.b64decode(info.get('m')).decode()
+            yield f'jstream:{host}:{publisher}:{mid}'

From f8f9250fe280d37f0988646cd5cc0072f4d33a6d Mon Sep 17 00:00:00 2001
From: Lesmiscore <nao20010128@gmail.com>
Date: Mon, 29 May 2023 18:35:10 +0900
Subject: [PATCH 302/871] [extractor/niconico:live] Add extractor (#5764)

Authored by: Lesmiscore
---
 yt_dlp/downloader/__init__.py   |   3 +-
 yt_dlp/downloader/niconico.py   | 101 +++++++++++++++++++-
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/niconico.py    | 163 ++++++++++++++++++++++++++++++++
 4 files changed, 266 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/downloader/__init__.py b/yt_dlp/downloader/__init__.py
index c34dbcea9..51a9f28f0 100644
--- a/yt_dlp/downloader/__init__.py
+++ b/yt_dlp/downloader/__init__.py
@@ -30,7 +30,7 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N
 from .http import HttpFD
 from .ism import IsmFD
 from .mhtml import MhtmlFD
-from .niconico import NiconicoDmcFD
+from .niconico import NiconicoDmcFD, NiconicoLiveFD
 from .rtmp import RtmpFD
 from .rtsp import RtspFD
 from .websocket import WebSocketFragmentFD
@@ -50,6 +50,7 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N
     'ism': IsmFD,
     'mhtml': MhtmlFD,
     'niconico_dmc': NiconicoDmcFD,
+    'niconico_live': NiconicoLiveFD,
     'fc2_live': FC2LiveFD,
     'websocket_frag': WebSocketFragmentFD,
     'youtube_live_chat': YoutubeLiveChatFD,
diff --git a/yt_dlp/downloader/niconico.py b/yt_dlp/downloader/niconico.py
index 77ed39e5b..cfe739784 100644
--- a/yt_dlp/downloader/niconico.py
+++ b/yt_dlp/downloader/niconico.py
@@ -1,8 +1,17 @@
+import json
 import threading
+import time
 
 from . import get_suitable_downloader
 from .common import FileDownloader
-from ..utils import sanitized_Request
+from .external import FFmpegFD
+from ..utils import (
+    DownloadError,
+    str_or_none,
+    sanitized_Request,
+    WebSocketsWrapper,
+    try_get,
+)
 
 
 class NiconicoDmcFD(FileDownloader):
@@ -50,3 +59,93 @@ def heartbeat():
                     timer[0].cancel()
                     download_complete = True
         return success
+
+
+class NiconicoLiveFD(FileDownloader):
+    """ Downloads niconico live without being stopped """
+
+    def real_download(self, filename, info_dict):
+        video_id = info_dict['video_id']
+        ws_url = info_dict['url']
+        ws_extractor = info_dict['ws']
+        ws_origin_host = info_dict['origin']
+        cookies = info_dict.get('cookies')
+        live_quality = info_dict.get('live_quality', 'high')
+        live_latency = info_dict.get('live_latency', 'high')
+        dl = FFmpegFD(self.ydl, self.params or {})
+
+        new_info_dict = info_dict.copy()
+        new_info_dict.update({
+            'protocol': 'm3u8',
+        })
+
+        def communicate_ws(reconnect):
+            if reconnect:
+                ws = WebSocketsWrapper(ws_url, {
+                    'Cookies': str_or_none(cookies) or '',
+                    'Origin': f'https://{ws_origin_host}',
+                    'Accept': '*/*',
+                    'User-Agent': self.params['http_headers']['User-Agent'],
+                })
+                if self.ydl.params.get('verbose', False):
+                    self.to_screen('[debug] Sending startWatching request')
+                ws.send(json.dumps({
+                    'type': 'startWatching',
+                    'data': {
+                        'stream': {
+                            'quality': live_quality,
+                            'protocol': 'hls+fmp4',
+                            'latency': live_latency,
+                            'chasePlay': False
+                        },
+                        'room': {
+                            'protocol': 'webSocket',
+                            'commentable': True
+                        },
+                        'reconnect': True,
+                    }
+                }))
+            else:
+                ws = ws_extractor
+            with ws:
+                while True:
+                    recv = ws.recv()
+                    if not recv:
+                        continue
+                    data = json.loads(recv)
+                    if not data or not isinstance(data, dict):
+                        continue
+                    if data.get('type') == 'ping':
+                        # pong back
+                        ws.send(r'{"type":"pong"}')
+                        ws.send(r'{"type":"keepSeat"}')
+                    elif data.get('type') == 'disconnect':
+                        self.write_debug(data)
+                        return True
+                    elif data.get('type') == 'error':
+                        self.write_debug(data)
+                        message = try_get(data, lambda x: x['body']['code'], str) or recv
+                        return DownloadError(message)
+                    elif self.ydl.params.get('verbose', False):
+                        if len(recv) > 100:
+                            recv = recv[:100] + '...'
+                        self.to_screen('[debug] Server said: %s' % recv)
+
+        def ws_main():
+            reconnect = False
+            while True:
+                try:
+                    ret = communicate_ws(reconnect)
+                    if ret is True:
+                        return
+                except BaseException as e:
+                    self.to_screen('[%s] %s: Connection error occured, reconnecting after 10 seconds: %s' % ('niconico:live', video_id, str_or_none(e)))
+                    time.sleep(10)
+                    continue
+                finally:
+                    reconnect = True
+
+        thread = threading.Thread(target=ws_main, daemon=True)
+        thread.start()
+
+        return dl.download(filename, new_info_dict)
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index d560ed91c..07249bba6 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1275,6 +1275,7 @@
     NicovideoSearchIE,
     NicovideoSearchURLIE,
     NicovideoTagURLIE,
+    NiconicoLiveIE,
 )
 from .ninecninemedia import (
     NineCNineMediaIE,
diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py
index 30b4d7216..89e8e6093 100644
--- a/yt_dlp/extractor/niconico.py
+++ b/yt_dlp/extractor/niconico.py
@@ -5,13 +5,17 @@
 import re
 import time
 
+from urllib.parse import urlparse
+
 from .common import InfoExtractor, SearchInfoExtractor
 from ..compat import (
     compat_HTTPError,
 )
+from ..dependencies import websockets
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
+    WebSocketsWrapper,
     bug_reports_message,
     clean_html,
     float_or_none,
@@ -895,3 +899,162 @@ def _entries(self, list_id):
     def _real_extract(self, url):
         list_id = self._match_id(url)
         return self.playlist_result(self._entries(list_id), list_id, ie=NiconicoIE.ie_key())
+
+
+class NiconicoLiveIE(InfoExtractor):
+    IE_NAME = 'niconico:live'
+    IE_DESC = 'ニコニコ生放送'
+    _VALID_URL = r'https?://(?:sp\.)?live2?\.nicovideo\.jp/(?:watch|gate)/(?P<id>lv\d+)'
+    _TESTS = [{
+        'note': 'this test case includes invisible characters for title, pasting them as-is',
+        'url': 'https://live.nicovideo.jp/watch/lv339533123',
+        'info_dict': {
+            'id': 'lv339533123',
+            'title': '激辛ペヤング食べます‪( ;ᯅ; )‬（歌枠オーディション参加中）',
+            'view_count': 1526,
+            'comment_count': 1772,
+            'description': '初めましてもかって言います❕\nのんびり自由に適当に暮らしてます',
+            'uploader': 'もか',
+            'channel': 'ゲストさんのコミュニティ',
+            'channel_id': 'co5776900',
+            'channel_url': 'https://com.nicovideo.jp/community/co5776900',
+            'timestamp': 1670677328,
+            'is_live': True,
+        },
+        'skip': 'livestream',
+    }, {
+        'url': 'https://live2.nicovideo.jp/watch/lv339533123',
+        'only_matching': True,
+    }, {
+        'url': 'https://sp.live.nicovideo.jp/watch/lv339533123',
+        'only_matching': True,
+    }, {
+        'url': 'https://sp.live2.nicovideo.jp/watch/lv339533123',
+        'only_matching': True,
+    }]
+
+    _KNOWN_LATENCY = ('high', 'low')
+
+    def _real_extract(self, url):
+        if not websockets:
+            raise ExtractorError('websockets library is not available. Please install it.', expected=True)
+        video_id = self._match_id(url)
+        webpage, urlh = self._download_webpage_handle(f'https://live.nicovideo.jp/watch/{video_id}', video_id)
+
+        embedded_data = self._parse_json(unescapeHTML(self._search_regex(
+            r'<script\s+id="embedded-data"\s*data-props="(.+?)"', webpage, 'embedded data')), video_id)
+
+        ws_url = traverse_obj(embedded_data, ('site', 'relive', 'webSocketUrl'))
+        if not ws_url:
+            raise ExtractorError('The live hasn\'t started yet or already ended.', expected=True)
+        ws_url = update_url_query(ws_url, {
+            'frontend_id': traverse_obj(embedded_data, ('site', 'frontendId')) or '9',
+        })
+
+        hostname = remove_start(urlparse(urlh.geturl()).hostname, 'sp.')
+        cookies = try_get(urlh.geturl(), self._downloader._calc_cookies)
+        latency = try_get(self._configuration_arg('latency'), lambda x: x[0])
+        if latency not in self._KNOWN_LATENCY:
+            latency = 'high'
+
+        ws = WebSocketsWrapper(ws_url, {
+            'Cookies': str_or_none(cookies) or '',
+            'Origin': f'https://{hostname}',
+            'Accept': '*/*',
+            'User-Agent': self.get_param('http_headers')['User-Agent'],
+        })
+
+        self.write_debug('[debug] Sending HLS server request')
+        ws.send(json.dumps({
+            'type': 'startWatching',
+            'data': {
+                'stream': {
+                    'quality': 'abr',
+                    'protocol': 'hls+fmp4',
+                    'latency': latency,
+                    'chasePlay': False
+                },
+                'room': {
+                    'protocol': 'webSocket',
+                    'commentable': True
+                },
+                'reconnect': False,
+            }
+        }))
+
+        while True:
+            recv = ws.recv()
+            if not recv:
+                continue
+            data = json.loads(recv)
+            if not isinstance(data, dict):
+                continue
+            if data.get('type') == 'stream':
+                m3u8_url = data['data']['uri']
+                qualities = data['data']['availableQualities']
+                break
+            elif data.get('type') == 'disconnect':
+                self.write_debug(recv)
+                raise ExtractorError('Disconnected at middle of extraction')
+            elif data.get('type') == 'error':
+                self.write_debug(recv)
+                message = traverse_obj(data, ('body', 'code')) or recv
+                raise ExtractorError(message)
+            elif self.get_param('verbose', False):
+                if len(recv) > 100:
+                    recv = recv[:100] + '...'
+                self.write_debug('Server said: %s' % recv)
+
+        title = traverse_obj(embedded_data, ('program', 'title')) or self._html_search_meta(
+            ('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
+
+        raw_thumbs = traverse_obj(embedded_data, ('program', 'thumbnail')) or {}
+        thumbnails = []
+        for name, value in raw_thumbs.items():
+            if not isinstance(value, dict):
+                thumbnails.append({
+                    'id': name,
+                    'url': value,
+                    **parse_resolution(value, lenient=True),
+                })
+                continue
+
+            for k, img_url in value.items():
+                res = parse_resolution(k, lenient=True) or parse_resolution(img_url, lenient=True)
+                width, height = res.get('width'), res.get('height')
+
+                thumbnails.append({
+                    'id': f'{name}_{width}x{height}',
+                    'url': img_url,
+                    **res,
+                })
+
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4', live=True)
+        for fmt, q in zip(formats, reversed(qualities[1:])):
+            fmt.update({
+                'format_id': q,
+                'protocol': 'niconico_live',
+                'ws': ws,
+                'video_id': video_id,
+                'cookies': cookies,
+                'live_latency': latency,
+                'origin': hostname,
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            **traverse_obj(embedded_data, {
+                'view_count': ('program', 'statistics', 'watchCount'),
+                'comment_count': ('program', 'statistics', 'commentCount'),
+                'uploader': ('program', 'supplier', 'name'),
+                'channel': ('socialGroup', 'name'),
+                'channel_id': ('socialGroup', 'id'),
+                'channel_url': ('socialGroup', 'socialGroupPageUrl'),
+            }),
+            'description': clean_html(traverse_obj(embedded_data, ('program', 'description'))),
+            'timestamp': int_or_none(traverse_obj(embedded_data, ('program', 'openTime'))),
+            'is_live': True,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From fd5d93f7040f9776fd541f4e4079dad7d3b3fb4f Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Mon, 29 May 2023 04:42:03 -0500
Subject: [PATCH 303/871] Bugfix for b844a3f8b16500663e7ab6c6ec061cc9b30f71ac

[extractor/weverse] Avoid unnecessary duplicate login

Authored by: bashonly
---
 yt_dlp/extractor/weverse.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/yt_dlp/extractor/weverse.py b/yt_dlp/extractor/weverse.py
index ab629c885..8f2a7ee06 100644
--- a/yt_dlp/extractor/weverse.py
+++ b/yt_dlp/extractor/weverse.py
@@ -34,6 +34,9 @@ class WeverseBaseIE(InfoExtractor):
     }
 
     def _perform_login(self, username, password):
+        if self._API_HEADERS.get('Authorization'):
+            return
+
         headers = {
             'x-acc-app-secret': '5419526f1c624b38b10787e5c10b2a7a',
             'x-acc-app-version': '2.2.6',

From f6e43d6fa9804c24525e1fed0a87782754dab7ed Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Mon, 29 May 2023 05:07:35 -0500
Subject: [PATCH 304/871] [extractor/cbsnews] Overhaul extractors (#6681)

Closes #6565
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   8 +-
 yt_dlp/extractor/anvato.py      |  10 +-
 yt_dlp/extractor/cbslocal.py    | 116 ----------
 yt_dlp/extractor/cbsnews.py     | 382 ++++++++++++++++++++++++++++----
 4 files changed, 346 insertions(+), 170 deletions(-)
 delete mode 100644 yt_dlp/extractor/cbslocal.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 07249bba6..31af6bd3f 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -313,14 +313,14 @@
     CBSIE,
     ParamountPressExpressIE,
 )
-from .cbslocal import (
-    CBSLocalIE,
-    CBSLocalArticleIE,
-)
 from .cbsinteractive import CBSInteractiveIE
 from .cbsnews import (
     CBSNewsEmbedIE,
     CBSNewsIE,
+    CBSLocalIE,
+    CBSLocalArticleIE,
+    CBSLocalLiveIE,
+    CBSNewsLiveIE,
     CBSNewsLiveVideoIE,
 )
 from .cbssports import (
diff --git a/yt_dlp/extractor/anvato.py b/yt_dlp/extractor/anvato.py
index 79bfe412b..0df50333c 100644
--- a/yt_dlp/extractor/anvato.py
+++ b/yt_dlp/extractor/anvato.py
@@ -336,7 +336,7 @@ def _get_anvato_videos(self, access_key, video_id, token):
             elif media_format == 'm3u8-variant' or ext == 'm3u8':
                 # For some videos the initial m3u8 URL returns JSON instead
                 manifest_json = self._download_json(
-                    video_url, video_id, note='Downloading manifest JSON', errnote=False)
+                    video_url, video_id, note='Downloading manifest JSON', fatal=False)
                 if manifest_json:
                     video_url = manifest_json.get('master_m3u8')
                     if not video_url:
@@ -392,14 +392,6 @@ def _extract_from_webpage(cls, url, webpage):
                 url = smuggle_url(url, {'token': anvplayer_data['token']})
             yield cls.url_result(url, AnvatoIE, video_id)
 
-    def _extract_anvato_videos(self, webpage, video_id):
-        anvplayer_data = self._parse_json(
-            self._html_search_regex(
-                self._ANVP_RE, webpage, 'Anvato player data', group='anvp'),
-            video_id)
-        return self._get_anvato_videos(
-            anvplayer_data['accessKey'], anvplayer_data['video'], 'default')  # cbslocal token = 'default'
-
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
         self._initialize_geo_bypass({
diff --git a/yt_dlp/extractor/cbslocal.py b/yt_dlp/extractor/cbslocal.py
deleted file mode 100644
index 3d50b0499..000000000
--- a/yt_dlp/extractor/cbslocal.py
+++ /dev/null
@@ -1,116 +0,0 @@
-from .anvato import AnvatoIE
-from .sendtonews import SendtoNewsIE
-from ..compat import compat_urlparse
-from ..utils import (
-    parse_iso8601,
-    unified_timestamp,
-)
-
-
-class CBSLocalIE(AnvatoIE):  # XXX: Do not subclass from concrete IE
-    _VALID_URL_BASE = r'https?://[a-z]+\.cbslocal\.com/'
-    _VALID_URL = _VALID_URL_BASE + r'video/(?P<id>\d+)'
-
-    _TESTS = [{
-        'url': 'http://newyork.cbslocal.com/video/3580809-a-very-blue-anniversary/',
-        'info_dict': {
-            'id': '3580809',
-            'ext': 'mp4',
-            'title': 'A Very Blue Anniversary',
-            'description': 'CBS2’s Cindy Hsu has more.',
-            'thumbnail': 're:^https?://.*',
-            'timestamp': int,
-            'upload_date': r're:^\d{8}$',
-            'uploader': 'CBS',
-            'subtitles': {
-                'en': 'mincount:5',
-            },
-            'categories': [
-                'Stations\\Spoken Word\\WCBSTV',
-                'Syndication\\AOL',
-                'Syndication\\MSN',
-                'Syndication\\NDN',
-                'Syndication\\Yahoo',
-                'Content\\News',
-                'Content\\News\\Local News',
-            ],
-            'tags': ['CBS 2 News Weekends', 'Cindy Hsu', 'Blue Man Group'],
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    def _real_extract(self, url):
-        mcp_id = self._match_id(url)
-        return self.url_result(
-            'anvato:anvato_cbslocal_app_web_prod_547f3e49241ef0e5d30c79b2efbca5d92c698f67:' + mcp_id, 'Anvato', mcp_id)
-
-
-class CBSLocalArticleIE(AnvatoIE):  # XXX: Do not subclass from concrete IE
-    _VALID_URL = CBSLocalIE._VALID_URL_BASE + r'\d+/\d+/\d+/(?P<id>[0-9a-z-]+)'
-
-    _TESTS = [{
-        # Anvato backend
-        'url': 'http://losangeles.cbslocal.com/2016/05/16/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis',
-        'md5': 'f0ee3081e3843f575fccef901199b212',
-        'info_dict': {
-            'id': '3401037',
-            'ext': 'mp4',
-            'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
-            'description': 'Collapsing seats have been the focus of scrutiny for decades, though experts say remarkably little has been done to address the issue. Randy Paige reports.',
-            'thumbnail': 're:^https?://.*',
-            'timestamp': 1463440500,
-            'upload_date': '20160516',
-            'uploader': 'CBS',
-            'subtitles': {
-                'en': 'mincount:5',
-            },
-            'categories': [
-                'Stations\\Spoken Word\\KCBSTV',
-                'Syndication\\MSN',
-                'Syndication\\NDN',
-                'Syndication\\AOL',
-                'Syndication\\Yahoo',
-                'Syndication\\Tribune',
-                'Syndication\\Curb.tv',
-                'Content\\News'
-            ],
-            'tags': ['CBS 2 News Evening'],
-        },
-    }, {
-        # SendtoNews embed
-        'url': 'http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/',
-        'info_dict': {
-            'id': 'GxfCe0Zo7D-175909-5588',
-        },
-        'playlist_count': 9,
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        sendtonews_url = SendtoNewsIE._extract_url(webpage)
-        if sendtonews_url:
-            return self.url_result(
-                compat_urlparse.urljoin(url, sendtonews_url),
-                ie=SendtoNewsIE.ie_key())
-
-        info_dict = self._extract_anvato_videos(webpage, display_id)
-
-        timestamp = unified_timestamp(self._html_search_regex(
-            r'class="(?:entry|post)-date"[^>]*>([^<]+)', webpage,
-            'released date', default=None)) or parse_iso8601(
-            self._html_search_meta('uploadDate', webpage))
-
-        info_dict.update({
-            'display_id': display_id,
-            'timestamp': timestamp,
-        })
-
-        return info_dict
diff --git a/yt_dlp/extractor/cbsnews.py b/yt_dlp/extractor/cbsnews.py
index 16edf3af8..65ecc62f0 100644
--- a/yt_dlp/extractor/cbsnews.py
+++ b/yt_dlp/extractor/cbsnews.py
@@ -1,36 +1,153 @@
+import base64
 import re
+import urllib.error
+import urllib.parse
 import zlib
 
+from .anvato import AnvatoIE
 from .common import InfoExtractor
-from .cbs import CBSIE
-from ..compat import (
-    compat_b64decode,
-    compat_urllib_parse_unquote,
-)
+from .paramountplus import ParamountPlusIE
 from ..utils import (
+    ExtractorError,
+    HEADRequest,
+    UserNotLive,
+    determine_ext,
+    float_or_none,
+    format_field,
+    int_or_none,
+    make_archive_id,
+    mimetype2ext,
     parse_duration,
+    smuggle_url,
+    traverse_obj,
+    url_or_none,
 )
 
 
-class CBSNewsEmbedIE(CBSIE):  # XXX: Do not subclass from concrete IE
+class CBSNewsBaseIE(InfoExtractor):
+    _LOCALES = {
+        'atlanta': None,
+        'baltimore': 'BAL',
+        'boston': 'BOS',
+        'chicago': 'CHI',
+        'colorado': 'DEN',
+        'detroit': 'DET',
+        'losangeles': 'LA',
+        'miami': 'MIA',
+        'minnesota': 'MIN',
+        'newyork': 'NY',
+        'philadelphia': 'PHI',
+        'pittsburgh': 'PIT',
+        'sacramento': 'SAC',
+        'sanfrancisco': 'SF',
+        'texas': 'DAL',
+    }
+    _LOCALE_RE = '|'.join(map(re.escape, _LOCALES))
+    _ANVACK = '5VD6Eyd6djewbCmNwBFnsJj17YAvGRwl'
+
+    def _get_item(self, webpage, display_id):
+        return traverse_obj(self._search_json(
+            r'CBSNEWS\.defaultPayload\s*=', webpage, 'payload', display_id,
+            default={}), ('items', 0, {dict})) or {}
+
+    def _get_video_url(self, item):
+        return traverse_obj(item, 'video', 'video2', expected_type=url_or_none)
+
+    def _extract_playlist(self, webpage, playlist_id):
+        entries = [self.url_result(embed_url, CBSNewsEmbedIE) for embed_url in re.findall(
+            r'<iframe[^>]+data-src="(https?://(?:www\.)?cbsnews\.com/embed/video/[^#]*#[^"]+)"', webpage)]
+        if entries:
+            return self.playlist_result(
+                entries, playlist_id, self._html_search_meta(['og:title', 'twitter:title'], webpage),
+                self._html_search_meta(['og:description', 'twitter:description', 'description'], webpage))
+
+    def _extract_video(self, item, video_url, video_id):
+        if mimetype2ext(item.get('format'), default=determine_ext(video_url)) == 'mp4':
+            formats = [{'url': video_url, 'ext': 'mp4'}]
+
+        else:
+            manifest = self._download_webpage(video_url, video_id, note='Downloading m3u8 information')
+
+            anvato_id = self._search_regex(r'anvato-(\d+)', manifest, 'Anvato ID', default=None)
+            # Prefer Anvato if available; cbsnews.com m3u8 formats are re-encoded from Anvato source
+            if anvato_id:
+                return self.url_result(
+                    smuggle_url(f'anvato:{self._ANVACK}:{anvato_id}', {'token': 'default'}),
+                    AnvatoIE, url_transparent=True, _old_archive_ids=[make_archive_id(self, anvato_id)])
+
+            formats, _ = self._parse_m3u8_formats_and_subtitles(
+                manifest, video_url, 'mp4', m3u8_id='hls', video_id=video_id)
+
+        def get_subtitles(subs_url):
+            return {
+                'en': [{
+                    'url': subs_url,
+                    'ext': 'dfxp',  # TTAF1
+                }],
+            } if url_or_none(subs_url) else None
+
+        episode_meta = traverse_obj(item, {
+            'season_number': ('season', {int_or_none}),
+            'episode_number': ('episode', {int_or_none}),
+        }) if item.get('isFullEpisode') else {}
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            **traverse_obj(item, {
+                'title': (None, ('fulltitle', 'title')),
+                'description': 'dek',
+                'timestamp': ('timestamp', {lambda x: float_or_none(x, 1000)}),
+                'duration': ('duration', {float_or_none}),
+                'subtitles': ('captions', {get_subtitles}),
+                'thumbnail': ('images', ('hd', 'sd'), {url_or_none}),
+                'is_live': ('type', {lambda x: x == 'live'}),
+            }, get_all=False),
+            **episode_meta,
+        }
+
+
+class CBSNewsEmbedIE(CBSNewsBaseIE):
     IE_NAME = 'cbsnews:embed'
     _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/embed/video[^#]*#(?P<id>.+)'
     _TESTS = [{
         'url': 'https://www.cbsnews.com/embed/video/?v=1.c9b5b61492913d6660db0b2f03579ef25e86307a#1Vb7b9s2EP5XBAHbT6Gt98PAMKTJ0se6LVjWYWtdGBR1stlIpEBSTtwi%2F%2FvuJNkNhmHdGxgM2NL57vjd6zt%2B8PngdN%2Fyg79qeGvhzN%2FLGrS%2F%2BuBLB531V28%2B%2BO7Qg7%2Fy97r2z3xZ42NW8yLhDbA0S0KWlHnIijwKWJBHZZnHBa8Cgbpdf%2F89NM9Hi9fXifhpr8sr%2FlP848tn%2BTdXycX25zh4cdX%2FvHl6PmmPqnWQv9w8Ed%2B9GjYRim07bFEqdG%2BZVHuwTm65A7bVRrYtR5lAyMox7pigF6W4k%2By91mjspGsJ%2BwVae4%2BsvdnaO1p73HkXs%2FVisUDTGm7R8IcdnOROeq%2B19qT1amhA1VJtPenoTUgrtfKc9m7Rq8dP7nnjwOB7wg7ADdNt7VX64DWAWlKhPtmDEq22g4GF99x6Dk9E8OSsankHXqPNKDxC%2FdK7MLKTircTDgsI3mmj4OBdSq64dy7fd1x577RU1rt4cvMtOaulFYOd%2FLewRWvDO9lIgXFpZSnkZmjbv5SxKTPoQXClFbpsf%2Fhbbpzs0IB3vb8KkyzJQ%2BywOAgCrMpgRrz%2BKk4fvb7kFbR4XJCu0gAdtNO7woCwZTu%2BBUs9bam%2Fds71drVerpeisgrubLjAB4nnOSkWQnfr5W6o1ku5Xpr1MgrCbL0M0vUyDtfLLK15WiYp47xKWSLyjFVpwVmVJSLIoCjSOFkv3W7oKsVliwZJcB9nwXpZ5GEQQwY8jNKqKCBrgjTLeFxgdCIpazojDgnRtn43J6kG7nZ6cAbxh0EeFFk4%2B1u867cY5u4344n%2FxXjCqAjucdTHgLKojNKmSfO8KRsOFY%2FzKEYCKEJBzv90QA9nfm9gL%2BHulaFqUkz9ULUYxl62B3U%2FRVNLA8IhggaPycOoBuwOCESciDQVSSUgiOMsROB%2FhKfwCKOzEk%2B4k6rWd4uuT%2FwTDz7K7t3d3WLO8ISD95jSPQbayBacthbz86XVgxHwhex5zawzgDOmtp%2F3GPcXn0VXHdSS029%2Fj99UC%2FwJUvyKQ%2FzKyixIEVlYJOn4RxxuaH43Ty9fbJ5OObykHH435XAzJTHeOF4hhEUXD8URe%2FQ%2FBT%2BMpf8d5GN02Ox%2FfiGsl7TA7POu1xZ5%2BbTzcAVKMe48mqcC21hkacVEVScM26liVVBnrKkC4CLKyzAvHu0lhEaTKMFwI3a4SN9MsrfYzdBLq2vkwRD1gVviLT8kY9h2CHH6Y%2Bix6609weFtey4ESp60WtyeWMy%2BsmBuhsoKIyuoT%2Bq2R%2FrW5qi3g%2FvzS2j40DoixDP8%2BKP0yUdpXJ4l6Vla%2Bg9vce%2BC4yM5YlUcbA%2F0jLKdpmTwvsdN5z88nAIe08%2F0HgxeG1iv%2B6Hlhjh7uiW0SDzYNI92L401uha3JKYk268UVRzdOzNQvAaJqoXzAc80dAV440NZ1WVVAAMRYQ2KrGJFmDUsq8saWSnjvIj8t78y%2FRa3JRnbHVfyFpfwoDiGpPgjzekyUiKNlU3OMlwuLMmzgvEojllYVE2Z1HhImvsnk%2BuhusTEoB21PAtSFodeFK3iYhXEH9WOG2%2FkOE833sfeG%2Ff5cfHtEFNXgYes0%2FXj7aGivUgJ9XpusCtoNcNYVVnJVrrDo0OmJAutHCpuZul4W9lLcfy7BnuLPT02%2ByXsCTk%2B9zhzswIN04YueNSK%2BPtM0jS88QdLqSLJDTLsuGZJNolm2yO0PXh3UPnz9Ix5bfIAqxPjvETQsDCEiPG4QbqNyhBZISxybLnZYCrW5H3Axp690%2F0BJdXtDZ5ITuM4xj3f4oUHGzc5JeJmZKpp%2FjwKh4wMV%2FV1yx3emLoR0MwbG4K%2F%2BZgVep3PnzXGDHZ6a3i%2Fk%2BJrONDN13%2Bnq6tBTYk4o7cLGhBtqCC4KwacGHpEVuoH5JNro%2FE6JfE6d5RydbiR76k%2BW5wioDHBIjw1euhHjUGRB0y5A97KoaPx6MlL%2BwgboUVtUFRI%2FLemgTpdtF59ii7pab08kuPcfWzs0l%2FRI5takWnFpka0zOgWRtYcuf9aIxZMxlwr6IiGpsb6j2DQUXPl%2FimXI599Ev7fWjoPD78A',
-        'only_matching': True,
+        'info_dict': {
+            'id': '6ZP4cXvo9FaX3VLH7MF4CgY30JFpY_GA',
+            'ext': 'mp4',
+            'title': 'Cops investigate gorilla incident at Cincinnati Zoo',
+            'description': 'md5:fee7441ab8aaeb3c693482394738102b',
+            'duration': 350,
+            'timestamp': 1464719713,
+            'upload_date': '20160531',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):
-        item = self._parse_json(zlib.decompress(compat_b64decode(
-            compat_urllib_parse_unquote(self._match_id(url))),
-            -zlib.MAX_WBITS).decode('utf-8'), None)['video']['items'][0]
-        return self._extract_video_info(item['mpxRefId'], 'cbsnews')
+        item = traverse_obj(self._parse_json(zlib.decompress(base64.b64decode(
+            urllib.parse.unquote(self._match_id(url))),
+            -zlib.MAX_WBITS).decode(), None), ('video', 'items', 0, {dict})) or {}
+
+        video_id = item['mpxRefId']
+        video_url = self._get_video_url(item)
+        if not video_url:
+            # Old embeds redirect user to ParamountPlus but most links are 404
+            pplus_url = f'https://www.paramountplus.com/shows/video/{video_id}'
+            try:
+                self._request_webpage(HEADRequest(pplus_url), video_id)
+                return self.url_result(pplus_url, ParamountPlusIE)
+            except ExtractorError:
+                self.raise_no_formats('This video is no longer available', True, video_id)
+
+        return self._extract_video(item, video_url, video_id)
 
 
-class CBSNewsIE(CBSIE):  # XXX: Do not subclass from concrete IE
+class CBSNewsIE(CBSNewsBaseIE):
     IE_NAME = 'cbsnews'
     IE_DESC = 'CBS News'
-    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|video)/(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|video)/(?P<id>[\w-]+)'
 
     _TESTS = [
         {
@@ -47,10 +164,7 @@ class CBSNewsIE(CBSIE):  # XXX: Do not subclass from concrete IE
                 'timestamp': 1476046464,
                 'upload_date': '20161009',
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
+            'skip': 'This video is no longer available',
         },
         {
             'url': 'https://www.cbsnews.com/video/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
@@ -61,48 +175,234 @@ class CBSNewsIE(CBSIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:4a6983e480542d8b333a947bfc64ddc7',
                 'upload_date': '20140404',
                 'timestamp': 1396650660,
-                'uploader': 'CBSI-NEW',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 205,
                 'subtitles': {
                     'en': [{
-                        'ext': 'ttml',
+                        'ext': 'dfxp',
                     }],
                 },
             },
             'params': {
-                # m3u8 download
-                'skip_download': True,
+                'skip_download': 'm3u8',
             },
         },
         {
             # 48 hours
             'url': 'http://www.cbsnews.com/news/maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved/',
             'info_dict': {
+                'id': 'maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved',
                 'title': 'Cold as Ice',
                 'description': 'Can a childhood memory solve the 1957 murder of 7-year-old Maria Ridulph?',
             },
             'playlist_mincount': 7,
         },
+        {
+            'url': 'https://www.cbsnews.com/video/032823-cbs-evening-news/',
+            'info_dict': {
+                'id': '_2wuO7hD9LwtyM_TwSnVwnKp6kxlcXgE',
+                'ext': 'mp4',
+                'title': 'CBS Evening News, March 28, 2023',
+                'description': 'md5:db20615aae54adc1d55a1fd69dc75d13',
+                'duration': 1189,
+                'timestamp': 1680042600,
+                'upload_date': '20230328',
+                'season': 'Season 2023',
+                'season_number': 2023,
+                'episode': 'Episode 83',
+                'episode_number': 83,
+                'thumbnail': r're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': 'm3u8',
+            },
+        },
     ]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-
         webpage = self._download_webpage(url, display_id)
 
-        entries = []
-        for embed_url in re.findall(r'<iframe[^>]+data-src="(https?://(?:www\.)?cbsnews\.com/embed/video/[^#]*#[^"]+)"', webpage):
-            entries.append(self.url_result(embed_url, CBSNewsEmbedIE.ie_key()))
-        if entries:
-            return self.playlist_result(
-                entries, playlist_title=self._html_search_meta(['og:title', 'twitter:title'], webpage),
-                playlist_description=self._html_search_meta(['og:description', 'twitter:description', 'description'], webpage))
+        playlist = self._extract_playlist(webpage, display_id)
+        if playlist:
+            return playlist
 
-        item = self._parse_json(self._html_search_regex(
-            r'CBSNEWS\.defaultPayload\s*=\s*({.+})',
-            webpage, 'video JSON info'), display_id)['items'][0]
-        return self._extract_video_info(item['mpxRefId'], 'cbsnews')
+        item = self._get_item(webpage, display_id)
+        video_id = item.get('mpxRefId') or display_id
+        video_url = self._get_video_url(item)
+        if not video_url:
+            self.raise_no_formats('No video content was found', expected=True, video_id=video_id)
+
+        return self._extract_video(item, video_url, video_id)
+
+
+class CBSLocalBaseIE(CBSNewsBaseIE):
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        item = self._get_item(webpage, display_id)
+        video_id = item.get('mpxRefId') or display_id
+        anvato_id = None
+        video_url = self._get_video_url(item)
+
+        if not video_url:
+            anv_params = self._search_regex(
+                r'<iframe[^>]+\bdata-src="https?://w3\.mp\.lura\.live/player/prod/v3/anvload\.html\?key=([^"]+)"',
+                webpage, 'Anvato URL', default=None)
+
+            if not anv_params:
+                playlist = self._extract_playlist(webpage, display_id)
+                if playlist:
+                    return playlist
+                self.raise_no_formats('No video content was found', expected=True, video_id=video_id)
+
+            anv_data = self._parse_json(base64.urlsafe_b64decode(f'{anv_params}===').decode(), video_id)
+            anvato_id = anv_data['v']
+            return self.url_result(
+                smuggle_url(f'anvato:{anv_data.get("anvack") or self._ANVACK}:{anvato_id}', {
+                    'token': anv_data.get('token') or 'default',
+                }), AnvatoIE, url_transparent=True, _old_archive_ids=[make_archive_id(self, anvato_id)])
+
+        return self._extract_video(item, video_url, video_id)
+
+
+class CBSLocalIE(CBSLocalBaseIE):
+    _VALID_URL = rf'https?://(?:www\.)?cbsnews\.com/(?:{CBSNewsBaseIE._LOCALE_RE})/(?:live/)?video/(?P<id>[\w-]+)'
+    _TESTS = [{
+        # Anvato video via defaultPayload JSON
+        'url': 'https://www.cbsnews.com/newyork/video/1st-cannabis-dispensary-opens-in-queens/',
+        'info_dict': {
+            'id': '6376747',
+            'ext': 'mp4',
+            'title': '1st cannabis dispensary opens in Queens',
+            'description': 'The dispensary is women-owned and located in Jamaica.',
+            'uploader': 'CBS',
+            'duration': 20,
+            'timestamp': 1680193657,
+            'upload_date': '20230330',
+            'categories': ['Stations\\Spoken Word\\WCBSTV', 'Content\\Google', 'Content\\News', 'Content\\News\\Local News'],
+            'tags': 'count:11',
+            'thumbnail': 're:^https?://.*',
+            '_old_archive_ids': ['cbslocal 6376747'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        # cbsnews.com video via defaultPayload JSON
+        'url': 'https://www.cbsnews.com/newyork/live/video/20230330171655-the-city-is-sounding-the-alarm-on-dangerous-social-media-challenges/',
+        'info_dict': {
+            'id': 'sJqfw7YvgSC6ant2zVmzt3y1jYKoL5J3',
+            'ext': 'mp4',
+            'title': 'the city is sounding the alarm on dangerous social media challenges',
+            'description': 'md5:8eccc9b1b73be5138a52e9c4350d2cd6',
+            'thumbnail': 'https://images-cbsn.cbsnews.com/prod/2023/03/30/story_22509622_1680196925.jpg',
+            'duration': 41.0,
+            'timestamp': 1680196615,
+            'upload_date': '20230330',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+
+class CBSLocalArticleIE(CBSLocalBaseIE):
+    _VALID_URL = rf'https?://(?:www\.)?cbsnews\.com/(?:{CBSNewsBaseIE._LOCALE_RE})/news/(?P<id>[\w-]+)'
+    _TESTS = [{
+        # Anvato video via iframe embed
+        'url': 'https://www.cbsnews.com/newyork/news/mta-station-agents-leaving-their-booths-to-provide-more-direct-customer-service/',
+        'playlist_count': 2,
+        'info_dict': {
+            'id': 'mta-station-agents-leaving-their-booths-to-provide-more-direct-customer-service',
+            'title': 'MTA station agents begin leaving their booths to provide more direct customer service',
+            'description': 'The more than 2,200 agents will provide face-to-face customer service to passengers.',
+        },
+    }, {
+        'url': 'https://www.cbsnews.com/losangeles/news/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis/',
+        'md5': 'f0ee3081e3843f575fccef901199b212',
+        'info_dict': {
+            'id': '3401037',
+            'ext': 'mp4',
+            'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
+            'thumbnail': 're:^https?://.*',
+            'timestamp': 1463440500,
+            'upload_date': '20160516',
+        },
+        'skip': 'Video has been removed',
+    }]
+
+
+class CBSNewsLiveBaseIE(CBSNewsBaseIE):
+    def _get_id(self, url):
+        raise NotImplementedError('This method must be implemented by subclasses')
+
+    def _real_extract(self, url):
+        video_id = self._get_id(url)
+        if not video_id:
+            raise ExtractorError('Livestream is not available', expected=True)
+
+        data = traverse_obj(self._download_json(
+            'https://feeds-cbsn.cbsnews.com/2.0/rundown/', video_id, query={
+                'partner': 'cbsnsite',
+                'edition': video_id,
+                'type': 'live',
+            }), ('navigation', 'data', 0, {dict}))
+
+        video_url = traverse_obj(data, (('videoUrlDAI', ('videoUrl', 'base')), {url_or_none}), get_all=False)
+        if not video_url:
+            raise UserNotLive(video_id=video_id)
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': True,
+            **traverse_obj(data, {
+                'title': 'headline',
+                'description': 'rundown_slug',
+                'thumbnail': ('images', 'thumbnail_url_hd', {url_or_none}),
+            }),
+        }
+
+
+class CBSLocalLiveIE(CBSNewsLiveBaseIE):
+    _VALID_URL = rf'https?://(?:www\.)?cbsnews\.com/(?P<id>{CBSNewsBaseIE._LOCALE_RE})/live/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://www.cbsnews.com/losangeles/live/',
+        'info_dict': {
+            'id': 'CBSN-LA',
+            'ext': 'mp4',
+            'title': str,
+            'description': r're:KCBS/CBSN_LA.CRISPIN.\w+.RUNDOWN \w+ \w+',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'live_status': 'is_live',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _get_id(self, url):
+        return format_field(self._LOCALES, self._match_id(url), 'CBSN-%s')
+
+
+class CBSNewsLiveIE(CBSNewsLiveBaseIE):
+    IE_NAME = 'cbsnews:live'
+    IE_DESC = 'CBS News Livestream'
+    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://www.cbsnews.com/live/',
+        'info_dict': {
+            'id': 'CBSN-US',
+            'ext': 'mp4',
+            'title': str,
+            'description': r're:\w+ \w+ CRISPIN RUNDOWN',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'live_status': 'is_live',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _get_id(self, url):
+        return 'CBSN-US'
 
 
 class CBSNewsLiveVideoIE(InfoExtractor):
@@ -111,7 +411,7 @@ class CBSNewsLiveVideoIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[^/?#]+)'
 
     # Live videos get deleted soon. See http://www.cbsnews.com/live/ for the latest examples
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
         'info_dict': {
             'id': 'clinton-sanders-prepare-to-face-off-in-nh',
@@ -120,7 +420,7 @@ class CBSNewsLiveVideoIE(InfoExtractor):
             'duration': 334,
         },
         'skip': 'Video gone',
-    }
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -131,13 +431,13 @@ def _real_extract(self, url):
                 'dvr_slug': display_id,
             })
 
-        formats = self._extract_akamai_formats(video_info['url'], display_id)
-
         return {
             'id': display_id,
             'display_id': display_id,
-            'title': video_info['headline'],
-            'thumbnail': video_info.get('thumbnail_url_hd') or video_info.get('thumbnail_url_sd'),
-            'duration': parse_duration(video_info.get('segmentDur')),
-            'formats': formats,
+            'formats': self._extract_akamai_formats(video_info['url'], display_id),
+            **traverse_obj(video_info, {
+                'title': 'headline',
+                'thumbnail': ('thumbnail_url_hd', {url_or_none}),
+                'duration': ('segmentDur', {parse_duration}),
+            }),
         }

From 2d306c03d6f2697fcbabb7da35aa62cc078359d3 Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Mon, 29 May 2023 06:17:29 -0400
Subject: [PATCH 305/871] [extractor/rottentomatoes] Fix extractor (#6844)

Closes #6729
Authored by: JChris246
---
 yt_dlp/extractor/rottentomatoes.py | 82 ++++++++++++++++++++++++------
 1 file changed, 66 insertions(+), 16 deletions(-)

diff --git a/yt_dlp/extractor/rottentomatoes.py b/yt_dlp/extractor/rottentomatoes.py
index f133c851b..e35717522 100644
--- a/yt_dlp/extractor/rottentomatoes.py
+++ b/yt_dlp/extractor/rottentomatoes.py
@@ -1,30 +1,80 @@
 from .common import InfoExtractor
-from .internetvideoarchive import InternetVideoArchiveIE
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    float_or_none,
+    get_element_by_class,
+    join_nonempty,
+    traverse_obj,
+    url_or_none,
+)
 
 
 class RottenTomatoesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rottentomatoes\.com/m/[^/]+/trailers/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rottentomatoes\.com/m/(?P<playlist>[^/]+)(?:/(?P<tr>trailers)(?:/(?P<id>\w+))?)?'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
         'info_dict': {
             'id': '11028566',
             'ext': 'mp4',
             'title': 'Toy Story 3',
-            'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.'
         },
-    }
+        'skip': 'No longer available',
+    }, {
+        'url': 'https://www.rottentomatoes.com/m/toy_story_3/trailers/VycaVoBKhGuk',
+        'info_dict': {
+            'id': 'VycaVoBKhGuk',
+            'ext': 'mp4',
+            'title': 'Toy Story 3: Trailer 2',
+            'description': '',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 149.941
+        },
+    }, {
+        'url': 'http://www.rottentomatoes.com/m/toy_story_3',
+        'info_dict': {
+            'id': 'toy_story_3',
+            'title': 'Toy Story 3',
+        },
+        'playlist_mincount': 4,
+    }, {
+        'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers',
+        'info_dict': {
+            'id': 'toy_story_3-trailers',
+        },
+        'playlist_mincount': 5,
+    }]
+
+    def _extract_videos(self, data, display_id):
+        for video in traverse_obj(data, (lambda _, v: v['publicId'] and v['file'] and v['type'] == 'hls')):
+            yield {
+                'formats': self._extract_m3u8_formats(
+                    video['file'], display_id, 'mp4', m3u8_id='hls', fatal=False),
+                **traverse_obj(video, {
+                    'id': 'publicId',
+                    'title': 'title',
+                    'description': 'description',
+                    'duration': ('durationInSeconds', {float_or_none}),
+                    'thumbnail': ('image', {url_or_none}),
+                }),
+            }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        iva_id = self._search_regex(r'publishedid=(\d+)', webpage, 'internet video archive id')
+        playlist_id, trailers, video_id = self._match_valid_url(url).group('playlist', 'tr', 'id')
+        playlist_id = join_nonempty(playlist_id, trailers)
+        webpage = self._download_webpage(url, playlist_id)
+        data = self._search_json(
+            r'<script[^>]+\bid=["\'](?:heroV|v)ideos["\'][^>]*>', webpage,
+            'data', playlist_id, contains_pattern=r'\[{(?s:.+)}\]')
 
-        return {
-            '_type': 'url_transparent',
-            'url': 'http://video.internetvideoarchive.net/player/6/configuration.ashx?domain=www.videodetective.com&customerid=69249&playerid=641&publishedid=' + iva_id,
-            'ie_key': InternetVideoArchiveIE.ie_key(),
-            'id': video_id,
-            'title': self._og_search_title(webpage),
-        }
+        if video_id:
+            video_data = traverse_obj(data, lambda _, v: v['publicId'] == video_id)
+            if not video_data:
+                raise ExtractorError('Unable to extract video from webpage')
+            return next(self._extract_videos(video_data, video_id))
+
+        return self.playlist_result(
+            self._extract_videos(data, playlist_id), playlist_id,
+            clean_html(get_element_by_class('scoreboard__title', webpage)))

From 489f51279d00318018478fd7461eddbe3b45297e Mon Sep 17 00:00:00 2001
From: hasezoey <hasezoey@gmail.com>
Date: Mon, 29 May 2023 12:52:01 +0200
Subject: [PATCH 306/871] [extractor/nekohacker] Add extractor (#7003)

Authored by: hasezoey
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/nekohacker.py  | 217 ++++++++++++++++++++++++++++++++
 2 files changed, 218 insertions(+)
 create mode 100644 yt_dlp/extractor/nekohacker.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 31af6bd3f..f9fa84c43 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1206,6 +1206,7 @@
     NebulaSubscriptionsIE,
     NebulaChannelIE,
 )
+from .nekohacker import NekoHackerIE
 from .nerdcubed import NerdCubedFeedIE
 from .netzkino import NetzkinoIE
 from .neteasemusic import (
diff --git a/yt_dlp/extractor/nekohacker.py b/yt_dlp/extractor/nekohacker.py
new file mode 100644
index 000000000..e10ffe925
--- /dev/null
+++ b/yt_dlp/extractor/nekohacker.py
@@ -0,0 +1,217 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    extract_attributes,
+    get_element_by_class,
+    get_element_text_and_html_by_tag,
+    parse_duration,
+    traverse_obj,
+    try_call,
+    url_or_none,
+)
+
+
+class NekoHackerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>(?!free-dl)[\w-]+)'
+    _TESTS = [{
+        'url': 'https://nekohacker.com/nekoverse/',
+        'info_dict': {
+            'id': 'nekoverse',
+            'title': 'Nekoverse',
+        },
+        'playlist': [
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/01-Spaceship.mp3',
+                'md5': '44223701ebedba0467ebda4cc07fb3aa',
+                'info_dict': {
+                    'id': '1712',
+                    'ext': 'mp3',
+                    'title': 'Spaceship',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20221101',
+                    'album': 'Nekoverse',
+                    'artist': 'Neko Hacker',
+                    'track': 'Spaceship',
+                    'track_number': 1,
+                    'duration': 195.0
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/02-City-Runner.mp3',
+                'md5': '8f853c71719389d32bbbd3f1a87b3f08',
+                'info_dict': {
+                    'id': '1713',
+                    'ext': 'mp3',
+                    'title': 'City Runner',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20221101',
+                    'album': 'Nekoverse',
+                    'artist': 'Neko Hacker',
+                    'track': 'City Runner',
+                    'track_number': 2,
+                    'duration': 148.0
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/03-Nature-Talk.mp3',
+                'md5': '5a8a8ae852720cee4c0ac95c7d1a7450',
+                'info_dict': {
+                    'id': '1714',
+                    'ext': 'mp3',
+                    'title': 'Nature Talk',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20221101',
+                    'album': 'Nekoverse',
+                    'artist': 'Neko Hacker',
+                    'track': 'Nature Talk',
+                    'track_number': 3,
+                    'duration': 174.0
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/04-Crystal-World.mp3',
+                'md5': 'd8e59a48061764e50d92386a294abd50',
+                'info_dict': {
+                    'id': '1715',
+                    'ext': 'mp3',
+                    'title': 'Crystal World',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20221101',
+                    'album': 'Nekoverse',
+                    'artist': 'Neko Hacker',
+                    'track': 'Crystal World',
+                    'track_number': 4,
+                    'duration': 199.0
+                }
+            }
+        ]
+    }, {
+        'url': 'https://nekohacker.com/susume/',
+        'info_dict': {
+            'id': 'susume',
+            'title': '進め！むじなカンパニー',
+        },
+        'playlist': [
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2021/01/進め！むじなカンパニー-feat.-六科なじむ-CV_-日高里菜-割戶真友-CV_-金元寿子-軽井沢ユキ-CV_-上坂すみれ-出稼ぎガルシア-CV_-金子彩花-.mp3',
+                'md5': 'fb13f008aa81f26ba48f91fd2d6186ce',
+                'info_dict': {
+                    'id': '711',
+                    'ext': 'mp3',
+                    'title': 'md5:1a5fcbc96ca3c3265b1c6f9f79f30fd0',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20210115',
+                    'album': '進め！むじなカンパニー',
+                    'artist': 'Neko Hacker',
+                    'track': 'md5:1a5fcbc96ca3c3265b1c6f9f79f30fd0',
+                    'track_number': 1,
+                    'duration': None
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2021/01/むじな-de-なじむ-feat.-六科なじむ-CV_-日高里菜-.mp3',
+                'md5': '028803f70241df512b7764e73396fdd1',
+                'info_dict': {
+                    'id': '709',
+                    'ext': 'mp3',
+                    'title': 'むじな de なじむ feat. 六科なじむ (CV: 日高里菜 )',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20210115',
+                    'album': '進め！むじなカンパニー',
+                    'artist': 'Neko Hacker',
+                    'track': 'むじな de なじむ feat. 六科なじむ (CV: 日高里菜 )',
+                    'track_number': 2,
+                    'duration': None
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2021/01/進め！むじなカンパニー-instrumental.mp3',
+                'md5': 'adde9e9a16e1da5e602b579c247d0fb9',
+                'info_dict': {
+                    'id': '710',
+                    'ext': 'mp3',
+                    'title': '進め！むじなカンパニー (instrumental)',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20210115',
+                    'album': '進め！むじなカンパニー',
+                    'artist': 'Neko Hacker',
+                    'track': '進め！むじなカンパニー (instrumental)',
+                    'track_number': 3,
+                    'duration': None
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2021/01/むじな-de-なじむ-instrumental.mp3',
+                'md5': 'ebb0443039cf5f9ff7fd557ed9b23599',
+                'info_dict': {
+                    'id': '712',
+                    'ext': 'mp3',
+                    'title': 'むじな de なじむ (instrumental)',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20210115',
+                    'album': '進め！むじなカンパニー',
+                    'artist': 'Neko Hacker',
+                    'track': 'むじな de なじむ (instrumental)',
+                    'track_number': 4,
+                    'duration': None
+                }
+            }
+        ]
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+        playlist = get_element_by_class('playlist', webpage)
+
+        if not playlist:
+            iframe = try_call(lambda: get_element_text_and_html_by_tag('iframe', webpage)[1]) or ''
+            iframe_src = url_or_none(extract_attributes(iframe).get('src'))
+            if not iframe_src:
+                raise ExtractorError('No playlist or embed found in webpage')
+            elif re.match(r'https?://(?:\w+\.)?spotify\.com/', iframe_src):
+                raise ExtractorError('Spotify embeds are not supported', expected=True)
+            return self.url_result(url, 'Generic')
+
+        entries = []
+        for track_number, track in enumerate(re.findall(r'(<li[^>]+data-audiopath[^>]+>)', playlist), 1):
+            entry = traverse_obj(extract_attributes(track), {
+                'url': ('data-audiopath', {url_or_none}),
+                'ext': ('data-audiopath', {determine_ext}),
+                'id': 'data-trackid',
+                'title': 'data-tracktitle',
+                'track': 'data-tracktitle',
+                'album': 'data-albumtitle',
+                'duration': ('data-tracktime', {parse_duration}),
+                'release_date': ('data-releasedate', {lambda x: re.match(r'\d{8}', x.replace('.', ''))}, 0),
+                'thumbnail': ('data-albumart', {url_or_none}),
+            })
+            entries.append({
+                **entry,
+                'track_number': track_number,
+                'artist': 'Neko Hacker',
+                'vcodec': 'none',
+                'acodec': 'mp3' if entry['ext'] == 'mp3' else None,
+            })
+
+        return self.playlist_result(entries, playlist_id, traverse_obj(entries, (0, 'album')))

From b38d4c941d1993ab27e4c0f8e024e23c2ec0f8f8 Mon Sep 17 00:00:00 2001
From: Matt Broadway <mattdbway@gmail.com>
Date: Mon, 29 May 2023 14:51:35 +0100
Subject: [PATCH 307/871] [cookies] Update for chromium changes (#6897)

Authored by: mbway
---
 test/test_cookies.py |  18 +++--
 yt_dlp/cookies.py    | 187 +++++++++++++++++++++++++++++++------------
 2 files changed, 150 insertions(+), 55 deletions(-)

diff --git a/test/test_cookies.py b/test/test_cookies.py
index 4155bcbf5..5282ef621 100644
--- a/test/test_cookies.py
+++ b/test/test_cookies.py
@@ -49,32 +49,38 @@ def test_get_desktop_environment(self):
         """ based on https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util_unittest.cc """
         test_cases = [
             ({}, _LinuxDesktopEnvironment.OTHER),
+            ({'DESKTOP_SESSION': 'my_custom_de'}, _LinuxDesktopEnvironment.OTHER),
+            ({'XDG_CURRENT_DESKTOP': 'my_custom_de'}, _LinuxDesktopEnvironment.OTHER),
 
             ({'DESKTOP_SESSION': 'gnome'}, _LinuxDesktopEnvironment.GNOME),
             ({'DESKTOP_SESSION': 'mate'}, _LinuxDesktopEnvironment.GNOME),
-            ({'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE),
-            ({'DESKTOP_SESSION': 'kde'}, _LinuxDesktopEnvironment.KDE),
+            ({'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE4),
+            ({'DESKTOP_SESSION': 'kde'}, _LinuxDesktopEnvironment.KDE3),
             ({'DESKTOP_SESSION': 'xfce'}, _LinuxDesktopEnvironment.XFCE),
 
             ({'GNOME_DESKTOP_SESSION_ID': 1}, _LinuxDesktopEnvironment.GNOME),
-            ({'KDE_FULL_SESSION': 1}, _LinuxDesktopEnvironment.KDE),
+            ({'KDE_FULL_SESSION': 1}, _LinuxDesktopEnvironment.KDE3),
+            ({'KDE_FULL_SESSION': 1, 'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE4),
 
             ({'XDG_CURRENT_DESKTOP': 'X-Cinnamon'}, _LinuxDesktopEnvironment.CINNAMON),
+            ({'XDG_CURRENT_DESKTOP': 'Deepin'}, _LinuxDesktopEnvironment.DEEPIN),
             ({'XDG_CURRENT_DESKTOP': 'GNOME'}, _LinuxDesktopEnvironment.GNOME),
             ({'XDG_CURRENT_DESKTOP': 'GNOME:GNOME-Classic'}, _LinuxDesktopEnvironment.GNOME),
             ({'XDG_CURRENT_DESKTOP': 'GNOME : GNOME-Classic'}, _LinuxDesktopEnvironment.GNOME),
 
             ({'XDG_CURRENT_DESKTOP': 'Unity', 'DESKTOP_SESSION': 'gnome-fallback'}, _LinuxDesktopEnvironment.GNOME),
-            ({'XDG_CURRENT_DESKTOP': 'KDE', 'KDE_SESSION_VERSION': '5'}, _LinuxDesktopEnvironment.KDE),
-            ({'XDG_CURRENT_DESKTOP': 'KDE'}, _LinuxDesktopEnvironment.KDE),
+            ({'XDG_CURRENT_DESKTOP': 'KDE', 'KDE_SESSION_VERSION': '5'}, _LinuxDesktopEnvironment.KDE5),
+            ({'XDG_CURRENT_DESKTOP': 'KDE', 'KDE_SESSION_VERSION': '6'}, _LinuxDesktopEnvironment.KDE6),
+            ({'XDG_CURRENT_DESKTOP': 'KDE'}, _LinuxDesktopEnvironment.KDE4),
             ({'XDG_CURRENT_DESKTOP': 'Pantheon'}, _LinuxDesktopEnvironment.PANTHEON),
+            ({'XDG_CURRENT_DESKTOP': 'UKUI'}, _LinuxDesktopEnvironment.UKUI),
             ({'XDG_CURRENT_DESKTOP': 'Unity'}, _LinuxDesktopEnvironment.UNITY),
             ({'XDG_CURRENT_DESKTOP': 'Unity:Unity7'}, _LinuxDesktopEnvironment.UNITY),
             ({'XDG_CURRENT_DESKTOP': 'Unity:Unity8'}, _LinuxDesktopEnvironment.UNITY),
         ]
 
         for env, expected_desktop_environment in test_cases:
-            self.assertEqual(_get_linux_desktop_environment(env), expected_desktop_environment)
+            self.assertEqual(_get_linux_desktop_environment(env, Logger()), expected_desktop_environment)
 
     def test_chrome_cookie_decryptor_linux_derive_key(self):
         key = LinuxChromeCookieDecryptor.derive_key(b'abc')
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index ee2af0f70..e46d19341 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -353,7 +353,9 @@ class ChromeCookieDecryptor:
         Linux:
         - cookies are either v10 or v11
             - v10: AES-CBC encrypted with a fixed key
+                - also attempts empty password if decryption fails
             - v11: AES-CBC encrypted with an OS protected key (keyring)
+                - also attempts empty password if decryption fails
             - v11 keys can be stored in various places depending on the activate desktop environment [2]
 
         Mac:
@@ -368,7 +370,7 @@ class ChromeCookieDecryptor:
 
     Sources:
     - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/
-    - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc
+    - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_linux.cc
         - KeyStorageLinux::CreateService
     """
 
@@ -390,6 +392,7 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
     def __init__(self, browser_keyring_name, logger, *, keyring=None):
         self._logger = logger
         self._v10_key = self.derive_key(b'peanuts')
+        self._empty_key = self.derive_key(b'')
         self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0}
         self._browser_keyring_name = browser_keyring_name
         self._keyring = keyring
@@ -402,25 +405,36 @@ def _v11_key(self):
     @staticmethod
     def derive_key(password):
         # values from
-        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
+        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_linux.cc
         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
 
     def decrypt(self, encrypted_value):
+        """
+
+        following the same approach as the fix in [1]: if cookies fail to decrypt then attempt to decrypt
+        with an empty password. The failure detection is not the same as what chromium uses so the
+        results won't be perfect
+
+        References:
+            - [1] https://chromium.googlesource.com/chromium/src/+/bbd54702284caca1f92d656fdcadf2ccca6f4165%5E%21/
+                - a bugfix to try an empty password as a fallback
+        """
         version = encrypted_value[:3]
         ciphertext = encrypted_value[3:]
 
         if version == b'v10':
             self._cookie_counts['v10'] += 1
-            return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
+            return _decrypt_aes_cbc_multi(ciphertext, (self._v10_key, self._empty_key), self._logger)
 
         elif version == b'v11':
             self._cookie_counts['v11'] += 1
             if self._v11_key is None:
                 self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True)
                 return None
-            return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
+            return _decrypt_aes_cbc_multi(ciphertext, (self._v11_key, self._empty_key), self._logger)
 
         else:
+            self._logger.warning(f'unknown cookie version: "{version}"', only_once=True)
             self._cookie_counts['other'] += 1
             return None
 
@@ -435,7 +449,7 @@ def __init__(self, browser_keyring_name, logger):
     @staticmethod
     def derive_key(password):
         # values from
-        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
+        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_mac.mm
         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
 
     def decrypt(self, encrypted_value):
@@ -448,12 +462,12 @@ def decrypt(self, encrypted_value):
                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
                 return None
 
-            return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
+            return _decrypt_aes_cbc_multi(ciphertext, (self._v10_key,), self._logger)
 
         else:
             self._cookie_counts['other'] += 1
             # other prefixes are considered 'old data' which were stored as plaintext
-            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
+            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_mac.mm
             return encrypted_value
 
 
@@ -473,7 +487,7 @@ def decrypt(self, encrypted_value):
                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
                 return None
 
-            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
+            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
             #   kNonceLength
             nonce_length = 96 // 8
             # boringssl
@@ -490,7 +504,7 @@ def decrypt(self, encrypted_value):
         else:
             self._cookie_counts['other'] += 1
             # any other prefix means the data is DPAPI encrypted
-            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
+            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
             return _decrypt_windows_dpapi(encrypted_value, self._logger).decode()
 
 
@@ -673,27 +687,35 @@ class _LinuxDesktopEnvironment(Enum):
     """
     OTHER = auto()
     CINNAMON = auto()
+    DEEPIN = auto()
     GNOME = auto()
-    KDE = auto()
+    KDE3 = auto()
+    KDE4 = auto()
+    KDE5 = auto()
+    KDE6 = auto()
     PANTHEON = auto()
+    UKUI = auto()
     UNITY = auto()
     XFCE = auto()
+    LXQT = auto()
 
 
 class _LinuxKeyring(Enum):
     """
-    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.h
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_util_linux.h
     SelectedLinuxBackend
     """
-    KWALLET = auto()
-    GNOMEKEYRING = auto()
-    BASICTEXT = auto()
+    KWALLET4 = auto()  # this value is just called KWALLET in the chromium source but it is for KDE4 only
+    KWALLET5 = auto()
+    KWALLET6 = auto()
+    GNOME_KEYRING = auto()
+    BASIC_TEXT = auto()
 
 
 SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
 
 
-def _get_linux_desktop_environment(env):
+def _get_linux_desktop_environment(env, logger):
     """
     https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.cc
     GetDesktopEnvironment
@@ -708,51 +730,97 @@ def _get_linux_desktop_environment(env):
                 return _LinuxDesktopEnvironment.GNOME
             else:
                 return _LinuxDesktopEnvironment.UNITY
+        elif xdg_current_desktop == 'Deepin':
+            return _LinuxDesktopEnvironment.DEEPIN
         elif xdg_current_desktop == 'GNOME':
             return _LinuxDesktopEnvironment.GNOME
         elif xdg_current_desktop == 'X-Cinnamon':
             return _LinuxDesktopEnvironment.CINNAMON
         elif xdg_current_desktop == 'KDE':
-            return _LinuxDesktopEnvironment.KDE
+            kde_version = env.get('KDE_SESSION_VERSION', None)
+            if kde_version == '5':
+                return _LinuxDesktopEnvironment.KDE5
+            elif kde_version == '6':
+                return _LinuxDesktopEnvironment.KDE6
+            elif kde_version == '4':
+                return _LinuxDesktopEnvironment.KDE4
+            else:
+                logger.info(f'unknown KDE version: "{kde_version}". Assuming KDE4')
+                return _LinuxDesktopEnvironment.KDE4
         elif xdg_current_desktop == 'Pantheon':
             return _LinuxDesktopEnvironment.PANTHEON
         elif xdg_current_desktop == 'XFCE':
             return _LinuxDesktopEnvironment.XFCE
+        elif xdg_current_desktop == 'UKUI':
+            return _LinuxDesktopEnvironment.UKUI
+        elif xdg_current_desktop == 'LXQt':
+            return _LinuxDesktopEnvironment.LXQT
+        else:
+            logger.info(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"')
+
     elif desktop_session is not None:
-        if desktop_session in ('mate', 'gnome'):
+        if desktop_session == 'deepin':
+            return _LinuxDesktopEnvironment.DEEPIN
+        elif desktop_session in ('mate', 'gnome'):
             return _LinuxDesktopEnvironment.GNOME
-        elif 'kde' in desktop_session:
-            return _LinuxDesktopEnvironment.KDE
-        elif 'xfce' in desktop_session:
+        elif desktop_session in ('kde4', 'kde-plasma'):
+            return _LinuxDesktopEnvironment.KDE4
+        elif desktop_session == 'kde':
+            if 'KDE_SESSION_VERSION' in env:
+                return _LinuxDesktopEnvironment.KDE4
+            else:
+                return _LinuxDesktopEnvironment.KDE3
+        elif 'xfce' in desktop_session or desktop_session == 'xubuntu':
             return _LinuxDesktopEnvironment.XFCE
+        elif desktop_session == 'ukui':
+            return _LinuxDesktopEnvironment.UKUI
+        else:
+            logger.info(f'DESKTOP_SESSION is set to an unknown value: "{desktop_session}"')
+
     else:
         if 'GNOME_DESKTOP_SESSION_ID' in env:
             return _LinuxDesktopEnvironment.GNOME
         elif 'KDE_FULL_SESSION' in env:
-            return _LinuxDesktopEnvironment.KDE
+            if 'KDE_SESSION_VERSION' in env:
+                return _LinuxDesktopEnvironment.KDE4
+            else:
+                return _LinuxDesktopEnvironment.KDE3
     return _LinuxDesktopEnvironment.OTHER
 
 
 def _choose_linux_keyring(logger):
     """
-    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.cc
-    SelectBackend
+    SelectBackend in [1]
+
+    There is currently support for forcing chromium to use BASIC_TEXT by creating a file called
+    `Disable Local Encryption` [1] in the user data dir. The function to write this file (`WriteBackendUse()` [1])
+    does not appear to be called anywhere other than in tests, so the user would have to create this file manually
+    and so would be aware enough to tell yt-dlp to use the BASIC_TEXT keyring.
+
+    References:
+        - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_util_linux.cc
     """
-    desktop_environment = _get_linux_desktop_environment(os.environ)
+    desktop_environment = _get_linux_desktop_environment(os.environ, logger)
     logger.debug(f'detected desktop environment: {desktop_environment.name}')
-    if desktop_environment == _LinuxDesktopEnvironment.KDE:
-        linux_keyring = _LinuxKeyring.KWALLET
-    elif desktop_environment == _LinuxDesktopEnvironment.OTHER:
-        linux_keyring = _LinuxKeyring.BASICTEXT
+    if desktop_environment == _LinuxDesktopEnvironment.KDE4:
+        linux_keyring = _LinuxKeyring.KWALLET4
+    elif desktop_environment == _LinuxDesktopEnvironment.KDE5:
+        linux_keyring = _LinuxKeyring.KWALLET5
+    elif desktop_environment == _LinuxDesktopEnvironment.KDE6:
+        linux_keyring = _LinuxKeyring.KWALLET6
+    elif desktop_environment in (
+        _LinuxDesktopEnvironment.KDE3, _LinuxDesktopEnvironment.LXQT, _LinuxDesktopEnvironment.OTHER
+    ):
+        linux_keyring = _LinuxKeyring.BASIC_TEXT
     else:
-        linux_keyring = _LinuxKeyring.GNOMEKEYRING
+        linux_keyring = _LinuxKeyring.GNOME_KEYRING
     return linux_keyring
 
 
-def _get_kwallet_network_wallet(logger):
+def _get_kwallet_network_wallet(keyring, logger):
     """ The name of the wallet used to store network passwords.
 
-    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/kwallet_dbus.cc
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/kwallet_dbus.cc
     KWalletDBus::NetworkWallet
     which does a dbus call to the following function:
     https://api.kde.org/frameworks/kwallet/html/classKWallet_1_1Wallet.html
@@ -760,10 +828,22 @@ def _get_kwallet_network_wallet(logger):
     """
     default_wallet = 'kdewallet'
     try:
+        if keyring == _LinuxKeyring.KWALLET4:
+            service_name = 'org.kde.kwalletd'
+            wallet_path = '/modules/kwalletd'
+        elif keyring == _LinuxKeyring.KWALLET5:
+            service_name = 'org.kde.kwalletd5'
+            wallet_path = '/modules/kwalletd5'
+        elif keyring == _LinuxKeyring.KWALLET6:
+            service_name = 'org.kde.kwalletd6'
+            wallet_path = '/modules/kwalletd6'
+        else:
+            raise ValueError(keyring)
+
         stdout, _, returncode = Popen.run([
             'dbus-send', '--session', '--print-reply=literal',
-            '--dest=org.kde.kwalletd5',
-            '/modules/kwalletd5',
+            f'--dest={service_name}',
+            wallet_path,
             'org.kde.KWallet.networkWallet'
         ], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
 
@@ -778,8 +858,8 @@ def _get_kwallet_network_wallet(logger):
         return default_wallet
 
 
-def _get_kwallet_password(browser_keyring_name, logger):
-    logger.debug('using kwallet-query to obtain password from kwallet')
+def _get_kwallet_password(browser_keyring_name, keyring, logger):
+    logger.debug(f'using kwallet-query to obtain password from {keyring.name}')
 
     if shutil.which('kwallet-query') is None:
         logger.error('kwallet-query command not found. KWallet and kwallet-query '
@@ -787,7 +867,7 @@ def _get_kwallet_password(browser_keyring_name, logger):
                      'included in the kwallet package for your distribution')
         return b''
 
-    network_wallet = _get_kwallet_network_wallet(logger)
+    network_wallet = _get_kwallet_network_wallet(keyring, logger)
 
     try:
         stdout, _, returncode = Popen.run([
@@ -809,8 +889,9 @@ def _get_kwallet_password(browser_keyring_name, logger):
                 # checks hasEntry. To verify this:
                 # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
                 # while starting chrome.
-                # this may be a bug as the intended behaviour is to generate a random password and store
-                # it, but that doesn't matter here.
+                # this was identified as a bug later and fixed in
+                # https://chromium.googlesource.com/chromium/src/+/bbd54702284caca1f92d656fdcadf2ccca6f4165%5E%21/#F0
+                # https://chromium.googlesource.com/chromium/src/+/5463af3c39d7f5b6d11db7fbd51e38cc1974d764
                 return b''
             else:
                 logger.debug('password found')
@@ -848,11 +929,11 @@ def _get_linux_keyring_password(browser_keyring_name, keyring, logger):
     keyring = _LinuxKeyring[keyring] if keyring else _choose_linux_keyring(logger)
     logger.debug(f'Chosen keyring: {keyring.name}')
 
-    if keyring == _LinuxKeyring.KWALLET:
-        return _get_kwallet_password(browser_keyring_name, logger)
-    elif keyring == _LinuxKeyring.GNOMEKEYRING:
+    if keyring in (_LinuxKeyring.KWALLET4, _LinuxKeyring.KWALLET5, _LinuxKeyring.KWALLET6):
+        return _get_kwallet_password(browser_keyring_name, keyring, logger)
+    elif keyring == _LinuxKeyring.GNOME_KEYRING:
         return _get_gnome_keyring_password(browser_keyring_name, logger)
-    elif keyring == _LinuxKeyring.BASICTEXT:
+    elif keyring == _LinuxKeyring.BASIC_TEXT:
         # when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
         return None
     assert False, f'Unknown keyring {keyring}'
@@ -877,6 +958,10 @@ def _get_mac_keyring_password(browser_keyring_name, logger):
 
 
 def _get_windows_v10_key(browser_root, logger):
+    """
+    References:
+        - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
+    """
     path = _find_most_recently_used_file(browser_root, 'Local State', logger)
     if path is None:
         logger.error('could not find local state file')
@@ -885,11 +970,13 @@ def _get_windows_v10_key(browser_root, logger):
     with open(path, encoding='utf8') as f:
         data = json.load(f)
     try:
+        # kOsCryptEncryptedKeyPrefName in [1]
         base64_key = data['os_crypt']['encrypted_key']
     except KeyError:
         logger.error('no encrypted key in Local State')
         return None
     encrypted_key = base64.b64decode(base64_key)
+    # kDPAPIKeyPrefix in [1]
     prefix = b'DPAPI'
     if not encrypted_key.startswith(prefix):
         logger.error('invalid key')
@@ -901,13 +988,15 @@ def pbkdf2_sha1(password, salt, iterations, key_length):
     return pbkdf2_hmac('sha1', password, salt, iterations, key_length)
 
 
-def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
-    plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector))
-    try:
-        return plaintext.decode()
-    except UnicodeDecodeError:
-        logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
-        return None
+def _decrypt_aes_cbc_multi(ciphertext, keys, logger, initialization_vector=b' ' * 16):
+    for key in keys:
+        plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector))
+        try:
+            return plaintext.decode()
+        except UnicodeDecodeError:
+            pass
+    logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
+    return None
 
 
 def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):

From f78eb41e1c0f1dcdb10317358a26bf541dc7ee15 Mon Sep 17 00:00:00 2001
From: Stefan Lobbenmeier <Stefan.Lobbenmeier@gmail.com>
Date: Mon, 29 May 2023 16:28:14 +0200
Subject: [PATCH 308/871] [extractor/ARDBetaMediathek] Add thumbnail (#6890)

Closes #6889
Authored by: StefanLobbenmeier
---
 yt_dlp/extractor/ard.py | 44 +++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/ard.py b/yt_dlp/extractor/ard.py
index 8660741ce..ca1faa7d0 100644
--- a/yt_dlp/extractor/ard.py
+++ b/yt_dlp/extractor/ard.py
@@ -13,6 +13,7 @@
     try_get,
     unified_strdate,
     unified_timestamp,
+    update_url,
     update_url_query,
     url_or_none,
     xpath_text,
@@ -408,6 +409,23 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         (?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
 
     _TESTS = [{
+        'url': 'https://www.ardmediathek.de/video/filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy8xZGY0ZGJmZS00ZWQwLTRmMGItYjhhYy0wOGQ4ZmYxNjVhZDI',
+        'md5': '3fd5fead7a370a819341129c8d713136',
+        'info_dict': {
+            'display_id': 'filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen',
+            'id': '12172961',
+            'title': 'Wolfsland - Die traurigen Schwestern',
+            'description': r're:^Als der Polizeiobermeister Raaben',
+            'duration': 5241,
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:efa186f7b0054957',
+            'timestamp': 1670710500,
+            'upload_date': '20221210',
+            'ext': 'mp4',
+            'age_limit': 12,
+            'episode': 'Wolfsland - Die traurigen Schwestern',
+            'series': 'Filme im MDR'
+        },
+    }, {
         'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
         'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
         'info_dict': {
@@ -424,7 +442,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         'skip': 'Error',
     }, {
         'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
-        'md5': 'f1837e563323b8a642a8ddeff0131f51',
+        'md5': '1e73ded21cb79bac065117e80c81dc88',
         'info_dict': {
             'id': '10049223',
             'ext': 'mp4',
@@ -432,13 +450,11 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'timestamp': 1636398000,
             'description': 'md5:39578c7b96c9fe50afdf5674ad985e6b',
             'upload_date': '20211108',
-        },
-    }, {
-        'url': 'https://www.ardmediathek.de/sendung/beforeigners/beforeigners/staffel-1/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw/1',
-        'playlist_count': 6,
-        'info_dict': {
-            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw',
-            'title': 'beforeigners/beforeigners/staffel-1',
+            'display_id': 'tagesschau-oder-tagesschau-20-00-uhr/das-erste',
+            'duration': 915,
+            'episode': 'tagesschau, 20:00 Uhr',
+            'series': 'tagesschau',
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49',
         },
     }, {
         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
@@ -602,6 +618,9 @@ def _real_extract(self, url):
     show {
       title
     }
+    image {
+      src
+    }
     synopsis
     title
     tracking {
@@ -640,6 +659,15 @@ def _real_extract(self, url):
             'description': description,
             'timestamp': unified_timestamp(player_page.get('broadcastedOn')),
             'series': try_get(player_page, lambda x: x['show']['title']),
+            'thumbnail': (media_collection.get('_previewImage')
+                          or try_get(player_page, lambda x: update_url(x['image']['src'], query=None, fragment=None))
+                          or self.get_thumbnail_from_html(display_id, url)),
         })
         info.update(self._ARD_extract_episode_info(info['title']))
         return info
+
+    def get_thumbnail_from_html(self, display_id, url):
+        webpage = self._download_webpage(url, display_id, fatal=False) or ''
+        return (
+            self._og_search_thumbnail(webpage, default=None)
+            or self._html_search_meta('thumbnailUrl', webpage, default=None))

From dbce5afa6bb61f6272ade613f2e9a3d66b88c7ea Mon Sep 17 00:00:00 2001
From: Florian Albrechtskirchinger <falbrechtskirchinger@gmail.com>
Date: Mon, 29 May 2023 16:30:20 +0200
Subject: [PATCH 309/871] [extractor/twitch:vod] Support links from schedule
 tab (#7071)

Authored by: falbrechtskirchinger
---
 yt_dlp/extractor/twitch.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index d7a1cc531..4a17d8048 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -194,7 +194,8 @@ class TwitchVodIE(TwitchBaseIE):
                     https?://
                         (?:
                             (?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/v(?:ideo)?|videos)/|
-                            player\.twitch\.tv/\?.*?\bvideo=v?
+                            player\.twitch\.tv/\?.*?\bvideo=v?|
+                            www\.twitch\.tv/[^/]+/schedule\?vodID=
                         )
                         (?P<id>\d+)
                     '''
@@ -363,6 +364,9 @@ class TwitchVodIE(TwitchBaseIE):
             'skip_download': True
         },
         'expected_warnings': ['Unable to download JSON metadata: HTTP Error 403: Forbidden']
+    }, {
+        'url': 'https://www.twitch.tv/tangotek/schedule?vodID=1822395420',
+        'only_matching': True,
     }]
 
     def _download_info(self, item_id):

From 45e87ea106ad37b2a002663fa30ee41ce97b16cd Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Mon, 29 May 2023 23:31:22 +0900
Subject: [PATCH 310/871] [extractor/eurosport] Improve `_VALID_URL` (#7076)

Closes #7042
Authored by: HobbyistDev
---
 yt_dlp/extractor/eurosport.py | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/eurosport.py b/yt_dlp/extractor/eurosport.py
index 654e11206..6c426bb89 100644
--- a/yt_dlp/extractor/eurosport.py
+++ b/yt_dlp/extractor/eurosport.py
@@ -3,7 +3,7 @@
 
 
 class EurosportIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.eurosport\.com/\w+/[\w-]+/\d+/[\w-]+_(?P<id>vid\d+)'
+    _VALID_URL = r'https?://www\.eurosport\.com/\w+/(?:[\w-]+/[\d-]+/)?[\w-]+_(?P<id>vid\d+)'
     _TESTS = [{
         'url': 'https://www.eurosport.com/tennis/roland-garros/2022/highlights-rafael-nadal-brushes-aside-caper-ruud-to-win-record-extending-14th-french-open-title_vid1694147/video.shtml',
         'info_dict': {
@@ -44,6 +44,32 @@ class EurosportIE(InfoExtractor):
             'description': 'md5:32bbe3a773ac132c57fb1e8cca4b7c71',
             'upload_date': '20220727',
         }
+    }, {
+        'url': 'https://www.eurosport.com/football/champions-league/2022-2023/pep-guardiola-emotionally-destroyed-after-manchester-city-win-over-bayern-munich-in-champions-league_vid1896254/video.shtml',
+        'info_dict': {
+            'id': '3096477',
+            'ext': 'mp4',
+            'title': 'md5:82edc17370124c7a19b3cf518517583b',
+            'duration': 84.0,
+            'description': 'md5:b3f44ef7f5b5b95b24a273b163083feb',
+            'thumbnail': 'https://imgresizer.eurosport.com/unsafe/1280x960/smart/filters:format(jpeg)/origin-imgresizer.eurosport.com/2023/04/12/3682873-74947393-2560-1440.jpg',
+            'timestamp': 1681292028,
+            'upload_date': '20230412',
+            'display_id': 'vid1896254',
+        }
+    }, {
+        'url': 'https://www.eurosport.com/football/last-year-s-semi-final-pain-was-still-there-pep-guardiola-after-man-city-reach-cl-final_vid1914115/video.shtml',
+        'info_dict': {
+            'id': '3149108',
+            'ext': 'mp4',
+            'title': '\'Last year\'s semi-final pain was still there\' - Pep Guardiola after Man City reach CL final',
+            'description': 'md5:89ef142fe0170a66abab77fac2955d8e',
+            'display_id': 'vid1914115',
+            'timestamp': 1684403618,
+            'thumbnail': 'https://imgresizer.eurosport.com/unsafe/1280x960/smart/filters:format(jpeg)/origin-imgresizer.eurosport.com/2023/05/18/3707254-75435008-2560-1440.jpg',
+            'duration': 105.0,
+            'upload_date': '20230518',
+        }
     }]
 
     _TOKEN = None

From 4cbfa570a1b9bd65b0f48770693377e8d842dcb0 Mon Sep 17 00:00:00 2001
From: garret <76261416+garret1317@users.noreply.github.com>
Date: Mon, 29 May 2023 15:44:26 +0100
Subject: [PATCH 311/871] [extractor/camfm] Add extractors (#7083)

Authored by: garret1317
---
 yt_dlp/extractor/_extractors.py |  4 ++
 yt_dlp/extractor/camfm.py       | 85 +++++++++++++++++++++++++++++++++
 yt_dlp/utils/_utils.py          |  1 +
 3 files changed, 90 insertions(+)
 create mode 100644 yt_dlp/extractor/camfm.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index f9fa84c43..73dcf4e0e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -284,6 +284,10 @@
     CamdemyIE,
     CamdemyFolderIE
 )
+from .camfm import (
+    CamFMEpisodeIE,
+    CamFMShowIE
+)
 from .cammodels import CamModelsIE
 from .camsoda import CamsodaIE
 from .camtasia import CamtasiaEmbedIE
diff --git a/yt_dlp/extractor/camfm.py b/yt_dlp/extractor/camfm.py
new file mode 100644
index 000000000..a9850f46e
--- /dev/null
+++ b/yt_dlp/extractor/camfm.py
@@ -0,0 +1,85 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    get_element_by_class,
+    get_elements_by_class,
+    join_nonempty,
+    traverse_obj,
+    unified_timestamp,
+    urljoin,
+)
+
+
+class CamFMShowIE(InfoExtractor):
+    _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/shows/(?P<id>[^/]+)'
+    _TESTS = [{
+        'playlist_mincount': 5,
+        'url': 'https://camfm.co.uk/shows/soul-mining/',
+        'info_dict': {
+            'id': 'soul-mining',
+            'thumbnail': 'md5:6a873091f92c936f23bdcce80f75e66a',
+            'title': 'Soul Mining',
+            'description': 'Telling the stories of jazz, funk and soul from all corners of the world.',
+        },
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        page = self._download_webpage(url, show_id)
+
+        return {
+            '_type': 'playlist',
+            'id': show_id,
+            'entries': [self.url_result(urljoin('https://camfm.co.uk', i), CamFMEpisodeIE)
+                        for i in re.findall(r"javascript:popup\('(/player/[^']+)', 'listen'", page)],
+            'thumbnail': urljoin('https://camfm.co.uk', self._search_regex(
+                r'<img[^>]+class="thumb-expand"[^>]+src="([^"]+)"', page, 'thumbnail', fatal=False)),
+            'title': self._html_search_regex('<h1>([^<]+)</h1>', page, 'title', fatal=False),
+            'description': clean_html(get_element_by_class('small-12 medium-8 cell', page))
+        }
+
+
+class CamFMEpisodeIE(InfoExtractor):
+    _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/player/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://camfm.co.uk/player/43336',
+        'skip': 'Episode will expire - don\'t actually know when, but it will go eventually',
+        'info_dict': {
+            'id': '43336',
+            'title': 'AITAA: Am I the Agony Aunt? - 19:00 Tue 16/05/2023',
+            'ext': 'mp3',
+            'upload_date': '20230516',
+            'description': 'md5:f165144f94927c0f1bfa2ee6e6ab7bbf',
+            'timestamp': 1684263600,
+            'series': 'AITAA: Am I the Agony Aunt?',
+            'thumbnail': 'md5:5980a831360d0744c3764551be3d09c1',
+            'categories': ['Entertainment'],
+        }
+    }]
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+        page = self._download_webpage(url, episode_id)
+        audios = self._parse_html5_media_entries('https://audio.camfm.co.uk', page, episode_id)
+
+        caption = get_element_by_class('caption', page)
+        series = clean_html(re.sub(r'<span[^<]+<[^<]+>', '', caption))
+
+        card_section = get_element_by_class('card-section', page)
+        date = self._html_search_regex('>Aired at ([^<]+)<', card_section, 'air date', fatal=False)
+
+        return {
+            'id': episode_id,
+            'title': join_nonempty(series, date, delim=' - '),
+            'formats': traverse_obj(audios, (..., 'formats', ...)),
+            'timestamp': unified_timestamp(date),  # XXX: Does not account for UK's daylight savings
+            'series': series,
+            'description': clean_html(re.sub(r'<b>[^<]+</b><br[^>]+/>', '', card_section)),
+            'thumbnail': urljoin('https://camfm.co.uk', self._search_regex(
+                r'<div[^>]+class="cover-art"[^>]+style="[^"]+url\(\'([^\']+)',
+                page, 'thumbnail', fatal=False)),
+            'categories': get_elements_by_class('label', caption),
+            'was_live': True,
+        }
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 7c91faff8..4179d58c1 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -223,6 +223,7 @@ def IDENTITY(x):
     '%d/%m/%y',
     '%d/%m/%Y %H:%M:%S',
     '%d-%m-%Y %H:%M',
+    '%H:%M %d/%m/%Y',
 ])
 
 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)

From 372a0f3b9dadd1e52234b498aa4c7040ef868c7d Mon Sep 17 00:00:00 2001
From: Ivan Skodje <ivanskodje@users.noreply.github.com>
Date: Mon, 29 May 2023 16:50:21 +0200
Subject: [PATCH 312/871] Auto-select default format in `-f-` (#7101)

Authored by: ivanskodje, pukkandan
Closes #6720
---
 yt_dlp/YoutubeDL.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index f49dbf07d..b4923920f 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2744,21 +2744,22 @@ def is_wellformed(f):
             return info_dict
 
         format_selector = self.format_selector
-        if format_selector is None:
-            req_format = self._default_format_spec(info_dict, download=download)
-            self.write_debug('Default format spec: %s' % req_format)
-            format_selector = self.build_format_selector(req_format)
-
         while True:
             if interactive_format_selection:
-                req_format = input(
-                    self._format_screen('\nEnter format selector: ', self.Styles.EMPHASIS))
+                req_format = input(self._format_screen('\nEnter format selector ', self.Styles.EMPHASIS)
+                                   + '(Press ENTER for default, or Ctrl+C to quit)'
+                                   + self._format_screen(': ', self.Styles.EMPHASIS))
                 try:
-                    format_selector = self.build_format_selector(req_format)
+                    format_selector = self.build_format_selector(req_format) if req_format else None
                 except SyntaxError as err:
                     self.report_error(err, tb=False, is_error=False)
                     continue
 
+            if format_selector is None:
+                req_format = self._default_format_spec(info_dict, download=download)
+                self.write_debug(f'Default format spec: {req_format}')
+                format_selector = self.build_format_selector(req_format)
+
             formats_to_download = list(format_selector({
                 'formats': formats,
                 'has_merged_format': any('none' not in (f.get('acodec'), f.get('vcodec')) for f in formats),

From 937264419f9bf375d5656785ae6e53282587c15d Mon Sep 17 00:00:00 2001
From: Ivan Skodje <ivanskodje@users.noreply.github.com>
Date: Mon, 29 May 2023 16:53:35 +0200
Subject: [PATCH 313/871] [extractor/tvplay] Remove outdated domains (#7106)

Closes #3920
Authored by: ivanskodje
---
 yt_dlp/extractor/_extractors.py |   1 -
 yt_dlp/extractor/tvplay.py      | 213 +-------------------------------
 2 files changed, 1 insertion(+), 213 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 73dcf4e0e..2963998cb 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2098,7 +2098,6 @@
 )
 from .tvplay import (
     TVPlayIE,
-    ViafreeIE,
     TVPlayHomeIE,
 )
 from .tvplayer import TVPlayerIE
diff --git a/yt_dlp/extractor/tvplay.py b/yt_dlp/extractor/tvplay.py
index 9ef4f962c..e056f9872 100644
--- a/yt_dlp/extractor/tvplay.py
+++ b/yt_dlp/extractor/tvplay.py
@@ -30,10 +30,7 @@ class TVPlayIE(InfoExtractor):
                             (?:
                                 tvplay(?:\.skaties)?\.lv(?:/parraides)?|
                                 (?:tv3play|play\.tv3)\.lt(?:/programos)?|
-                                tv3play(?:\.tv3)?\.ee/sisu|
-                                (?:tv(?:3|6|8|10)play)\.se/program|
-                                (?:(?:tv3play|viasat4play|tv6play)\.no|(?:tv3play)\.dk)/programmer|
-                                play\.nova(?:tv)?\.bg/programi
+                                tv3play(?:\.tv3)?\.ee/sisu
                             )
                             /(?:[^/]+/)+
                         )
@@ -92,117 +89,6 @@ class TVPlayIE(InfoExtractor):
                 'skip_download': True,
             },
         },
-        {
-            'url': 'http://www.tv3play.se/program/husraddarna/395385?autostart=true',
-            'info_dict': {
-                'id': '395385',
-                'ext': 'mp4',
-                'title': 'Husräddarna S02E07',
-                'description': 'md5:f210c6c89f42d4fc39faa551be813777',
-                'duration': 2574,
-                'timestamp': 1400596321,
-                'upload_date': '20140520',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.tv6play.se/program/den-sista-dokusapan/266636?autostart=true',
-            'info_dict': {
-                'id': '266636',
-                'ext': 'mp4',
-                'title': 'Den sista dokusåpan S01E08',
-                'description': 'md5:295be39c872520221b933830f660b110',
-                'duration': 1492,
-                'timestamp': 1330522854,
-                'upload_date': '20120229',
-                'age_limit': 18,
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.tv8play.se/program/antikjakten/282756?autostart=true',
-            'info_dict': {
-                'id': '282756',
-                'ext': 'mp4',
-                'title': 'Antikjakten S01E10',
-                'description': 'md5:1b201169beabd97e20c5ad0ad67b13b8',
-                'duration': 2646,
-                'timestamp': 1348575868,
-                'upload_date': '20120925',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.tv3play.no/programmer/anna-anka-soker-assistent/230898?autostart=true',
-            'info_dict': {
-                'id': '230898',
-                'ext': 'mp4',
-                'title': 'Anna Anka søker assistent - Ep. 8',
-                'description': 'md5:f80916bf5bbe1c5f760d127f8dd71474',
-                'duration': 2656,
-                'timestamp': 1277720005,
-                'upload_date': '20100628',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.viasat4play.no/programmer/budbringerne/21873?autostart=true',
-            'info_dict': {
-                'id': '21873',
-                'ext': 'mp4',
-                'title': 'Budbringerne program 10',
-                'description': 'md5:4db78dc4ec8a85bb04fd322a3ee5092d',
-                'duration': 1297,
-                'timestamp': 1254205102,
-                'upload_date': '20090929',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.tv6play.no/programmer/hotelinspektor-alex-polizzi/361883?autostart=true',
-            'info_dict': {
-                'id': '361883',
-                'ext': 'mp4',
-                'title': 'Hotelinspektør Alex Polizzi - Ep. 10',
-                'description': 'md5:3ecf808db9ec96c862c8ecb3a7fdaf81',
-                'duration': 2594,
-                'timestamp': 1393236292,
-                'upload_date': '20140224',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://play.novatv.bg/programi/zdravei-bulgariya/624952?autostart=true',
-            'info_dict': {
-                'id': '624952',
-                'ext': 'flv',
-                'title': 'Здравей, България (12.06.2015 г.) ',
-                'description': 'md5:99f3700451ac5bb71a260268b8daefd7',
-                'duration': 8838,
-                'timestamp': 1434100372,
-                'upload_date': '20150612',
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'https://play.nova.bg/programi/zdravei-bulgariya/764300?autostart=true',
-            'only_matching': True,
-        },
         {
             'url': 'http://tvplay.skaties.lv/parraides/vinas-melo-labak/418113?autostart=true',
             'only_matching': True,
@@ -327,103 +213,6 @@ def _real_extract(self, url):
         }
 
 
-class ViafreeIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?
-                        viafree\.(?P<country>dk|no|se|fi)
-                        /(?P<id>(?:program(?:mer)?|ohjelmat)?/(?:[^/]+/)+[^/?#&]+)
-                    '''
-    _TESTS = [{
-        'url': 'http://www.viafree.no/programmer/underholdning/det-beste-vorspielet/sesong-2/episode-1',
-        'info_dict': {
-            'id': '757786',
-            'ext': 'mp4',
-            'title': 'Det beste vorspielet - Sesong 2 - Episode 1',
-            'description': 'md5:b632cb848331404ccacd8cd03e83b4c3',
-            'series': 'Det beste vorspielet',
-            'season_number': 2,
-            'duration': 1116,
-            'timestamp': 1471200600,
-            'upload_date': '20160814',
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://www.viafree.dk/programmer/humor/comedy-central-roast-of-charlie-sheen/film/1047660',
-        'info_dict': {
-            'id': '1047660',
-            'ext': 'mp4',
-            'title': 'Comedy Central Roast of Charlie Sheen - Comedy Central Roast of Charlie Sheen',
-            'description': 'md5:ec956d941ae9fd7c65a48fd64951dc6d',
-            'series': 'Comedy Central Roast of Charlie Sheen',
-            'season_number': 1,
-            'duration': 3747,
-            'timestamp': 1608246060,
-            'upload_date': '20201217'
-        },
-        'params': {
-            'skip_download': True
-        }
-    }, {
-        # with relatedClips
-        'url': 'http://www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-1',
-        'only_matching': True,
-    }, {
-        # Different og:image URL schema
-        'url': 'http://www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-2',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.viafree.se/program/livsstil/husraddarna/sasong-2/avsnitt-2',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.viafree.dk/programmer/reality/paradise-hotel/saeson-7/episode-5',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.viafree.se/program/underhallning/i-like-radio-live/sasong-1/676869',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.viafree.fi/ohjelmat/entertainment/amazing-makeovers/kausi-7/jakso-2',
-        'only_matching': True,
-    }]
-    _GEO_BYPASS = False
-
-    def _real_extract(self, url):
-        country, path = self._match_valid_url(url).groups()
-        content = self._download_json(
-            'https://viafree-content.mtg-api.com/viafree-content/v1/%s/path/%s' % (country, path), path)
-        program = content['_embedded']['viafreeBlocks'][0]['_embedded']['program']
-        guid = program['guid']
-        meta = content['meta']
-        title = meta['title']
-
-        try:
-            stream_href = self._download_json(
-                program['_links']['streamLink']['href'], guid,
-                headers=self.geo_verification_headers())['embedded']['prioritizedStreams'][0]['links']['stream']['href']
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                self.raise_geo_restricted(countries=[country])
-            raise
-
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(stream_href, guid, 'mp4')
-        episode = program.get('episode') or {}
-        return {
-            'id': guid,
-            'title': title,
-            'thumbnail': meta.get('image'),
-            'description': meta.get('description'),
-            'series': episode.get('seriesTitle'),
-            'subtitles': subtitles,
-            'episode_number': int_or_none(episode.get('episodeNumber')),
-            'season_number': int_or_none(episode.get('seasonNumber')),
-            'duration': int_or_none(try_get(program, lambda x: x['video']['duration']['milliseconds']), 1000),
-            'timestamp': parse_iso8601(try_get(program, lambda x: x['availability']['start'])),
-            'formats': formats,
-        }
-
-
 class TVPlayHomeIE(InfoExtractor):
     _VALID_URL = r'''(?x)
             https?://

From dc3c44f349ba85af320e706e2a27ad81a78b1c6e Mon Sep 17 00:00:00 2001
From: Mohamed Al Mehairbi <62325490+ItzMaxTV@users.noreply.github.com>
Date: Mon, 29 May 2023 19:19:13 +0400
Subject: [PATCH 314/871] [extractor/Mzaalo] Add extractor (#7163)

Authored by: ItzMaxTV
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/mzaalo.py      | 92 +++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 yt_dlp/extractor/mzaalo.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 2963998cb..5b4ed44ef 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1162,6 +1162,7 @@
 )
 from .myvideoge import MyVideoGeIE
 from .myvidster import MyVidsterIE
+from .mzaalo import MzaaloIE
 from .n1 import (
     N1InfoAssetIE,
     N1InfoIIE,
diff --git a/yt_dlp/extractor/mzaalo.py b/yt_dlp/extractor/mzaalo.py
new file mode 100644
index 000000000..c6f420cea
--- /dev/null
+++ b/yt_dlp/extractor/mzaalo.py
@@ -0,0 +1,92 @@
+from .common import InfoExtractor
+from ..utils import (
+    parse_age_limit,
+    parse_duration,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class MzaaloIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?mzaalo\.com/play/(?P<type>movie|original|clip)/(?P<id>[a-fA-F0-9-]+)/[\w-]+'
+    _TESTS = [{
+        # Movies
+        'url': 'https://www.mzaalo.com/play/movie/c0958d9f-f90e-4503-a755-44358758921d/Jamun',
+        'info_dict': {
+            'id': 'c0958d9f-f90e-4503-a755-44358758921d',
+            'title': 'Jamun',
+            'ext': 'mp4',
+            'description': 'md5:24fe9ebb9bbe5b36f7b54b90ab1e2f31',
+            'thumbnails': 'count:15',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 5527.0,
+            'language': 'hin',
+            'categories': ['Drama'],
+            'age_limit': 13,
+        },
+        'params': {'skip_download': 'm3u8'}
+    }, {
+        # Shows
+        'url': 'https://www.mzaalo.com/play/original/93d42b2b-f373-4c2d-bca4-997412cb069d/Modi-Season-2-CM-TO-PM/Episode-1:Decision,-Not-Promises',
+        'info_dict': {
+            'id': '93d42b2b-f373-4c2d-bca4-997412cb069d',
+            'title': 'Episode 1:Decision, Not Promises',
+            'ext': 'mp4',
+            'description': 'md5:16f76058432a54774fbb2561a1955652',
+            'thumbnails': 'count:22',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2040.0,
+            'language': 'hin',
+            'categories': ['Drama'],
+            'age_limit': 13,
+        },
+        'params': {'skip_download': 'm3u8'}
+    }, {
+        # Streams/Clips
+        'url': 'https://www.mzaalo.com/play/clip/83cdbcb5-400a-42f1-a1d2-459053cfbda5/Manto-Ki-Kahaaniya',
+        'info_dict': {
+            'id': '83cdbcb5-400a-42f1-a1d2-459053cfbda5',
+            'title': 'Manto Ki Kahaaniya',
+            'ext': 'mp4',
+            'description': 'md5:c3c5f1d05f0fd1bfcb05b673d1cc9f2f',
+            'thumbnails': 'count:3',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1937.0,
+            'language': 'hin',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }]
+
+    def _real_extract(self, url):
+        video_id, type_ = self._match_valid_url(url).group('id', 'type')
+        path = (f'partner/streamurl?&assetId={video_id}&getClipDetails=YES' if type_ == 'clip'
+                else f'api/v2/player/details?assetType={type_.upper()}&assetId={video_id}')
+        data = self._download_json(
+            f'https://production.mzaalo.com/platform/{path}', video_id, headers={
+                'Ocp-Apim-Subscription-Key': '1d0caac2702049b89a305929fdf4cbae',
+            })['data']
+
+        formats = self._extract_m3u8_formats(data['streamURL'], video_id)
+
+        subtitles = {}
+        for subs_lang, subs_url in traverse_obj(data, ('subtitles', {dict.items}, ...)):
+            if url_or_none(subs_url):
+                subtitles[subs_lang] = [{'url': subs_url, 'ext': 'vtt'}]
+
+        lang = traverse_obj(data, ('language', {str.lower}))
+        for f in formats:
+            f['language'] = lang
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'duration': ('duration', {parse_duration}),
+                'age_limit': ('maturity_rating', {parse_age_limit}),
+                'thumbnails': ('images', ..., {'url': {url_or_none}}),
+                'categories': ('genre', ..., {str}),
+            }),
+        }

From 03789976d301eaed3e957dbc041573098f6af059 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Tue, 30 May 2023 00:20:07 +0900
Subject: [PATCH 315/871] [extractor/europarl] Rewrite extractor (#7114)

Authored by: HobbyistDev
Closes #6396
---
 yt_dlp/extractor/europa.py | 82 +++++++++++++++++++-------------------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/yt_dlp/extractor/europa.py b/yt_dlp/extractor/europa.py
index 29daabe4a..f3da95f5c 100644
--- a/yt_dlp/extractor/europa.py
+++ b/yt_dlp/extractor/europa.py
@@ -6,6 +6,7 @@
     parse_iso8601,
     parse_qs,
     qualities,
+    traverse_obj,
     unified_strdate,
     xpath_text
 )
@@ -92,42 +93,17 @@ def get_item(type_, preference):
 
 class EuroParlWebstreamIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?:multimedia|webstreaming)\.europarl\.europa\.eu/[^/#?]+/
-        (?:embed/embed\.html\?event=|(?!video)[^/#?]+/[\w-]+_)(?P<id>[\w-]+)
+        https?://multimedia\.europarl\.europa\.eu/[^/#?]+/
+        (?:(?!video)[^/#?]+/[\w-]+_)(?P<id>[\w-]+)
     '''
     _TESTS = [{
         'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/plenary-session_20220914-0900-PLENARY',
         'info_dict': {
-            'id': 'bcaa1db4-76ef-7e06-8da7-839bd0ad1dbe',
-            'ext': 'mp4',
-            'release_timestamp': 1663137900,
-            'title': 'Plenary session',
-            'release_date': '20220914',
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/eu-cop27-un-climate-change-conference-in-sharm-el-sheikh-egypt-ep-delegation-meets-with-ngo-represen_20221114-1600-SPECIAL-OTHER',
-        'info_dict': {
-            'id': 'a8428de8-b9cd-6a2e-11e4-3805d9c9ff5c',
-            'ext': 'mp4',
-            'release_timestamp': 1668434400,
-            'release_date': '20221114',
-            'title': 'md5:d3550280c33cc70e0678652e3d52c028',
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        # embed webpage
-        'url': 'https://webstreaming.europarl.europa.eu/ep/embed/embed.html?event=20220914-0900-PLENARY&language=en&autoplay=true&logo=true',
-        'info_dict': {
-            'id': 'bcaa1db4-76ef-7e06-8da7-839bd0ad1dbe',
+            'id': '62388b15-d85b-4add-99aa-ba12ccf64f0d',
             'ext': 'mp4',
             'title': 'Plenary session',
+            'release_timestamp': 1663139069,
             'release_date': '20220914',
-            'release_timestamp': 1663137900,
         },
         'params': {
             'skip_download': True,
@@ -144,30 +120,54 @@ class EuroParlWebstreamIE(InfoExtractor):
             'live_status': 'is_live',
         },
         'skip': 'not live anymore'
+    }, {
+        'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-culture-and-education_20230301-1130-COMMITTEE-CULT',
+        'info_dict': {
+            'id': '7355662c-8eac-445e-4bb9-08db14b0ddd7',
+            'ext': 'mp4',
+            'release_date': '20230301',
+            'title': 'Committee on Culture and Education',
+            'release_timestamp': 1677666641,
+        }
+    }, {
+        # live stream
+        'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-environment-public-health-and-food-safety_20230524-0900-COMMITTEE-ENVI',
+        'info_dict': {
+            'id': 'e4255f56-10aa-4b3c-6530-08db56d5b0d9',
+            'ext': 'mp4',
+            'release_date': '20230524',
+            'title': r're:Committee on Environment, Public Health and Food Safety \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}',
+            'release_timestamp': 1684911541,
+            'live_status': 'is_live',
+        },
+        'skip': 'Not live anymore'
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        webpage_nextjs = self._search_nextjs_data(webpage, display_id)['props']['pageProps']
 
         json_info = self._download_json(
-            'https://vis-api.vuplay.co.uk/event/external', display_id,
+            'https://acs-api.europarl.connectedviews.eu/api/FullMeeting', display_id,
             query={
-                'player_key': 'europarl|718f822c-a48c-4841-9947-c9cb9bb1743c',
-                'external_id': display_id,
+                'api-version': 1.0,
+                'tenantId': 'bae646ca-1fc8-4363-80ba-2c04f06b4968',
+                'externalReference': display_id
             })
 
-        formats, subtitles = self._extract_mpd_formats_and_subtitles(json_info['streaming_url'], display_id)
-        fmts, subs = self._extract_m3u8_formats_and_subtitles(
-            json_info['streaming_url'].replace('.mpd', '.m3u8'), display_id)
-
-        formats.extend(fmts)
-        self._merge_subtitles(subs, target=subtitles)
+        formats, subtitles = [], {}
+        for hls_url in traverse_obj(json_info, ((('meetingVideo'), ('meetingVideos', ...)), 'hlsUrl')):
+            fmt, subs = self._extract_m3u8_formats_and_subtitles(hls_url, display_id)
+            formats.extend(fmt)
+            self._merge_subtitles(subs, target=subtitles)
 
         return {
             'id': json_info['id'],
-            'title': json_info.get('title'),
+            'title': traverse_obj(webpage_nextjs, (('mediaItem', 'title'), ('title', )), get_all=False),
             'formats': formats,
             'subtitles': subtitles,
-            'release_timestamp': parse_iso8601(json_info.get('published_start')),
-            'is_live': 'LIVE' in json_info.get('state', '')
+            'release_timestamp': parse_iso8601(json_info.get('startDateTime')),
+            'is_live': traverse_obj(webpage_nextjs, ('mediaItem', 'mediaSubType')) == 'Live'
         }

From 6f10cdcf7eeaeae5b75e0a4428cd649c156a2d83 Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Mon, 29 May 2023 09:30:30 -0600
Subject: [PATCH 316/871] [extractor/bilibili:SpaceVideo] Extract signature
 (#7149)

Authored by: elyse0
Closes #6956, closes #7081
---
 yt_dlp/extractor/bilibili.py | 57 ++++++++++++++++++++++++++++++++----
 1 file changed, 52 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index faa2218ce..6629fbc08 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -1,7 +1,9 @@
 import base64
 import functools
+import hashlib
 import itertools
 import math
+import time
 import urllib.error
 import urllib.parse
 
@@ -26,6 +28,7 @@
     srt_subtitles_timecode,
     str_or_none,
     traverse_obj,
+    try_call,
     unified_timestamp,
     unsmuggle_url,
     url_or_none,
@@ -514,19 +517,63 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
             'id': '3985676',
         },
         'playlist_mincount': 178,
+    }, {
+        'url': 'https://space.bilibili.com/313580179/video',
+        'info_dict': {
+            'id': '313580179',
+        },
+        'playlist_mincount': 92,
     }]
 
+    def _extract_signature(self, playlist_id):
+        session_data = self._download_json('https://api.bilibili.com/x/web-interface/nav', playlist_id, fatal=False)
+
+        key_from_url = lambda x: x[x.rfind('/') + 1:].split('.')[0]
+        img_key = traverse_obj(
+            session_data, ('data', 'wbi_img', 'img_url', {key_from_url})) or '34478ba821254d9d93542680e3b86100'
+        sub_key = traverse_obj(
+            session_data, ('data', 'wbi_img', 'sub_url', {key_from_url})) or '7e16a90d190a4355a78fd00b32a38de6'
+
+        session_key = img_key + sub_key
+
+        signature_values = []
+        for position in (
+            46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, 27, 43, 5, 49, 33, 9, 42, 19, 29, 28, 14, 39,
+            12, 38, 41, 13, 37, 48, 7, 16, 24, 55, 40, 61, 26, 17, 0, 1, 60, 51, 30, 4, 22, 25, 54, 21, 56, 59, 6, 63,
+            57, 62, 11, 36, 20, 34, 44, 52
+        ):
+            char_at_position = try_call(lambda: session_key[position])
+            if char_at_position:
+                signature_values.append(char_at_position)
+
+        return ''.join(signature_values)[:32]
+
     def _real_extract(self, url):
         playlist_id, is_video_url = self._match_valid_url(url).group('id', 'video')
         if not is_video_url:
             self.to_screen('A channel URL was given. Only the channel\'s videos will be downloaded. '
                            'To download audios, add a "/audio" to the URL')
 
+        signature = self._extract_signature(playlist_id)
+
         def fetch_page(page_idx):
+            query = {
+                'keyword': '',
+                'mid': playlist_id,
+                'order': 'pubdate',
+                'order_avoided': 'true',
+                'platform': 'web',
+                'pn': page_idx + 1,
+                'ps': 30,
+                'tid': 0,
+                'web_location': 1550101,
+                'wts': int(time.time()),
+            }
+            query['w_rid'] = hashlib.md5(f'{urllib.parse.urlencode(query)}{signature}'.encode()).hexdigest()
+
             try:
-                response = self._download_json('https://api.bilibili.com/x/space/arc/search',
-                                               playlist_id, note=f'Downloading page {page_idx}',
-                                               query={'mid': playlist_id, 'pn': page_idx + 1, 'jsonp': 'jsonp'})
+                response = self._download_json('https://api.bilibili.com/x/space/wbi/arc/search',
+                                               playlist_id, note=f'Downloading page {page_idx}', query=query)
             except ExtractorError as e:
                 if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 412:
                     raise ExtractorError(
@@ -556,9 +603,9 @@ def get_entries(page_data):
 class BilibiliSpaceAudioIE(BilibiliSpaceBaseIE):
     _VALID_URL = r'https?://space\.bilibili\.com/(?P<id>\d+)/audio'
     _TESTS = [{
-        'url': 'https://space.bilibili.com/3985676/audio',
+        'url': 'https://space.bilibili.com/313580179/audio',
         'info_dict': {
-            'id': '3985676',
+            'id': '313580179',
         },
         'playlist_mincount': 1,
     }]

From 26c517b29c8727e47948d6fff749d5297f0efb60 Mon Sep 17 00:00:00 2001
From: Mohamed Al Mehairbi <62325490+ItzMaxTV@users.noreply.github.com>
Date: Tue, 30 May 2023 17:40:56 +0400
Subject: [PATCH 317/871] [extractor/crtvg] Add extractor (#7168)

Closes #6609
Authored by: ItzMaxTV
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/crtvg.py       | 34 +++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 yt_dlp/extractor/crtvg.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 5b4ed44ef..85c584f5e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -411,6 +411,7 @@
     CrowdBunkerIE,
     CrowdBunkerChannelIE,
 )
+from .crtvg import CrtvgIE
 from .crunchyroll import (
     CrunchyrollBetaIE,
     CrunchyrollBetaShowIE,
diff --git a/yt_dlp/extractor/crtvg.py b/yt_dlp/extractor/crtvg.py
new file mode 100644
index 000000000..1aa8d7705
--- /dev/null
+++ b/yt_dlp/extractor/crtvg.py
@@ -0,0 +1,34 @@
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class CrtvgIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?crtvg\.es/tvg/a-carta/[^/#?]+-(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.crtvg.es/tvg/a-carta/os-caimans-do-tea-5839623',
+        'md5': 'c0958d9ff90e4503a75544358758921d',
+        'info_dict': {
+            'id': '5839623',
+            'title': 'Os caimáns do Tea',
+            'ext': 'mp4',
+            'description': 'md5:f71cfba21ae564f0a6f415b31de1f842',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_url = self._search_regex(r'var\s+url\s*=\s*["\']([^"\']+)', webpage, 'video url')
+        formats = self._extract_m3u8_formats(video_url + '/playlist.m3u8', video_id, fatal=False)
+        formats.extend(self._extract_mpd_formats(video_url + '/manifest.mpd', video_id, fatal=False))
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': remove_end(self._html_search_meta(
+                ['og:title', 'twitter:title'], webpage, 'title', default=None), ' | CRTVG'),
+            'description': self._html_search_meta('description', webpage, 'description', default=None),
+            'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage, 'thumbnail', default=None),
+        }

From 1fe5bf240e6ade487d18079a62aa36bcc440a27a Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 30 May 2023 10:43:01 -0500
Subject: [PATCH 318/871] [extractor/bravotv] Detect DRM (#7171)

Authored by: bashonly
---
 yt_dlp/extractor/bravotv.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/bravotv.py b/yt_dlp/extractor/bravotv.py
index d4bf9b53b..13cc1927f 100644
--- a/yt_dlp/extractor/bravotv.py
+++ b/yt_dlp/extractor/bravotv.py
@@ -1,5 +1,6 @@
 from .adobepass import AdobePassIE
 from ..utils import (
+    HEADRequest,
     extract_attributes,
     float_or_none,
     get_element_html_by_class,
@@ -153,8 +154,11 @@ def _real_extract(self, url):
         if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')):
             chapters = None
 
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
-            update_url_query(f'{tp_url}/stream.m3u8', query), video_id, 'mp4', m3u8_id='hls')
+        m3u8_url = self._request_webpage(HEADRequest(
+            update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').geturl()
+        if 'mpeg_cenc' in m3u8_url:
+            self.report_drm(video_id)
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
 
         return {
             'id': video_id,

From c2502cfed91415c7ccfff925fd3404d230046484 Mon Sep 17 00:00:00 2001
From: mrscrapy <116454146+mrscrapy@users.noreply.github.com>
Date: Wed, 31 May 2023 04:41:21 +0100
Subject: [PATCH 319/871] [extractor/recurbate] Add extractor (#6297)

Authored by: mrscrapy
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/recurbate.py   | 43 +++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 yt_dlp/extractor/recurbate.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 85c584f5e..abe3c2288 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1594,6 +1594,7 @@
     RCTIPlusTVIE,
 )
 from .rds import RDSIE
+from .recurbate import RecurbateIE
 from .redbee import ParliamentLiveUKIE, RTBFIE
 from .redbulltv import (
     RedBullTVIE,
diff --git a/yt_dlp/extractor/recurbate.py b/yt_dlp/extractor/recurbate.py
new file mode 100644
index 000000000..5534cf3c3
--- /dev/null
+++ b/yt_dlp/extractor/recurbate.py
@@ -0,0 +1,43 @@
+import urllib.error
+
+from .common import InfoExtractor
+from ..utils import ExtractorError, merge_dicts
+
+
+class RecurbateIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?recurbate\.com/play\.php\?video=(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://recurbate.com/play.php?video=39161415',
+        'md5': 'dd2b4ec57aa3e3572cb5cf0997fca99f',
+        'info_dict': {
+            'id': '39161415',
+            'ext': 'mp4',
+            'description': 'md5:db48d09e4d93fc715f47fd3d6b7edd51',
+            'title': 'Performer zsnicole33 show on 2022-10-25 20:23, Chaturbate Archive – Recurbate',
+            'age_limit': 18,
+        },
+        'skip': 'Website require membership.',
+    }]
+
+    def _real_extract(self, url):
+        SUBSCRIPTION_MISSING_MESSAGE = 'This video is only available for registered users; Set your authenticated browser user agent via the --user-agent parameter.'
+        video_id = self._match_id(url)
+        try:
+            webpage = self._download_webpage(url, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
+            raise
+        token = self._html_search_regex(r'data-token="([^"]+)"', webpage, 'token')
+        video_url = f'https://recurbate.com/api/get.php?video={video_id}&token={token}'
+
+        video_webpage = self._download_webpage(video_url, video_id)
+        if video_webpage == 'shall_subscribe':
+            self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
+        entries = self._parse_html5_media_entries(video_url, video_webpage, video_id)
+        return merge_dicts({
+            'id': video_id,
+            'title': self._html_extract_title(webpage, 'title'),
+            'description': self._og_search_description(webpage),
+            'age_limit': self._rta_search(webpage),
+        }, entries[0])

From 18f8fba7c89a87f99cc3313a1795848867e84fff Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Wed, 31 May 2023 19:08:28 +1200
Subject: [PATCH 320/871] [extractor/youtube] Fix continuation loop with no
 comments (#7148)

Deep check the response for incomplete data.

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 3f0a4cd20..ae4b58205 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3314,7 +3314,7 @@ def extract_header(contents):
                 expected_comment_count = self._get_count(
                     comments_header_renderer, 'countText', 'commentsCount')
 
-                if expected_comment_count:
+                if expected_comment_count is not None:
                     tracker['est_total'] = expected_comment_count
                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
@@ -3385,7 +3385,7 @@ def extract_thread(contents):
         if not tracker:
             tracker = dict(
                 running_total=0,
-                est_total=0,
+                est_total=None,
                 current_page_thread=0,
                 total_parent_comments=0,
                 total_reply_comments=0,
@@ -3418,11 +3418,13 @@ def extract_thread(contents):
             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
             is_forced_continuation = True
 
+        continuation_items_path = (
+            'onResponseReceivedEndpoints', ..., ('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems')
         for page_num in itertools.count(0):
             if not continuation:
                 break
             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
-            comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
+            comment_prog_str = f"({tracker['running_total']}/~{tracker['est_total']})"
             if page_num == 0:
                 if is_first_continuation:
                     note_prefix = 'Downloading comment section API JSON'
@@ -3433,11 +3435,18 @@ def extract_thread(contents):
                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
                     '       ' if parent else '', ' replies' if parent else '',
                     page_num, comment_prog_str)
+
+            # Do a deep check for incomplete data as sometimes YouTube may return no comments for a continuation
+            # Ignore check if YouTube says the comment count is 0.
+            check_get_keys = None
+            if not is_forced_continuation and not (tracker['est_total'] == 0 and tracker['running_total'] == 0):
+                check_get_keys = [[*continuation_items_path, ..., (
+                    'commentsHeaderRenderer' if is_first_continuation else ('commentThreadRenderer', 'commentRenderer'))]]
             try:
                 response = self._extract_response(
                     item_id=None, query=continuation,
                     ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
-                    check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
+                    check_get_keys=check_get_keys)
             except ExtractorError as e:
                 # Ignore incomplete data error for replies if retries didn't work.
                 # This is to allow any other parent comments and comment threads to be downloaded.
@@ -3449,15 +3458,8 @@ def extract_thread(contents):
                 else:
                     raise
             is_forced_continuation = False
-            continuation_contents = traverse_obj(
-                response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
-
             continuation = None
-            for continuation_section in continuation_contents:
-                continuation_items = traverse_obj(
-                    continuation_section,
-                    (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
-                    get_all=False, expected_type=list) or []
+            for continuation_items in traverse_obj(response, continuation_items_path, expected_type=list, default=[]):
                 if is_first_continuation:
                     continuation = extract_header(continuation_items)
                     is_first_continuation = False

From ecfe47973f6603b5367fe2cc3c65274627d94516 Mon Sep 17 00:00:00 2001
From: Mohamed Al Mehairbi <62325490+ItzMaxTV@users.noreply.github.com>
Date: Wed, 31 May 2023 17:12:56 +0400
Subject: [PATCH 321/871] [extractor/elevensports] Add extractor (#7172)

Closes #6737
Authored by: ItzMaxTV
---
 yt_dlp/extractor/_extractors.py  |  1 +
 yt_dlp/extractor/elevensports.py | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 yt_dlp/extractor/elevensports.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index abe3c2288..808ede5ba 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -550,6 +550,7 @@
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE
 from .eitb import EitbIE
+from .elevensports import ElevenSportsIE
 from .ellentube import (
     EllenTubeIE,
     EllenTubeVideoIE,
diff --git a/yt_dlp/extractor/elevensports.py b/yt_dlp/extractor/elevensports.py
new file mode 100644
index 000000000..99c52b3a9
--- /dev/null
+++ b/yt_dlp/extractor/elevensports.py
@@ -0,0 +1,59 @@
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class ElevenSportsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?elevensports\.com/view/event/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://elevensports.com/view/event/clf46yr3kenn80jgrqsjmwefk',
+        'md5': 'c0958d9ff90e4503a75544358758921d',
+        'info_dict': {
+            'id': 'clf46yr3kenn80jgrqsjmwefk',
+            'title': 'Cleveland SC vs Lionsbridge FC',
+            'ext': 'mp4',
+            'description': 'md5:03b5238d6549f4ea1fddadf69b5e0b58',
+            'upload_date': '20230323',
+            'timestamp': 1679612400,
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }, {
+        'url': 'https://elevensports.com/view/event/clhpyd53b06160jez74qhgkmf',
+        'md5': 'c0958d9ff90e4503a75544358758921d',
+        'info_dict': {
+            'id': 'clhpyd53b06160jez74qhgkmf',
+            'title': 'AJNLF vs ARRAF',
+            'ext': 'mp4',
+            'description': 'md5:c8c5e75c78f37c6d15cd6c475e43a8c1',
+            'upload_date': '20230521',
+            'timestamp': 1684684800,
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        event_id = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['event']['mclsEventId']
+        event_data = self._download_json(
+            f'https://mcls-api.mycujoo.tv/bff/events/v1beta1/{event_id}', video_id,
+            headers={'Authorization': 'Bearer FBVKACGN37JQC5SFA0OVK8KKSIOP153G'})
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            event_data['streams'][0]['full_url'], video_id, 'mp4', m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(event_data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'timestamp': ('start_time', {parse_iso8601}),
+                'thumbnail': ('thumbnail_url', {url_or_none}),
+            }),
+        }

From 1c16d9df5330819cc79ad588b24aa5b72765c168 Mon Sep 17 00:00:00 2001
From: CeruleanSky <CeruleanSky@users.noreply.github.com>
Date: Thu, 1 Jun 2023 02:35:41 -0400
Subject: [PATCH 322/871] [extractor/twitter:spaces] Add `release_timestamp`
 (#7186)

Authored by: CeruleanSky
---
 yt_dlp/extractor/twitter.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index d9a89c44b..4624ce503 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -705,6 +705,7 @@ class TwitterIE(TwitterBaseIE):
             'uploader': r're:Monique Camarra.+?',
             'uploader_id': 'MoniqueCamarra',
             'live_status': 'was_live',
+            'release_timestamp': 1658417414,
             'description': 'md5:acce559345fd49f129c20dbcda3f1201',
             'timestamp': 1658407771464,
         },
@@ -1327,6 +1328,8 @@ def _real_extract(self, url):
             'uploader_id': traverse_obj(
                 metadata, ('creator_results', 'result', 'legacy', 'screen_name')),
             'live_status': live_status,
+            'release_timestamp': try_call(
+                lambda: int_or_none(metadata['scheduled_start'], scale=1000)),
             'timestamp': metadata.get('created_at'),
             'formats': formats,
         }

From c35448b7b14113b35c4415dbfbf488c4731f006f Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Thu, 1 Jun 2023 20:43:32 +1200
Subject: [PATCH 323/871] [extractor/youtube] Extract more metadata for
 comments (#7179)

Adds new comment fields:
* `author_url` - The url to the comment author's page
* `author_is_verified` - Whether the author is verified on the platform
* `is_pinned` - Whether the comment is pinned to the top of the comments

Closes https://github.com/yt-dlp/yt-dlp/issues/5411

Authored by: coletdjnz
---
 yt_dlp/extractor/common.py  |  9 +++--
 yt_dlp/extractor/youtube.py | 68 ++++++++++++++++++++++---------------
 2 files changed, 47 insertions(+), 30 deletions(-)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 306911a6c..fa46a5240 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -314,6 +314,11 @@ class InfoExtractor:
                         * "author" - human-readable name of the comment author
                         * "author_id" - user ID of the comment author
                         * "author_thumbnail" - The thumbnail of the comment author
+                        * "author_url" - The url to the comment author's page
+                        * "author_is_verified" - Whether the author is verified
+                                                 on the platform
+                        * "author_is_uploader" - Whether the comment is made by
+                                                 the video uploader
                         * "id" - Comment ID
                         * "html" - Comment as HTML
                         * "text" - Plain text of the comment
@@ -325,8 +330,8 @@ class InfoExtractor:
                         * "dislike_count" - Number of negative ratings of the comment
                         * "is_favorited" - Whether the comment is marked as
                                            favorite by the video uploader
-                        * "author_is_uploader" - Whether the comment is made by
-                                                 the video uploader
+                        * "is_pinned" - Whether the comment is pinned to
+                                        the top of the comments
     age_limit:      Age restriction for the video, as an integer (years)
     webpage_url:    The URL to the video webpage, if given to yt-dlp it
                     should allow to get the same result again. (It will be set
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index ae4b58205..ccf97705a 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3271,37 +3271,50 @@ def _extract_comment(self, comment_renderer, parent=None):
         if not comment_id:
             return
 
-        text = self._get_text(comment_renderer, 'contentText')
+        info = {
+            'id': comment_id,
+            'text': self._get_text(comment_renderer, 'contentText'),
+            'like_count': self._get_count(comment_renderer, 'voteCount'),
+            'author_id': traverse_obj(comment_renderer, ('authorEndpoint', 'browseEndpoint', 'browseId', {self.ucid_or_none})),
+            'author': self._get_text(comment_renderer, 'authorText'),
+            'author_thumbnail': traverse_obj(comment_renderer, ('authorThumbnail', 'thumbnails', -1, 'url', {url_or_none})),
+            'parent': parent or 'root',
+        }
 
         # Timestamp is an estimate calculated from the current time and time_text
         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
         timestamp = self._parse_time_text(time_text)
 
-        author = self._get_text(comment_renderer, 'authorText')
-        author_id = try_get(comment_renderer,
-                            lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
-
-        votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
-                                                       lambda x: x['likeCount']), str)) or 0
-        author_thumbnail = try_get(comment_renderer,
-                                   lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
-
-        author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
-        is_favorited = 'creatorHeart' in (try_get(
-            comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
-        return {
-            'id': comment_id,
-            'text': text,
+        info.update({
+            # FIXME: non-standard, but we need a way of showing that it is an estimate.
+            '_time_text': time_text,
             'timestamp': timestamp,
-            'time_text': time_text,
-            'like_count': votes,
-            'is_favorited': is_favorited,
-            'author': author,
-            'author_id': author_id,
-            'author_thumbnail': author_thumbnail,
-            'author_is_uploader': author_is_uploader,
-            'parent': parent or 'root'
-        }
+        })
+
+        info['author_url'] = urljoin(
+            'https://www.youtube.com', traverse_obj(comment_renderer, ('authorEndpoint', (
+                ('browseEndpoint', 'canonicalBaseUrl'), ('commandMetadata', 'webCommandMetadata', 'url'))),
+                expected_type=str, get_all=False))
+
+        author_is_uploader = traverse_obj(comment_renderer, 'authorIsChannelOwner')
+        if author_is_uploader is not None:
+            info['author_is_uploader'] = author_is_uploader
+
+        comment_abr = traverse_obj(
+            comment_renderer, ('actionsButtons', 'commentActionButtonsRenderer'), expected_type=dict)
+        if comment_abr is not None:
+            info['is_favorited'] = 'creatorHeart' in comment_abr
+
+        comment_ab_icontype = traverse_obj(
+            comment_renderer, ('authorCommentBadge', 'authorCommentBadgeRenderer', 'icon', 'iconType'))
+        if comment_ab_icontype is not None:
+            info['author_is_verified'] = comment_ab_icontype in ('CHECK_CIRCLE_THICK', 'OFFICIAL_ARTIST_BADGE')
+
+        is_pinned = traverse_obj(comment_renderer, 'pinnedCommentBadge')
+        if is_pinned:
+            info['is_pinned'] = True
+
+        return info
 
     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
 
@@ -3349,14 +3362,13 @@ def extract_thread(contents):
                 comment = self._extract_comment(comment_renderer, parent)
                 if not comment:
                     continue
-                is_pinned = bool(traverse_obj(comment_renderer, 'pinnedCommentBadge'))
                 comment_id = comment['id']
-                if is_pinned:
+                if comment.get('is_pinned'):
                     tracker['pinned_comment_ids'].add(comment_id)
                 # Sometimes YouTube may break and give us infinite looping comments.
                 # See: https://github.com/yt-dlp/yt-dlp/issues/6290
                 if comment_id in tracker['seen_comment_ids']:
-                    if comment_id in tracker['pinned_comment_ids'] and not is_pinned:
+                    if comment_id in tracker['pinned_comment_ids'] and not comment.get('is_pinned'):
                         # Pinned comments may appear a second time in newest first sort
                         # See: https://github.com/yt-dlp/yt-dlp/issues/6712
                         continue

From f41b949a2ef646fbc36375febbe3f0c19d742c0f Mon Sep 17 00:00:00 2001
From: Daniel Rich <drich@employees.org>
Date: Thu, 1 Jun 2023 14:52:03 -0700
Subject: [PATCH 324/871] [extractor/nhk] Fix API extraction (#7180)

Closes #6992
Authored by: sjthespian, menschel

Co-authored-by: Patrick Menschel <menschel.p@posteo.de>
---
 yt_dlp/extractor/nhk.py    | 36 ++++++++++++++++++++++++++++--------
 yt_dlp/extractor/piksel.py | 16 +++++++++-------
 2 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py
index 1597962ac..a3efa326a 100644
--- a/yt_dlp/extractor/nhk.py
+++ b/yt_dlp/extractor/nhk.py
@@ -67,7 +67,7 @@ def get_clean_field(key):
             info.update({
                 '_type': 'url_transparent',
                 'ie_key': 'Piksel',
-                'url': 'https://player.piksel.com/v/refid/nhkworld/prefid/' + vod_id,
+                'url': 'https://movie-s.nhk.or.jp/v/refid/nhkworld/prefid/' + vod_id,
                 'id': vod_id,
             })
         else:
@@ -94,6 +94,19 @@ class NhkVodIE(NhkBaseIE):
     # Content available only for a limited period of time. Visit
     # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
     _TESTS = [{
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2061601/',
+        'info_dict': {
+            'id': 'yd8322ch',
+            'ext': 'mp4',
+            'description': 'md5:109c8b05d67a62d0592f2b445d2cd898',
+            'title': 'GRAND SUMO Highlights - [Recap] May Tournament Day 1 (Opening Day)',
+            'upload_date': '20230514',
+            'timestamp': 1684083791,
+            'series': 'GRAND SUMO Highlights',
+            'episode': '[Recap] May Tournament Day 1 (Opening Day)',
+            'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1684084443/4028649.jpg?w=1920&h=1080',
+        },
+    }, {
         # video clip
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
         'md5': '7a90abcfe610ec22a6bfe15bd46b30ca',
@@ -104,6 +117,9 @@ class NhkVodIE(NhkBaseIE):
             'description': 'md5:5aee4a9f9d81c26281862382103b0ea5',
             'timestamp': 1565965194,
             'upload_date': '20190816',
+            'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1567086278/3715195.jpg?w=1920&h=1080',
+            'series': 'Dining with the Chef',
+            'episode': 'Chef Saito\'s Family recipe: MENCHI-KATSU',
         },
     }, {
         # audio clip
@@ -114,10 +130,7 @@ class NhkVodIE(NhkBaseIE):
             'title': "Japan's Top Inventions - Miniature Video Cameras",
             'description': 'md5:07ea722bdbbb4936fdd360b6a480c25b',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
+        'skip': '404 Not Found',
     }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/',
         'only_matching': True,
@@ -133,7 +146,6 @@ class NhkVodIE(NhkBaseIE):
     }, {
         # video, alphabetic character in ID #29670
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999a34/',
-        'only_matching': True,
         'info_dict': {
             'id': 'qfjay6cg',
             'ext': 'mp4',
@@ -142,7 +154,8 @@ class NhkVodIE(NhkBaseIE):
             'thumbnail': r're:^https?:/(/[a-z0-9.-]+)+\.jpg\?w=1920&h=1080$',
             'upload_date': '20210615',
             'timestamp': 1623722008,
-        }
+        },
+        'skip': '404 Not Found',
     }]
 
     def _real_extract(self, url):
@@ -153,12 +166,19 @@ class NhkVodProgramIE(NhkBaseIE):
     _VALID_URL = r'%s/program%s(?P<id>[0-9a-z]+)(?:.+?\btype=(?P<episode_type>clip|(?:radio|tv)Episode))?' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
     _TESTS = [{
         # video program episodes
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/sumo',
+        'info_dict': {
+            'id': 'sumo',
+            'title': 'GRAND SUMO Highlights',
+        },
+        'playlist_mincount': 12,
+    }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway',
         'info_dict': {
             'id': 'japanrailway',
             'title': 'Japan Railway Journal',
         },
-        'playlist_mincount': 1,
+        'playlist_mincount': 12,
     }, {
         # video program clips
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway/?type=clip',
diff --git a/yt_dlp/extractor/piksel.py b/yt_dlp/extractor/piksel.py
index cc60b304e..97a9bf574 100644
--- a/yt_dlp/extractor/piksel.py
+++ b/yt_dlp/extractor/piksel.py
@@ -7,8 +7,10 @@
     int_or_none,
     join_nonempty,
     parse_iso8601,
+    traverse_obj,
     try_get,
     unescapeHTML,
+    urljoin,
 )
 
 
@@ -63,11 +65,11 @@ class PikselIE(InfoExtractor):
         }
     ]
 
-    def _call_api(self, app_token, resource, display_id, query, fatal=True):
-        response = (self._download_json(
-            'http://player.piksel.com/ws/ws_%s/api/%s/mode/json/apiv/5' % (resource, app_token),
-            display_id, query=query, fatal=fatal) or {}).get('response')
-        failure = try_get(response, lambda x: x['failure']['reason'])
+    def _call_api(self, app_token, resource, display_id, query, host='https://player.piksel.com', fatal=True):
+        url = urljoin(host, f'/ws/ws_{resource}/api/{app_token}/mode/json/apiv/5')
+        response = traverse_obj(
+            self._download_json(url, display_id, query=query, fatal=fatal), ('response', {dict})) or {}
+        failure = traverse_obj(response, ('failure', 'reason')) if response else 'Empty response from API'
         if failure:
             if fatal:
                 raise ExtractorError(failure, expected=True)
@@ -83,7 +85,7 @@ def _real_extract(self, url):
         ], webpage, 'app token')
         query = {'refid': ref_id, 'prefid': display_id} if ref_id else {'v': display_id}
         program = self._call_api(
-            app_token, 'program', display_id, query)['WsProgramResponse']['program']
+            app_token, 'program', display_id, query, url)['WsProgramResponse']['program']
         video_id = program['uuid']
         video_data = program['asset']
         title = video_data['title']
@@ -129,7 +131,7 @@ def process_asset_files(asset_files):
                 process_asset_files(try_get(self._call_api(
                     app_token, 'asset_file', display_id, {
                         'assetid': asset_id,
-                    }, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
+                    }, url, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
 
         m3u8_url = dict_get(video_data, [
             'm3u8iPadURL',

From 01231feb142e80828985aabdec04ac608e3d43e2 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 2 Jun 2023 08:39:24 -0500
Subject: [PATCH 325/871] [extractor/twitch] Update `_CLIENT_ID` and add
 extractor-arg (#7200)

Closes #7058, Closes #7183
Authored by: bashonly
---
 README.md                  | 3 +++
 yt_dlp/extractor/twitch.py | 6 +++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 25ed3b844..3d89c0af9 100644
--- a/README.md
+++ b/README.md
@@ -1846,6 +1846,9 @@ #### twitter
 ### wrestleuniverse
 * `device_id`: UUID value assigned by the website and used to enforce device limits for paid livestream content. Can be found in browser local storage
 
+#### twitchstream (Twitch)
+* `client_id`: Client ID value to be sent with GraphQL requests, e.g. `twitchstream:client_id=kimne78kx3ncx6brgo4mv6wki5h1ko`
+
 **Note**: These options may be changed/removed in the future without concern for backward compatibility
 
 <!-- MANPAGE: MOVE "INSTALLATION" SECTION HERE -->
diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index 4a17d8048..31b349bc6 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -41,7 +41,6 @@ class TwitchBaseIE(InfoExtractor):
     _USHER_BASE = 'https://usher.ttvnw.net'
     _LOGIN_FORM_URL = 'https://www.twitch.tv/login'
     _LOGIN_POST_URL = 'https://passport.twitch.tv/login'
-    _CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
     _NETRC_MACHINE = 'twitch'
 
     _OPERATION_HASHES = {
@@ -58,6 +57,11 @@ class TwitchBaseIE(InfoExtractor):
         'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
     }
 
+    @property
+    def _CLIENT_ID(self):
+        return self._configuration_arg(
+            'client_id', ['ue6666qo983tsx6so1t0vnawi233wa'], ie_key=TwitchStreamIE, casesense=True)[0]
+
     def _perform_login(self, username, password):
         def fail(message):
             raise ExtractorError(

From 55ed4ff73487feb3177b037dfc2ea527e777da3e Mon Sep 17 00:00:00 2001
From: Mohamed Al Mehairbi <62325490+ItzMaxTV@users.noreply.github.com>
Date: Fri, 2 Jun 2023 19:01:55 +0400
Subject: [PATCH 326/871] [extractor/DigitalConcertHall] Support films (#7202)

Authored by: ItzMaxTV
Closes #7184
---
 yt_dlp/extractor/digitalconcerthall.py | 27 +++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/extractor/digitalconcerthall.py b/yt_dlp/extractor/digitalconcerthall.py
index 3461e36eb..c11cd790b 100644
--- a/yt_dlp/extractor/digitalconcerthall.py
+++ b/yt_dlp/extractor/digitalconcerthall.py
@@ -11,7 +11,7 @@
 
 class DigitalConcertHallIE(InfoExtractor):
     IE_DESC = 'DigitalConcertHall extractor'
-    _VALID_URL = r'https?://(?:www\.)?digitalconcerthall\.com/(?P<language>[a-z]+)/concert/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?digitalconcerthall\.com/(?P<language>[a-z]+)/(?P<type>film|concert)/(?P<id>[0-9]+)'
     _OAUTH_URL = 'https://api.digitalconcerthall.com/v2/oauth2/token'
     _ACCESS_TOKEN = None
     _NETRC_MACHINE = 'digitalconcerthall'
@@ -40,6 +40,19 @@ class DigitalConcertHallIE(InfoExtractor):
         },
         'params': {'skip_download': 'm3u8'},
         'playlist_count': 3,
+    }, {
+        'url': 'https://www.digitalconcerthall.com/en/film/388',
+        'info_dict': {
+            'id': '388',
+            'ext': 'mp4',
+            'title': 'The Berliner Philharmoniker and Frank Peter Zimmermann',
+            'description': 'md5:cfe25a7044fa4be13743e5089b5b5eb2',
+            'thumbnail': r're:^https?://images.digitalconcerthall.com/cms/thumbnails.*\.jpg$',
+            'upload_date': '20220714',
+            'timestamp': 1657785600,
+            'album_artist': 'Frank Peter Zimmermann / Benedikt von Bernstorff / Jakob von Bernstorff',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _perform_login(self, username, password):
@@ -75,7 +88,7 @@ def _real_initialize(self):
         if not self._ACCESS_TOKEN:
             self.raise_login_required(method='password')
 
-    def _entries(self, items, language, **kwargs):
+    def _entries(self, items, language, type_, **kwargs):
         for item in items:
             video_id = item['id']
             stream_info = self._download_json(
@@ -103,11 +116,11 @@ def _entries(self, items, language, **kwargs):
                     'start_time': chapter.get('time'),
                     'end_time': try_get(chapter, lambda x: x['time'] + x['duration']),
                     'title': chapter.get('text'),
-                } for chapter in item['cuepoints']] if item.get('cuepoints') else None,
+                } for chapter in item['cuepoints']] if item.get('cuepoints') and type_ == 'concert' else None,
             }
 
     def _real_extract(self, url):
-        language, video_id = self._match_valid_url(url).group('language', 'id')
+        language, type_, video_id = self._match_valid_url(url).group('language', 'type', 'id')
         if not language:
             language = 'en'
 
@@ -120,18 +133,18 @@ def _real_extract(self, url):
         }]
 
         vid_info = self._download_json(
-            f'https://api.digitalconcerthall.com/v2/concert/{video_id}', video_id, headers={
+            f'https://api.digitalconcerthall.com/v2/{type_}/{video_id}', video_id, headers={
                 'Accept': 'application/json',
                 'Accept-Language': language
             })
         album_artist = ' / '.join(traverse_obj(vid_info, ('_links', 'artist', ..., 'name')) or '')
+        videos = [vid_info] if type_ == 'film' else traverse_obj(vid_info, ('_embedded', ..., ...))
 
         return {
             '_type': 'playlist',
             'id': video_id,
             'title': vid_info.get('title'),
-            'entries': self._entries(traverse_obj(vid_info, ('_embedded', ..., ...)), language,
-                                     thumbnails=thumbnails, album_artist=album_artist),
+            'entries': self._entries(videos, language, thumbnails=thumbnails, album_artist=album_artist, type_=type_),
             'thumbnails': thumbnails,
             'album_artist': album_artist,
         }

From 1a7dcca378e80a387923ee05c250d8ba122441c6 Mon Sep 17 00:00:00 2001
From: Jeroen Jacobs <git@jeroenj.be>
Date: Fri, 2 Jun 2023 20:29:00 +0200
Subject: [PATCH 327/871] [extractor/vrt] Overhaul extractors (#6244)

* Fixes `VrtNU` extractor to work with the VRT MAX site change
* Adapts `VRT`, `Ketnet` and `DagelijkseKost` extractors to the new VRT API
* Removes `Canvas` and `CanvasEen` extractors; the sites and API no longer exist
* Moves all remaining VRT-related extractors into the `vrt` module

Closes #4908
Authored by: jeroenj, bergoid, bashonly

Co-authored-by: bergoid <bergoid@users.noreply.github.com>
Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
---
 yt_dlp/extractor/_extractors.py |  14 +-
 yt_dlp/extractor/canvas.py      | 383 -----------------------------
 yt_dlp/extractor/ketnet.py      |  70 ------
 yt_dlp/extractor/vrt.py         | 413 +++++++++++++++++++++++++++++---
 4 files changed, 384 insertions(+), 496 deletions(-)
 delete mode 100644 yt_dlp/extractor/canvas.py
 delete mode 100644 yt_dlp/extractor/ketnet.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 808ede5ba..7120fd37d 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -295,12 +295,6 @@
 from .canalalpha import CanalAlphaIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
-from .canvas import (
-    CanvasIE,
-    CanvasEenIE,
-    VrtNUIE,
-    DagelijkseKostIE,
-)
 from .carambatv import (
     CarambaTVIE,
     CarambaTVPageIE,
@@ -894,7 +888,6 @@
 from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .kelbyone import KelbyOneIE
-from .ketnet import KetnetIE
 from .khanacademy import (
     KhanAcademyIE,
     KhanAcademyUnitIE,
@@ -2285,7 +2278,12 @@
     VoxMediaVolumeIE,
     VoxMediaIE,
 )
-from .vrt import VRTIE
+from .vrt import (
+    VRTIE,
+    VrtNUIE,
+    KetnetIE,
+    DagelijkseKostIE,
+)
 from .vrak import VrakIE
 from .vrv import (
     VRVIE,
diff --git a/yt_dlp/extractor/canvas.py b/yt_dlp/extractor/canvas.py
deleted file mode 100644
index ae6e03a4d..000000000
--- a/yt_dlp/extractor/canvas.py
+++ /dev/null
@@ -1,383 +0,0 @@
-import json
-
-
-from .common import InfoExtractor
-from .gigya import GigyaBaseIE
-from ..compat import compat_HTTPError
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    extract_attributes,
-    float_or_none,
-    get_element_by_class,
-    int_or_none,
-    merge_dicts,
-    str_or_none,
-    strip_or_none,
-    url_or_none,
-    urlencode_postdata
-)
-
-
-class CanvasIE(InfoExtractor):
-    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
-        'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
-        'info_dict': {
-            'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
-            'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
-            'ext': 'mp4',
-            'title': 'Nachtwacht: De Greystook',
-            'description': 'Nachtwacht: De Greystook',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 1468.02,
-        },
-        'expected_warnings': ['is not a supported codec'],
-    }, {
-        'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
-        'only_matching': True,
-    }]
-    _GEO_BYPASS = False
-    _HLS_ENTRY_PROTOCOLS_MAP = {
-        'HLS': 'm3u8_native',
-        'HLS_AES': 'm3u8_native',
-    }
-    _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        site_id, video_id = mobj.group('site_id'), mobj.group('id')
-
-        data = None
-        if site_id != 'vrtvideo':
-            # Old API endpoint, serves more formats but may fail for some videos
-            data = self._download_json(
-                'https://mediazone.vrt.be/api/v1/%s/assets/%s'
-                % (site_id, video_id), video_id, 'Downloading asset JSON',
-                'Unable to download asset JSON', fatal=False)
-
-        # New API endpoint
-        if not data:
-            vrtnutoken = self._download_json('https://token.vrt.be/refreshtoken',
-                                             video_id, note='refreshtoken: Retrieve vrtnutoken',
-                                             errnote='refreshtoken failed')['vrtnutoken']
-            headers = self.geo_verification_headers()
-            headers.update({'Content-Type': 'application/json; charset=utf-8'})
-            vrtPlayerToken = self._download_json(
-                '%s/tokens' % self._REST_API_BASE, video_id,
-                'Downloading token', headers=headers, data=json.dumps({
-                    'identityToken': vrtnutoken
-                }).encode('utf-8'))['vrtPlayerToken']
-            data = self._download_json(
-                '%s/videos/%s' % (self._REST_API_BASE, video_id),
-                video_id, 'Downloading video JSON', query={
-                    'vrtPlayerToken': vrtPlayerToken,
-                    'client': 'null',
-                }, expected_status=400)
-            if 'title' not in data:
-                code = data.get('code')
-                if code == 'AUTHENTICATION_REQUIRED':
-                    self.raise_login_required()
-                elif code == 'INVALID_LOCATION':
-                    self.raise_geo_restricted(countries=['BE'])
-                raise ExtractorError(data.get('message') or code, expected=True)
-
-        # Note: The title may be an empty string
-        title = data['title'] or f'{site_id} {video_id}'
-        description = data.get('description')
-
-        formats = []
-        subtitles = {}
-        for target in data['targetUrls']:
-            format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type'))
-            if not format_url or not format_type:
-                continue
-            format_type = format_type.upper()
-            if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
-                fmts, subs = self._extract_m3u8_formats_and_subtitles(
-                    format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
-                    m3u8_id=format_type, fatal=False)
-                formats.extend(fmts)
-                subtitles = self._merge_subtitles(subtitles, subs)
-            elif format_type == 'HDS':
-                formats.extend(self._extract_f4m_formats(
-                    format_url, video_id, f4m_id=format_type, fatal=False))
-            elif format_type == 'MPEG_DASH':
-                fmts, subs = self._extract_mpd_formats_and_subtitles(
-                    format_url, video_id, mpd_id=format_type, fatal=False)
-                formats.extend(fmts)
-                subtitles = self._merge_subtitles(subtitles, subs)
-            elif format_type == 'HSS':
-                fmts, subs = self._extract_ism_formats_and_subtitles(
-                    format_url, video_id, ism_id='mss', fatal=False)
-                formats.extend(fmts)
-                subtitles = self._merge_subtitles(subtitles, subs)
-            else:
-                formats.append({
-                    'format_id': format_type,
-                    'url': format_url,
-                })
-
-        subtitle_urls = data.get('subtitleUrls')
-        if isinstance(subtitle_urls, list):
-            for subtitle in subtitle_urls:
-                subtitle_url = subtitle.get('url')
-                if subtitle_url and subtitle.get('type') == 'CLOSED':
-                    subtitles.setdefault('nl', []).append({'url': subtitle_url})
-
-        return {
-            'id': video_id,
-            'display_id': video_id,
-            'title': title,
-            'description': description,
-            'formats': formats,
-            'duration': float_or_none(data.get('duration'), 1000),
-            'thumbnail': data.get('posterImageUrl'),
-            'subtitles': subtitles,
-        }
-
-
-class CanvasEenIE(InfoExtractor):
-    IE_DESC = 'canvas.be and een.be'
-    _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
-        'md5': 'ed66976748d12350b118455979cca293',
-        'info_dict': {
-            'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
-            'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
-            'ext': 'flv',
-            'title': 'De afspraak veilt voor de Warmste Week',
-            'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 49.02,
-        },
-        'expected_warnings': ['is not a supported codec'],
-    }, {
-        # with subtitles
-        'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
-        'info_dict': {
-            'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
-            'display_id': 'pieter-0167',
-            'ext': 'mp4',
-            'title': 'Pieter 0167',
-            'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 2553.08,
-            'subtitles': {
-                'nl': [{
-                    'ext': 'vtt',
-                }],
-            },
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'skip': 'Pagina niet gevonden',
-    }, {
-        'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
-        'info_dict': {
-            'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
-            'display_id': 'emma-pakt-thilly-aan',
-            'ext': 'mp4',
-            'title': 'Emma pakt Thilly aan',
-            'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 118.24,
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'expected_warnings': ['is not a supported codec'],
-    }, {
-        'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        site_id, display_id = mobj.group('site_id'), mobj.group('id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        title = strip_or_none(self._search_regex(
-            r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
-            webpage, 'title', default=None) or self._og_search_title(
-            webpage, default=None))
-
-        video_id = self._html_search_regex(
-            r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
-            group='id')
-
-        return {
-            '_type': 'url_transparent',
-            'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
-            'ie_key': CanvasIE.ie_key(),
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': self._og_search_description(webpage),
-        }
-
-
-class VrtNUIE(GigyaBaseIE):
-    IE_DESC = 'VrtNU.be'
-    _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        # Available via old API endpoint
-        'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
-        'info_dict': {
-            'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
-            'ext': 'mp4',
-            'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
-            'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
-            'duration': 1457.04,
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'series': 'Postbus X',
-            'season': 'Seizoen 1989',
-            'season_number': 1989,
-            'episode': 'De zwarte weduwe',
-            'episode_number': 1,
-            'timestamp': 1595822400,
-            'upload_date': '20200727',
-        },
-        'skip': 'This video is only available for registered users',
-        'expected_warnings': ['is not a supported codec'],
-    }, {
-        # Only available via new API endpoint
-        'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
-        'info_dict': {
-            'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
-            'ext': 'mp4',
-            'title': 'Aflevering 5',
-            'description': 'Wie valt door de mand tijdens een missie?',
-            'duration': 2967.06,
-            'season': 'Season 1',
-            'season_number': 1,
-            'episode_number': 5,
-        },
-        'skip': 'This video is only available for registered users',
-        'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
-    }]
-    _NETRC_MACHINE = 'vrtnu'
-    _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
-    _CONTEXT_ID = 'R3595707040'
-
-    def _perform_login(self, username, password):
-        auth_info = self._gigya_login({
-            'APIKey': self._APIKEY,
-            'targetEnv': 'jssdk',
-            'loginID': username,
-            'password': password,
-            'authMode': 'cookie',
-        })
-
-        if auth_info.get('errorDetails'):
-            raise ExtractorError('Unable to login: VrtNU said: ' + auth_info.get('errorDetails'), expected=True)
-
-        # Sometimes authentication fails for no good reason, retry
-        login_attempt = 1
-        while login_attempt <= 3:
-            try:
-                self._request_webpage('https://token.vrt.be/vrtnuinitlogin',
-                                      None, note='Requesting XSRF Token', errnote='Could not get XSRF Token',
-                                      query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'})
-
-                post_data = {
-                    'UID': auth_info['UID'],
-                    'UIDSignature': auth_info['UIDSignature'],
-                    'signatureTimestamp': auth_info['signatureTimestamp'],
-                    '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
-                }
-
-                self._request_webpage(
-                    'https://login.vrt.be/perform_login',
-                    None, note='Performing login', errnote='perform login failed',
-                    headers={}, query={
-                        'client_id': 'vrtnu-site'
-                    }, data=urlencode_postdata(post_data))
-
-            except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                    login_attempt += 1
-                    self.report_warning('Authentication failed')
-                    self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
-                else:
-                    raise e
-            else:
-                break
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        attrs = extract_attributes(self._search_regex(
-            r'(<nui-media[^>]+>)', webpage, 'media element'))
-        video_id = attrs['videoid']
-        publication_id = attrs.get('publicationid')
-        if publication_id:
-            video_id = publication_id + '$' + video_id
-
-        page = (self._parse_json(self._search_regex(
-            r'digitalData\s*=\s*({.+?});', webpage, 'digial data',
-            default='{}'), video_id, fatal=False) or {}).get('page') or {}
-
-        info = self._search_json_ld(webpage, display_id, default={})
-        return merge_dicts(info, {
-            '_type': 'url_transparent',
-            'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
-            'ie_key': CanvasIE.ie_key(),
-            'id': video_id,
-            'display_id': display_id,
-            'season_number': int_or_none(page.get('episode_season')),
-        })
-
-
-class DagelijkseKostIE(InfoExtractor):
-    IE_DESC = 'dagelijksekost.een.be'
-    _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
-    _TEST = {
-        'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
-        'md5': '30bfffc323009a3e5f689bef6efa2365',
-        'info_dict': {
-            'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
-            'display_id': 'hachis-parmentier-met-witloof',
-            'ext': 'mp4',
-            'title': 'Hachis parmentier met witloof',
-            'description': 'md5:9960478392d87f63567b5b117688cdc5',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 283.02,
-        },
-        'expected_warnings': ['is not a supported codec'],
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        title = strip_or_none(get_element_by_class(
-            'dish-metadata__title', webpage
-        ) or self._html_search_meta(
-            'twitter:title', webpage))
-
-        description = clean_html(get_element_by_class(
-            'dish-description', webpage)
-        ) or self._html_search_meta(
-            ('description', 'twitter:description', 'og:description'),
-            webpage)
-
-        video_id = self._html_search_regex(
-            r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
-            group='id')
-
-        return {
-            '_type': 'url_transparent',
-            'url': 'https://mediazone.vrt.be/api/v1/dako/assets/%s' % video_id,
-            'ie_key': CanvasIE.ie_key(),
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-        }
diff --git a/yt_dlp/extractor/ketnet.py b/yt_dlp/extractor/ketnet.py
deleted file mode 100644
index ab6276727..000000000
--- a/yt_dlp/extractor/ketnet.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from .canvas import CanvasIE
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
-from ..utils import (
-    int_or_none,
-    parse_iso8601,
-)
-
-
-class KetnetIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?P<id>(?:[^/]+/)*[^/?#&]+)'
-    _TESTS = [{
-        'url': 'https://www.ketnet.be/kijken/n/nachtwacht/3/nachtwacht-s3a1-de-greystook',
-        'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
-        'info_dict': {
-            'id': 'pbs-pub-aef8b526-115e-4006-aa24-e59ff6c6ef6f$vid-ddb815bf-c8e7-467b-8879-6bad7a32cebd',
-            'ext': 'mp4',
-            'title': 'Nachtwacht - Reeks 3: Aflevering 1',
-            'description': 'De Nachtwacht krijgt te maken met een parasiet',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 1468.02,
-            'timestamp': 1609225200,
-            'upload_date': '20201229',
-            'series': 'Nachtwacht',
-            'season': 'Reeks 3',
-            'episode': 'De Greystook',
-            'episode_number': 1,
-        },
-        'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
-    }, {
-        'url': 'https://www.ketnet.be/themas/karrewiet/jaaroverzicht-20200/karrewiet-het-jaar-van-black-mamba',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        video = self._download_json(
-            'https://senior-bff.ketnet.be/graphql', display_id, query={
-                'query': '''{
-  video(id: "content/ketnet/nl/%s.model.json") {
-    description
-    episodeNr
-    imageUrl
-    mediaReference
-    programTitle
-    publicationDate
-    seasonTitle
-    subtitleVideodetail
-    titleVideodetail
-  }
-}''' % display_id,
-            })['data']['video']
-
-        mz_id = compat_urllib_parse_unquote(video['mediaReference'])
-
-        return {
-            '_type': 'url_transparent',
-            'id': mz_id,
-            'title': video['titleVideodetail'],
-            'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/' + mz_id,
-            'thumbnail': video.get('imageUrl'),
-            'description': video.get('description'),
-            'timestamp': parse_iso8601(video.get('publicationDate')),
-            'series': video.get('programTitle'),
-            'season': video.get('seasonTitle'),
-            'episode': video.get('subtitleVideodetail'),
-            'episode_number': int_or_none(video.get('episodeNr')),
-            'ie_key': CanvasIE.ie_key(),
-        }
diff --git a/yt_dlp/extractor/vrt.py b/yt_dlp/extractor/vrt.py
index 26f48bf67..bacd3df29 100644
--- a/yt_dlp/extractor/vrt.py
+++ b/yt_dlp/extractor/vrt.py
@@ -1,45 +1,137 @@
-from .common import InfoExtractor
+import functools
+import json
+import time
+import urllib.error
+import urllib.parse
+
+from .gigya import GigyaBaseIE
 from ..utils import (
+    ExtractorError,
+    clean_html,
     extract_attributes,
     float_or_none,
     get_element_by_class,
+    get_element_html_by_class,
+    int_or_none,
+    join_nonempty,
+    jwt_encode_hs256,
+    make_archive_id,
+    parse_age_limit,
+    parse_iso8601,
+    str_or_none,
     strip_or_none,
-    unified_timestamp,
+    traverse_obj,
+    url_or_none,
+    urlencode_postdata,
 )
 
 
-class VRTIE(InfoExtractor):
+class VRTBaseIE(GigyaBaseIE):
+    _GEO_BYPASS = False
+    _PLAYER_INFO = {
+        'platform': 'desktop',
+        'app': {
+            'type': 'browser',
+            'name': 'Chrome',
+        },
+        'device': 'undefined (undefined)',
+        'os': {
+            'name': 'Windows',
+            'version': 'x86_64'
+        },
+        'player': {
+            'name': 'VRT web player',
+            'version': '2.7.4-prod-2023-04-19T06:05:45'
+        }
+    }
+    # From https://player.vrt.be/vrtnws/js/main.js & https://player.vrt.be/ketnet/js/main.fd1de01a40a1e3d842ea.js
+    _JWT_KEY_ID = '0-0Fp51UZykfaiCJrfTE3+oMI8zvDteYfPtR+2n1R+z8w='
+    _JWT_SIGNING_KEY = '2a9251d782700769fb856da5725daf38661874ca6f80ae7dc2b05ec1a81a24ae'
+
+    def _extract_formats_and_subtitles(self, data, video_id):
+        if traverse_obj(data, 'drm'):
+            self.report_drm(video_id)
+
+        formats, subtitles = [], {}
+        for target in traverse_obj(data, ('targetUrls', lambda _, v: url_or_none(v['url']) and v['type'])):
+            format_type = target['type'].upper()
+            format_url = target['url']
+            if format_type in ('HLS', 'HLS_AES'):
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    format_url, video_id, 'mp4', m3u8_id=format_type, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            elif format_type == 'HDS':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_type, fatal=False))
+            elif format_type == 'MPEG_DASH':
+                fmts, subs = self._extract_mpd_formats_and_subtitles(
+                    format_url, video_id, mpd_id=format_type, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            elif format_type == 'HSS':
+                fmts, subs = self._extract_ism_formats_and_subtitles(
+                    format_url, video_id, ism_id='mss', fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            else:
+                formats.append({
+                    'format_id': format_type,
+                    'url': format_url,
+                })
+
+        for sub in traverse_obj(data, ('subtitleUrls', lambda _, v: v['url'] and v['type'] == 'CLOSED')):
+            subtitles.setdefault('nl', []).append({'url': sub['url']})
+
+        return formats, subtitles
+
+    def _call_api(self, video_id, client='null', id_token=None, version='v2'):
+        player_info = {'exp': (round(time.time(), 3) + 900), **self._PLAYER_INFO}
+        player_token = self._download_json(
+            'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2/tokens',
+            video_id, 'Downloading player token', headers={
+                **self.geo_verification_headers(),
+                'Content-Type': 'application/json',
+            }, data=json.dumps({
+                'identityToken': id_token or {},
+                'playerInfo': jwt_encode_hs256(player_info, self._JWT_SIGNING_KEY, headers={
+                    'kid': self._JWT_KEY_ID
+                }).decode()
+            }, separators=(',', ':')).encode())['vrtPlayerToken']
+
+        return self._download_json(
+            f'https://media-services-public.vrt.be/media-aggregator/{version}/media-items/{video_id}',
+            video_id, 'Downloading API JSON', query={
+                'vrtPlayerToken': player_token,
+                'client': client,
+            }, expected_status=400)
+
+
+class VRTIE(VRTBaseIE):
     IE_DESC = 'VRT NWS, Flanders News, Flandern Info and Sporza'
     _VALID_URL = r'https?://(?:www\.)?(?P<site>vrt\.be/vrtnws|sporza\.be)/[a-z]{2}/\d{4}/\d{2}/\d{2}/(?P<id>[^/?&#]+)'
     _TESTS = [{
         'url': 'https://www.vrt.be/vrtnws/nl/2019/05/15/beelden-van-binnenkant-notre-dame-een-maand-na-de-brand/',
-        'md5': 'e1663accf5cf13f375f3cd0d10476669',
         'info_dict': {
             'id': 'pbs-pub-7855fc7b-1448-49bc-b073-316cb60caa71$vid-2ca50305-c38a-4762-9890-65cbd098b7bd',
             'ext': 'mp4',
             'title': 'Beelden van binnenkant Notre-Dame, één maand na de brand',
-            'description': 'Op maandagavond 15 april ging een deel van het dakgebinte van de Parijse kathedraal in vlammen op.',
-            'timestamp': 1557924660,
-            'upload_date': '20190515',
+            'description': 'md5:6fd85f999b2d1841aa5568f4bf02c3ff',
             'duration': 31.2,
+            'thumbnail': 'https://images.vrt.be/orig/2019/05/15/2d914d61-7710-11e9-abcc-02b7b76bf47f.jpg',
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://sporza.be/nl/2019/05/15/de-belgian-cats-zijn-klaar-voor-het-ek/',
-        'md5': '910bba927566e9ab992278f647eb4b75',
         'info_dict': {
             'id': 'pbs-pub-f2c86a46-8138-413a-a4b9-a0015a16ce2c$vid-1f112b31-e58e-4379-908d-aca6d80f8818',
             'ext': 'mp4',
-            'title': 'De Belgian Cats zijn klaar voor het EK mét Ann Wauters',
-            'timestamp': 1557923760,
-            'upload_date': '20190515',
+            'title': 'De Belgian Cats zijn klaar voor het EK',
+            'description': 'Video: De Belgian Cats zijn klaar voor het EK mét Ann Wauters | basketbal, sport in het journaal',
             'duration': 115.17,
+            'thumbnail': 'https://images.vrt.be/orig/2019/05/15/11c0dba3-770e-11e9-abcc-02b7b76bf47f.jpg',
         },
-    }, {
-        'url': 'https://www.vrt.be/vrtnws/en/2019/05/15/belgium_s-eurovision-entry-falls-at-the-first-hurdle/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.vrt.be/vrtnws/de/2019/05/15/aus-fuer-eliott-im-halbfinale-des-eurosongfestivals/',
-        'only_matching': True,
+        'params': {'skip_download': 'm3u8'},
     }]
     _CLIENT_MAP = {
         'vrt.be/vrtnws': 'vrtnieuws',
@@ -49,34 +141,285 @@ class VRTIE(InfoExtractor):
     def _real_extract(self, url):
         site, display_id = self._match_valid_url(url).groups()
         webpage = self._download_webpage(url, display_id)
-        attrs = extract_attributes(self._search_regex(
-            r'(<[^>]+class="vrtvideo( [^"]*)?"[^>]*>)', webpage, 'vrt video'))
+        attrs = extract_attributes(get_element_html_by_class('vrtvideo', webpage) or '')
 
-        asset_id = attrs['data-video-id']
-        publication_id = attrs.get('data-publication-id')
+        asset_id = attrs.get('data-video-id') or attrs['data-videoid']
+        publication_id = traverse_obj(attrs, 'data-publication-id', 'data-publicationid')
         if publication_id:
-            asset_id = publication_id + '$' + asset_id
-        client = attrs.get('data-client-code') or self._CLIENT_MAP[site]
+            asset_id = f'{publication_id}${asset_id}'
+        client = traverse_obj(attrs, 'data-client-code', 'data-client') or self._CLIENT_MAP[site]
+
+        data = self._call_api(asset_id, client)
+        formats, subtitles = self._extract_formats_and_subtitles(data, asset_id)
 
-        title = strip_or_none(get_element_by_class(
-            'vrt-title', webpage) or self._html_search_meta(
-            ['og:title', 'twitter:title', 'name'], webpage))
         description = self._html_search_meta(
             ['og:description', 'twitter:description', 'description'], webpage)
         if description == '…':
             description = None
-        timestamp = unified_timestamp(self._html_search_meta(
-            'article:published_time', webpage))
 
         return {
-            '_type': 'url_transparent',
             'id': asset_id,
-            'display_id': display_id,
-            'title': title,
+            'formats': formats,
+            'subtitles': subtitles,
             'description': description,
-            'thumbnail': attrs.get('data-posterimage'),
-            'timestamp': timestamp,
+            'thumbnail': url_or_none(attrs.get('data-posterimage')),
             'duration': float_or_none(attrs.get('data-duration'), 1000),
-            'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (client, asset_id),
-            'ie_key': 'Canvas',
+            '_old_archive_ids': [make_archive_id('Canvas', asset_id)],
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('shortDescription', {str}),
+                'duration': ('duration', {functools.partial(float_or_none, scale=1000)}),
+                'thumbnail': ('posterImageUrl', {url_or_none}),
+            }),
+        }
+
+
+class VrtNUIE(VRTBaseIE):
+    IE_DESC = 'VRT MAX'
+    _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        # CONTENT_IS_AGE_RESTRICTED
+        'url': 'https://www.vrt.be/vrtnu/a-z/de-ideale-wereld/2023-vj/de-ideale-wereld-d20230116/',
+        'info_dict': {
+            'id': 'pbs-pub-855b00a8-6ce2-4032-ac4f-1fcf3ae78524$vid-d2243aa1-ec46-4e34-a55b-92568459906f',
+            'ext': 'mp4',
+            'title': 'Tom Waes',
+            'description': 'Satirisch actualiteitenmagazine met Ella Leyers. Tom Waes is te gast.',
+            'timestamp': 1673905125,
+            'release_timestamp': 1673905125,
+            'series': 'De ideale wereld',
+            'season_id': '1672830988794',
+            'episode': 'Aflevering 1',
+            'episode_number': 1,
+            'episode_id': '1672830988861',
+            'display_id': 'de-ideale-wereld-d20230116',
+            'channel': 'VRT',
+            'duration': 1939.0,
+            'thumbnail': 'https://images.vrt.be/orig/2023/01/10/1bb39cb3-9115-11ed-b07d-02b7b76bf47f.jpg',
+            'release_date': '20230116',
+            'upload_date': '20230116',
+            'age_limit': 12,
+        },
+    }, {
+        'url': 'https://www.vrt.be/vrtnu/a-z/buurman--wat-doet-u-nu-/6/buurman--wat-doet-u-nu--s6-trailer/',
+        'info_dict': {
+            'id': 'pbs-pub-ad4050eb-d9e5-48c2-9ec8-b6c355032361$vid-0465537a-34a8-4617-8352-4d8d983b4eee',
+            'ext': 'mp4',
+            'title': 'Trailer seizoen 6 \'Buurman, wat doet u nu?\'',
+            'description': 'md5:197424726c61384b4e5c519f16c0cf02',
+            'timestamp': 1652940000,
+            'release_timestamp': 1652940000,
+            'series': 'Buurman, wat doet u nu?',
+            'season': 'Seizoen 6',
+            'season_number': 6,
+            'season_id': '1652344200907',
+            'episode': 'Aflevering 0',
+            'episode_number': 0,
+            'episode_id': '1652951873524',
+            'display_id': 'buurman--wat-doet-u-nu--s6-trailer',
+            'channel': 'VRT',
+            'duration': 33.13,
+            'thumbnail': 'https://images.vrt.be/orig/2022/05/23/3c234d21-da83-11ec-b07d-02b7b76bf47f.jpg',
+            'release_date': '20220519',
+            'upload_date': '20220519',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+    _NETRC_MACHINE = 'vrtnu'
+    _authenticated = False
+
+    def _perform_login(self, username, password):
+        auth_info = self._gigya_login({
+            'APIKey': '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy',
+            'targetEnv': 'jssdk',
+            'loginID': username,
+            'password': password,
+            'authMode': 'cookie',
+        })
+
+        if auth_info.get('errorDetails'):
+            raise ExtractorError(f'Unable to login. VrtNU said: {auth_info["errorDetails"]}', expected=True)
+
+        # Sometimes authentication fails for no good reason, retry
+        for retry in self.RetryManager():
+            if retry.attempt > 1:
+                self._sleep(1, None)
+            try:
+                self._request_webpage(
+                    'https://token.vrt.be/vrtnuinitlogin', None, note='Requesting XSRF Token',
+                    errnote='Could not get XSRF Token', query={
+                        'provider': 'site',
+                        'destination': 'https://www.vrt.be/vrtnu/',
+                    })
+                self._request_webpage(
+                    'https://login.vrt.be/perform_login', None,
+                    note='Performing login', errnote='Login failed',
+                    query={'client_id': 'vrtnu-site'}, data=urlencode_postdata({
+                        'UID': auth_info['UID'],
+                        'UIDSignature': auth_info['UIDSignature'],
+                        'signatureTimestamp': auth_info['signatureTimestamp'],
+                        '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
+                    }))
+            except ExtractorError as e:
+                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                    retry.error = e
+                    continue
+                raise
+
+        self._authenticated = True
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        parsed_url = urllib.parse.urlparse(url)
+        details = self._download_json(
+            f'{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rstrip("/")}.model.json',
+            display_id, 'Downloading asset JSON', 'Unable to download asset JSON')['details']
+
+        watch_info = traverse_obj(details, (
+            'actions', lambda _, v: v['type'] == 'watch-episode', {dict}), get_all=False) or {}
+        video_id = join_nonempty(
+            'episodePublicationId', 'episodeVideoId', delim='$', from_dict=watch_info)
+        if '$' not in video_id:
+            raise ExtractorError('Unable to extract video ID')
+
+        vrtnutoken = self._download_json(
+            'https://token.vrt.be/refreshtoken', video_id, note='Retrieving vrtnutoken',
+            errnote='Token refresh failed')['vrtnutoken'] if self._authenticated else None
+
+        video_info = self._call_api(video_id, 'vrtnu-web@PROD', vrtnutoken)
+
+        if 'title' not in video_info:
+            code = video_info.get('code')
+            if code in ('AUTHENTICATION_REQUIRED', 'CONTENT_IS_AGE_RESTRICTED'):
+                self.raise_login_required(code, method='password')
+            elif code in ('INVALID_LOCATION', 'CONTENT_AVAILABLE_ONLY_IN_BE'):
+                self.raise_geo_restricted(countries=['BE'])
+            elif code == 'CONTENT_AVAILABLE_ONLY_FOR_BE_RESIDENTS_AND_EXPATS':
+                if not self._authenticated:
+                    self.raise_login_required(code, method='password')
+                self.raise_geo_restricted(countries=['BE'])
+            raise ExtractorError(code, expected=True)
+
+        formats, subtitles = self._extract_formats_and_subtitles(video_info, video_id)
+
+        return {
+            **traverse_obj(details, {
+                'title': 'title',
+                'description': ('description', {clean_html}),
+                'timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}),
+                'release_timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}),
+                'series': ('data', 'program', 'title'),
+                'season': ('data', 'season', 'title', 'value'),
+                'season_number': ('data', 'season', 'title', 'raw', {int_or_none}),
+                'season_id': ('data', 'season', 'id', {str_or_none}),
+                'episode': ('data', 'episode', 'number', 'value', {str_or_none}),
+                'episode_number': ('data', 'episode', 'number', 'raw', {int_or_none}),
+                'episode_id': ('data', 'episode', 'id', {str_or_none}),
+                'age_limit': ('data', 'episode', 'age', 'raw', {parse_age_limit}),
+            }),
+            'id': video_id,
+            'display_id': display_id,
+            'channel': 'VRT',
+            'formats': formats,
+            'duration': float_or_none(video_info.get('duration'), 1000),
+            'thumbnail': url_or_none(video_info.get('posterImageUrl')),
+            'subtitles': subtitles,
+            '_old_archive_ids': [make_archive_id('Canvas', video_id)],
+        }
+
+
+class KetnetIE(VRTBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?P<id>(?:[^/]+/)*[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.ketnet.be/kijken/m/meisjes/6/meisjes-s6a5',
+        'info_dict': {
+            'id': 'pbs-pub-39f8351c-a0a0-43e6-8394-205d597d6162$vid-5e306921-a9aa-4fa9-9f39-5b82c8f1028e',
+            'ext': 'mp4',
+            'title': 'Meisjes',
+            'episode': 'Reeks 6: Week 5',
+            'season': 'Reeks 6',
+            'series': 'Meisjes',
+            'timestamp': 1685251800,
+            'upload_date': '20230528',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        video = self._download_json(
+            'https://senior-bff.ketnet.be/graphql', display_id, query={
+                'query': '''{
+  video(id: "content/ketnet/nl/%s.model.json") {
+    description
+    episodeNr
+    imageUrl
+    mediaReference
+    programTitle
+    publicationDate
+    seasonTitle
+    subtitleVideodetail
+    titleVideodetail
+  }
+}''' % display_id,
+            })['data']['video']
+
+        video_id = urllib.parse.unquote(video['mediaReference'])
+        data = self._call_api(video_id, 'ketnet@PROD', version='v1')
+        formats, subtitles = self._extract_formats_and_subtitles(data, video_id)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            '_old_archive_ids': [make_archive_id('Canvas', video_id)],
+            **traverse_obj(video, {
+                'title': ('titleVideodetail', {str}),
+                'description': ('description', {str}),
+                'thumbnail': ('thumbnail', {url_or_none}),
+                'timestamp': ('publicationDate', {parse_iso8601}),
+                'series': ('programTitle', {str}),
+                'season': ('seasonTitle', {str}),
+                'episode': ('subtitleVideodetail', {str}),
+                'episode_number': ('episodeNr', {int_or_none}),
+            }),
+        }
+
+
+class DagelijkseKostIE(VRTBaseIE):
+    IE_DESC = 'dagelijksekost.een.be'
+    _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
+        'info_dict': {
+            'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
+            'ext': 'mp4',
+            'title': 'Hachis parmentier met witloof',
+            'description': 'md5:9960478392d87f63567b5b117688cdc5',
+            'display_id': 'hachis-parmentier-met-witloof',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._html_search_regex(
+            r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id', group='id')
+
+        data = self._call_api(video_id, 'dako@prod', version='v1')
+        formats, subtitles = self._extract_formats_and_subtitles(data, video_id)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'display_id': display_id,
+            'title': strip_or_none(get_element_by_class(
+                'dish-metadata__title', webpage) or self._html_search_meta('twitter:title', webpage)),
+            'description': clean_html(get_element_by_class(
+                'dish-description', webpage)) or self._html_search_meta(
+                ['description', 'twitter:description', 'og:description'], webpage),
+            '_old_archive_ids': [make_archive_id('Canvas', video_id)],
         }

From 2fb35f6004c7625f0dd493da4a5abf0690f7777c Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 3 Jun 2023 18:33:51 +1200
Subject: [PATCH 328/871] [extractor/youtube] Support shorter relative time
 format (#7191)

See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/1067

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index ccf97705a..6e7485c03 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -893,9 +893,16 @@ def _extract_thumbnails(data, *path_list):
     def extract_relative_time(relative_time_text):
         """
         Extracts a relative time from string and converts to dt object
-        e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
+        e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today', '8 yr ago'
         """
-        mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
+
+        # XXX: this could be moved to a general function in utils.py
+        # The relative time text strings are roughly the same as what
+        # Javascript's Intl.RelativeTimeFormat function generates.
+        # See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/RelativeTimeFormat
+        mobj = re.search(
+            r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>sec(?:ond)?|s|min(?:ute)?|h(?:our|r)?|d(?:ay)?|w(?:eek|k)?|mo(?:nth)?|y(?:ear|r)?)s?\s*ago',
+            relative_time_text)
         if mobj:
             start = mobj.group('start')
             if start:

From c91ac833ea99b00506e470a44cf930e4e23378c9 Mon Sep 17 00:00:00 2001
From: Paul Wise <pabs3@bonedaddy.net>
Date: Sun, 4 Jun 2023 16:04:47 +0800
Subject: [PATCH 329/871] [extractor/acast] Support embeds (#7212)

Authored by: pabs3
---
 yt_dlp/extractor/acast.py | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/acast.py b/yt_dlp/extractor/acast.py
index f2f828f8e..427d04c31 100644
--- a/yt_dlp/extractor/acast.py
+++ b/yt_dlp/extractor/acast.py
@@ -40,28 +40,33 @@ def _call_api(self, path, video_id, query=None):
 
 class ACastIE(ACastBaseIE):
     IE_NAME = 'acast'
-    _VALID_URL = r'''(?x)
+    _VALID_URL = r'''(?x:
                     https?://
                         (?:
                             (?:(?:embed|www)\.)?acast\.com/|
                             play\.acast\.com/s/
                         )
-                        (?P<channel>[^/]+)/(?P<id>[^/#?]+)
-                    '''
+                        (?P<channel>[^/]+)/(?P<id>[^/#?"]+)
+                    )'''
+    _EMBED_REGEX = [rf'(?x)<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL})']
     _TESTS = [{
         'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
-        'md5': 'f5598f3ad1e4776fed12ec1407153e4b',
         'info_dict': {
             'id': '2a92b283-1a75-4ad8-8396-499c641de0d9',
             'ext': 'mp3',
             'title': '2. Raggarmordet - Röster ur det förflutna',
-            'description': 'md5:a992ae67f4d98f1c0141598f7bebbf67',
+            'description': 'md5:013959207e05011ad14a222cf22278cc',
             'timestamp': 1477346700,
             'upload_date': '20161024',
             'duration': 2766,
-            'creator': 'Anton Berg & Martin Johnson',
+            'creator': 'Third Ear Studio',
             'series': 'Spår',
             'episode': '2. Raggarmordet - Röster ur det förflutna',
+            'thumbnail': 'https://assets.pippa.io/shows/616ebe1886d7b1398620b943/616ebe33c7e6e70013cae7da.jpg',
+            'episode_number': 2,
+            'display_id': '2.raggarmordet-rosterurdetforflutna',
+            'season_number': 4,
+            'season': 'Season 4',
         }
     }, {
         'url': 'http://embed.acast.com/adambuxton/ep.12-adam-joeschristmaspodcast2015',
@@ -73,6 +78,23 @@ class ACastIE(ACastBaseIE):
         'url': 'https://play.acast.com/s/sparpodcast/2a92b283-1a75-4ad8-8396-499c641de0d9',
         'only_matching': True,
     }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://ausi.anu.edu.au/news/democracy-sausage-episode-can-labor-be-long-form-government',
+        'info_dict': {
+            'id': '646c68fb21fbf20011e9c651',
+            'ext': 'mp3',
+            'creator': 'The Australian National University',
+            'display_id': 'can-labor-be-a-long-form-government',
+            'duration': 2618,
+            'thumbnail': 'https://assets.pippa.io/shows/6113e8578b4903809f16f7e5/1684821529295-515b9520db9ce53275b995eb302f941c.jpeg',
+            'title': 'Can Labor be a long-form government?',
+            'episode': 'Can Labor be a long-form government?',
+            'upload_date': '20230523',
+            'series': 'Democracy Sausage with Mark Kenny',
+            'timestamp': 1684826362,
+            'description': 'md5:feabe1fc5004c78ee59c84a46bf4ba16',
+        }
+    }]
 
     def _real_extract(self, url):
         channel, display_id = self._match_valid_url(url).groups()

From 12037d8b0a578fcc78a5c8f98964e48ee6060e25 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 4 Jun 2023 06:10:30 -0500
Subject: [PATCH 330/871] [extractor/substack] Fix extraction (#7218)

Closes #7155
Authored by: bashonly
---
 yt_dlp/extractor/substack.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/substack.py b/yt_dlp/extractor/substack.py
index fa3826388..3782ceed1 100644
--- a/yt_dlp/extractor/substack.py
+++ b/yt_dlp/extractor/substack.py
@@ -2,7 +2,7 @@
 import urllib.parse
 
 from .common import InfoExtractor
-from ..utils import str_or_none, traverse_obj
+from ..utils import js_to_json, str_or_none, traverse_obj
 
 
 class SubstackIE(InfoExtractor):
@@ -14,7 +14,7 @@ class SubstackIE(InfoExtractor):
             'id': '47660949',
             'ext': 'mp4',
             'title': 'I MADE A VLOG',
-            'description': 'md5:10c01ff93439a62e70ce963b2aa0b7f6',
+            'description': 'md5:9248af9a759321e1027226f988f54d96',
             'thumbnail': 'md5:bec758a34d8ee9142d43bcebdf33af18',
             'uploader': 'Maybe Baby',
             'uploader_id': '33628',
@@ -77,7 +77,9 @@ def _real_extract(self, url):
         display_id, username = self._match_valid_url(url).group('id', 'username')
         webpage = self._download_webpage(url, display_id)
 
-        webpage_info = self._search_json(r'<script[^>]*>\s*window\._preloads\s*=', webpage, 'preloads', display_id)
+        webpage_info = self._parse_json(self._search_json(
+            r'window\._preloads\s*=\s*JSON\.parse\(', webpage, 'json string',
+            display_id, transform_source=js_to_json, contains_pattern=r'"{(?s:.+)}"'), display_id)
 
         post_type = webpage_info['post']['type']
         formats, subtitles = [], {}

From 971d901d129403e875a04dd92109507a03fbc070 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 4 Jun 2023 07:03:44 -0500
Subject: [PATCH 331/871] [extractor/tencent] Fix fatal metadata extraction
 (#7219)

Closes #7177
Authored by: bashonly
---
 yt_dlp/extractor/tencent.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/extractor/tencent.py b/yt_dlp/extractor/tencent.py
index 42a2175b0..6618ea4e6 100644
--- a/yt_dlp/extractor/tencent.py
+++ b/yt_dlp/extractor/tencent.py
@@ -163,11 +163,9 @@ class VQQBaseIE(TencentBaseIE):
     _REFERER = 'v.qq.com'
 
     def _get_webpage_metadata(self, webpage, video_id):
-        return self._parse_json(
-            self._search_regex(
-                r'(?s)<script[^>]*>[^<]*window\.__pinia\s*=\s*([^<]+)</script>',
-                webpage, 'pinia data', fatal=False),
-            video_id, transform_source=js_to_json, fatal=False)
+        return self._search_json(
+            r'<script[^>]*>[^<]*window\.__(?:pinia|PINIA__)\s*=',
+            webpage, 'pinia data', video_id, transform_source=js_to_json, fatal=False)
 
 
 class VQQVideoIE(VQQBaseIE):
@@ -176,7 +174,7 @@ class VQQVideoIE(VQQBaseIE):
 
     _TESTS = [{
         'url': 'https://v.qq.com/x/page/q326831cny0.html',
-        'md5': '84568b3722e15e9cd023b5594558c4a7',
+        'md5': 'b11c9cb781df710d686b950376676e2a',
         'info_dict': {
             'id': 'q326831cny0',
             'ext': 'mp4',
@@ -187,7 +185,7 @@ class VQQVideoIE(VQQBaseIE):
         },
     }, {
         'url': 'https://v.qq.com/x/page/o3013za7cse.html',
-        'md5': 'cc431c4f9114a55643893c2c8ebf5592',
+        'md5': 'a1bcf42c6d28c189bd2fe2d468abb287',
         'info_dict': {
             'id': 'o3013za7cse',
             'ext': 'mp4',
@@ -208,6 +206,7 @@ class VQQVideoIE(VQQBaseIE):
             'series': '鸡毛飞上天',
             'format_id': r're:^shd',
         },
+        'skip': '404',
     }, {
         'url': 'https://v.qq.com/x/cover/mzc00200p29k31e/s0043cwsgj0.html',
         'md5': 'fadd10bf88aec3420f06f19ee1d24c5b',
@@ -220,6 +219,7 @@ class VQQVideoIE(VQQBaseIE):
             'series': '青年理工工作者生活研究所',
             'format_id': r're:^shd',
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         # Geo-restricted to China
         'url': 'https://v.qq.com/x/cover/mcv8hkc8zk8lnov/x0036x5qqsr.html',

From 5ee9a7d6e18ceea956e831994cf11c423979354f Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 4 Jun 2023 07:15:09 -0500
Subject: [PATCH 332/871] [extractor/sverigesradio] Support slug URLs (#7220)

Closes #7145
Authored by: bashonly
---
 yt_dlp/extractor/sverigesradio.py | 62 +++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/sverigesradio.py b/yt_dlp/extractor/sverigesradio.py
index 65da615d0..01a07b399 100644
--- a/yt_dlp/extractor/sverigesradio.py
+++ b/yt_dlp/extractor/sverigesradio.py
@@ -1,8 +1,13 @@
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
+    extract_attributes,
+    get_element_by_id,
+    get_element_html_by_class,
     int_or_none,
     str_or_none,
+    traverse_obj,
+    url_or_none,
 )
 
 
@@ -21,7 +26,15 @@ class SverigesRadioBaseIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        audio_id = self._match_id(url)
+        audio_id, display_id = self._match_valid_url(url).group('id', 'slug')
+        if not audio_id:
+            webpage = self._download_webpage(url, display_id)
+            audio_id = (
+                traverse_obj(
+                    get_element_html_by_class('audio-button', webpage),
+                    ({extract_attributes}, ('data-audio-id', 'data-publication-id')), get_all=False)
+                or self._parse_json(get_element_by_id('gtm-metadata', webpage), display_id)['pageId'])
+
         query = {
             'id': audio_id,
             'type': self._AUDIO_TYPE,
@@ -30,7 +43,6 @@ def _real_extract(self, url):
         item = self._download_json(
             self._BASE_URL + 'audiometadata', audio_id,
             'Downloading audio JSON metadata', query=query)['items'][0]
-        title = item['subtitle']
 
         query['format'] = 'iis'
         urls = []
@@ -61,18 +73,20 @@ def _real_extract(self, url):
 
         return {
             'id': audio_id,
-            'title': title,
             'formats': formats,
-            'series': item.get('title'),
-            'duration': int_or_none(item.get('duration')),
-            'thumbnail': item.get('displayimageurl'),
-            'description': item.get('description'),
+            **traverse_obj(item, {
+                'title': 'subtitle',
+                'series': 'title',
+                'duration': ('duration', {int_or_none}),
+                'thumbnail': ('displayimageurl', {url_or_none}),
+                'description': 'description',
+            }),
         }
 
 
 class SverigesRadioPublicationIE(SverigesRadioBaseIE):
     IE_NAME = 'sverigesradio:publication'
-    _VALID_URL = r'https?://(?:www\.)?sverigesradio\.se/sida/(?:artikel|gruppsida)\.aspx\?.*?\bartikel=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?sverigesradio\.se/(?:sida/)?(?:artikel|gruppsida)(?:\.aspx\?.*?\bartikel=(?P<id>[0-9]+)|/(?P<slug>[\w-]+))'
     _TESTS = [{
         'url': 'https://sverigesradio.se/sida/artikel.aspx?programid=83&artikel=7038546',
         'md5': '6a4917e1923fccb080e5a206a5afa542',
@@ -85,6 +99,18 @@ class SverigesRadioPublicationIE(SverigesRadioBaseIE):
             'description': 'md5:daf7ce66a8f0a53d5465a5984d3839df',
             'thumbnail': r're:^https?://.*\.jpg',
         },
+    }, {
+        'url': 'https://sverigesradio.se/artikel/tysk-fotbollsfeber-bayern-munchens-10-ariga-segersvit-kan-brytas',
+        'md5': 'f8a914ad50f491bb74eed403ab4bfef6',
+        'info_dict': {
+            'id': '8360345',
+            'ext': 'm4a',
+            'title': 'Tysk fotbollsfeber när Bayern Münchens 10-åriga segersvit kan brytas',
+            'series': 'Radiosporten',
+            'description': 'md5:5254610e20ce527ecb3a6102a06dcc5f',
+            'duration': 72,
+            'thumbnail': r're:^https?://.*\.jpg',
+        },
     }, {
         'url': 'https://sverigesradio.se/sida/gruppsida.aspx?programid=3304&grupp=6247&artikel=7146887',
         'only_matching': True,
@@ -94,8 +120,8 @@ class SverigesRadioPublicationIE(SverigesRadioBaseIE):
 
 class SverigesRadioEpisodeIE(SverigesRadioBaseIE):
     IE_NAME = 'sverigesradio:episode'
-    _VALID_URL = r'https?://(?:www\.)?sverigesradio\.se/(?:sida/)?avsnitt/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?sverigesradio\.se/(?:sida/)?avsnitt/(?:(?P<id>\d+)|(?P<slug>[\w-]+))(?:$|[#?])'
+    _TESTS = [{
         'url': 'https://sverigesradio.se/avsnitt/1140922?programid=1300',
         'md5': '20dc4d8db24228f846be390b0c59a07c',
         'info_dict': {
@@ -106,6 +132,18 @@ class SverigesRadioEpisodeIE(SverigesRadioBaseIE):
             'title': 'Metoo och valen',
             'description': 'md5:fcb5c1f667f00badcc702b196f10a27e',
             'thumbnail': r're:^https?://.*\.jpg',
-        }
-    }
+        },
+    }, {
+        'url': 'https://sverigesradio.se/avsnitt/p4-live-med-first-aid-kit-scandinavium-mars-2023',
+        'md5': 'ce17fb82520a8033dbb846993d5589fe',
+        'info_dict': {
+            'id': '2160416',
+            'ext': 'm4a',
+            'title': 'P4 Live med First Aid Kit',
+            'description': 'md5:6d5b78eed3d2b65f6de04daa45e9285d',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'series': 'P4 Live',
+            'duration': 5640,
+        },
+    }]
     _AUDIO_TYPE = 'episode'

From 97d60ad8cd6c99f01e463a9acfce8693aff2a609 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 4 Jun 2023 08:37:59 -0500
Subject: [PATCH 333/871] [extractor/foxnews] Fix extractors (#7222)

Closes #6050
Authored by: bashonly
---
 yt_dlp/extractor/amp.py     |  9 +++--
 yt_dlp/extractor/foxnews.py | 77 +++++++++++++++++++++++++++----------
 2 files changed, 62 insertions(+), 24 deletions(-)

diff --git a/yt_dlp/extractor/amp.py b/yt_dlp/extractor/amp.py
index b0cbd775c..0d259c549 100644
--- a/yt_dlp/extractor/amp.py
+++ b/yt_dlp/extractor/amp.py
@@ -5,6 +5,7 @@
     int_or_none,
     mimetype2ext,
     parse_iso8601,
+    strip_jsonp,
     unified_timestamp,
     url_or_none,
 )
@@ -15,7 +16,7 @@ class AMPIE(InfoExtractor):  # XXX: Conventionally, base classes should end with
     def _extract_feed_info(self, url):
         feed = self._download_json(
             url, None, 'Downloading Akamai AMP feed',
-            'Unable to download Akamai AMP feed')
+            'Unable to download Akamai AMP feed', transform_source=strip_jsonp)
         item = feed.get('channel', {}).get('item')
         if not item:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, feed['error']))
@@ -73,8 +74,10 @@ def get_media_node(name, default=None):
                     media_url + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',
                     video_id, f4m_id='hds', fatal=False))
             elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    media_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
             else:
                 formats.append({
                     'format_id': media_data.get('media-category', {}).get('@attributes', {}).get('label'),
diff --git a/yt_dlp/extractor/foxnews.py b/yt_dlp/extractor/foxnews.py
index 52172aace..6aa63614e 100644
--- a/yt_dlp/extractor/foxnews.py
+++ b/yt_dlp/extractor/foxnews.py
@@ -7,8 +7,37 @@
 class FoxNewsIE(AMPIE):
     IE_NAME = 'foxnews'
     IE_DESC = 'Fox News and Fox Business Video'
-    _VALID_URL = r'https?://(?P<host>video\.(?:insider\.)?fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
+    _VALID_URL = r'https?://video\.(?:insider\.)?fox(?:news|business)\.com/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
+        {
+            'url': 'https://video.foxnews.com/v/6320653836112',
+            'info_dict': {
+                'id': '6320653836112',
+                'ext': 'mp4',
+                'title': 'Tucker Carlson joins \'Gutfeld!\' to discuss his new documentary',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'duration': 404,
+                'upload_date': '20230217',
+                'description': 'md5:858a8a36f59e9ca897d758855bcdfa02',
+                'timestamp': 1676611344.0,
+            },
+            'params': {'skip_download': 'm3u8'},
+        },
+        {
+            # From http://insider.foxnews.com/2016/08/25/univ-wisconsin-student-group-pushing-silence-certain-words
+            'url': 'http://video.insider.foxnews.com/v/video-embed.html?video_id=5099377331001&autoplay=true&share_url=http://insider.foxnews.com/2016/08/25/univ-wisconsin-student-group-pushing-silence-certain-words&share_title=Student%20Group:%20Saying%20%27Politically%20Correct,%27%20%27Trash%27%20and%20%27Lame%27%20Is%20Offensive&share=true',
+            'info_dict': {
+                'id': '5099377331001',
+                'ext': 'mp4',
+                'title': '82416_censoring',
+                'description': '82416_censoring',
+                'upload_date': '20160826',
+                'timestamp': 1472169708.0,
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'duration': 521,
+            },
+            'params': {'skip_download': 'm3u8'},
+        },
         {
             'url': 'http://video.foxnews.com/v/3937480/frozen-in-time/#sp=show-clips',
             'md5': '32aaded6ba3ef0d1c04e238d01031e5e',
@@ -22,6 +51,7 @@ class FoxNewsIE(AMPIE):
                 'upload_date': '20110503',
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
+            'skip': '404 page',
         },
         {
             'url': 'http://video.foxnews.com/v/3922535568001/rep-luis-gutierrez-on-if-obamas-immigration-plan-is-legal/#sp=show-clips',
@@ -36,10 +66,7 @@ class FoxNewsIE(AMPIE):
                 'upload_date': '20141204',
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
+            'skip': 'm3u8 HTTP error 400 in web browser',
         },
         {
             'url': 'http://video.foxnews.com/v/video-embed.html?video_id=3937480&d=video.foxnews.com',
@@ -49,11 +76,6 @@ class FoxNewsIE(AMPIE):
             'url': 'http://video.foxbusiness.com/v/4442309889001',
             'only_matching': True,
         },
-        {
-            # From http://insider.foxnews.com/2016/08/25/univ-wisconsin-student-group-pushing-silence-certain-words
-            'url': 'http://video.insider.foxnews.com/v/video-embed.html?video_id=5099377331001&autoplay=true&share_url=http://insider.foxnews.com/2016/08/25/univ-wisconsin-student-group-pushing-silence-certain-words&share_title=Student%20Group:%20Saying%20%27Politically%20Correct,%27%20%27Trash%27%20and%20%27Lame%27%20Is%20Offensive&share=true',
-            'only_matching': True,
-        },
     ]
 
     @classmethod
@@ -67,10 +89,10 @@ def _extract_embed_urls(cls, url, webpage):
             yield f'https://video.foxnews.com/v/video-embed.html?video_id={mobj.group("video_id")}'
 
     def _real_extract(self, url):
-        host, video_id = self._match_valid_url(url).groups()
+        video_id = self._match_id(url)
 
         info = self._extract_feed_info(
-            'http://%s/v/feed/video/%s.js?template=fox' % (host, video_id))
+            f'https://api.foxnews.com/v3/video-player/{video_id}?callback=uid_{video_id}')
         info['id'] = video_id
         return info
 
@@ -78,6 +100,19 @@ def _real_extract(self, url):
 class FoxNewsVideoIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?foxnews\.com/video/(?P<id>\d+)'
     _TESTS = [{
+        'url': 'https://www.foxnews.com/video/6328632286112',
+        'info_dict': {
+            'id': '6328632286112',
+            'ext': 'mp4',
+            'title': 'Review: 2023 Toyota Prius Prime',
+            'duration': 155,
+            'thumbnail': r're:^https://.+\.jpg$',
+            'timestamp': 1685720177.0,
+            'upload_date': '20230602',
+            'description': 'md5:b69aafb125b41c1402e9744f53d6edc4',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
         'url': 'https://www.foxnews.com/video/6313058664112',
         'info_dict': {
             'id': '6313058664112',
@@ -89,8 +124,7 @@ class FoxNewsVideoIE(InfoExtractor):
             'title': 'Gutfeld! - Thursday, September 29',
             'timestamp': 1664527538,
         },
-        'expected_warnings': ['Ignoring subtitle tracks'],
-        'params': {'skip_download': 'm3u8'},
+        'skip': '404 page',
     }]
 
     def _real_extract(self, url):
@@ -104,19 +138,22 @@ class FoxNewsArticleIE(InfoExtractor):
 
     _TESTS = [{
         # data-video-id
-        'url': 'http://www.foxnews.com/politics/2016/09/08/buzz-about-bud-clinton-camp-denies-claims-wore-earpiece-at-forum.html',
-        'md5': '83d44e1aff1433e7a29a7b537d1700b5',
+        'url': 'https://www.foxnews.com/politics/2016/09/08/buzz-about-bud-clinton-camp-denies-claims-wore-earpiece-at-forum.html',
+        'md5': 'd2dd6ce809cedeefa96460e964821437',
         'info_dict': {
             'id': '5116295019001',
             'ext': 'mp4',
             'title': 'Trump and Clinton asked to defend positions on Iraq War',
-            'description': 'Veterans react on \'The Kelly File\'',
+            'description': 'Veterans and Fox News host Dana Perino react on \'The Kelly File\' to NBC\'s presidential forum',
             'timestamp': 1473301045,
             'upload_date': '20160908',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 426,
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         # iframe embed
-        'url': 'http://www.foxnews.com/us/2018/03/09/parkland-survivor-kyle-kashuv-on-meeting-trump-his-app-to-prevent-another-school-shooting.amp.html?__twitter_impression=true',
+        'url': 'https://www.foxnews.com/us/2018/03/09/parkland-survivor-kyle-kashuv-on-meeting-trump-his-app-to-prevent-another-school-shooting.amp.html?__twitter_impression=true',
         'info_dict': {
             'id': '5748266721001',
             'ext': 'flv',
@@ -127,9 +164,7 @@ class FoxNewsArticleIE(InfoExtractor):
             'timestamp': 1520594670,
             'upload_date': '20180309',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'skip': '404 page',
     }, {
         'url': 'http://insider.foxnews.com/2016/08/25/univ-wisconsin-student-group-pushing-silence-certain-words',
         'only_matching': True,

From 4815d35c191e7d375b94492a6486dd2ba43a8954 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 4 Jun 2023 08:49:10 -0500
Subject: [PATCH 334/871] [extractor/sonyliv] Fix login with token (#7223)

Authored by: bashonly
---
 yt_dlp/extractor/sonyliv.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/sonyliv.py b/yt_dlp/extractor/sonyliv.py
index aaad420f1..5ebe20df7 100644
--- a/yt_dlp/extractor/sonyliv.py
+++ b/yt_dlp/extractor/sonyliv.py
@@ -10,6 +10,8 @@
 from ..utils import (
     ExtractorError,
     int_or_none,
+    jwt_decode_hs256,
+    try_call,
     try_get,
 )
 
@@ -77,8 +79,10 @@ def _perform_login(self, username, password):
         self._HEADERS['device_id'] = self._get_device_id()
         self._HEADERS['content-type'] = 'application/json'
 
-        if username.lower() == 'token' and len(password) > 1198:
+        if username.lower() == 'token' and try_call(lambda: jwt_decode_hs256(password)):
             self._HEADERS['authorization'] = password
+            self.report_login()
+            return
         elif len(username) != 10 or not username.isdigit():
             raise ExtractorError(f'Invalid username/password; {self._LOGIN_HINT}')
 

From 7bc92517463f5766e9d9b92c3823b5cf403c0e3d Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 4 Jun 2023 09:07:13 -0500
Subject: [PATCH 335/871] [extractor/shemaroome] Pass `stream_key` header to
 downloader (#7224)

Closes #7133
Authored by: bashonly
---
 yt_dlp/extractor/shemaroome.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/shemaroome.py b/yt_dlp/extractor/shemaroome.py
index 7a78c6e05..ec9938b8c 100644
--- a/yt_dlp/extractor/shemaroome.py
+++ b/yt_dlp/extractor/shemaroome.py
@@ -73,7 +73,10 @@ def _real_extract(self, url):
         key = bytes_to_intlist(compat_b64decode(data_json['key']))
         iv = [0] * 16
         m3u8_url = unpad_pkcs7(intlist_to_bytes(aes_cbc_decrypt(url_data, key, iv))).decode('ascii')
-        formats, m3u8_subs = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, fatal=False, headers={'stream_key': data_json['stream_key']})
+        headers = {'stream_key': data_json['stream_key']}
+        formats, m3u8_subs = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, fatal=False, headers=headers)
+        for fmt in formats:
+            fmt['http_headers'] = headers
 
         release_date = self._html_search_regex(
             (r'itemprop="uploadDate">\s*([\d-]+)', r'id="release_date" value="([\d-]+)'),

From 7f8ddebbb51c9fd4a347306332a718ba41b371b8 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 4 Jun 2023 09:19:16 -0500
Subject: [PATCH 336/871] [extractor/hotstar] Support `/shows/` URLs (#7225)

Closes #6463
Authored by: bashonly
---
 yt_dlp/extractor/hotstar.py | 40 +++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/hotstar.py b/yt_dlp/extractor/hotstar.py
index cea1812f1..591e23b8a 100644
--- a/yt_dlp/extractor/hotstar.py
+++ b/yt_dlp/extractor/hotstar.py
@@ -83,7 +83,7 @@ class HotStarIE(HotStarBaseIE):
     _VALID_URL = r'''(?x)
         https?://(?:www\.)?hotstar\.com(?:/in)?/(?!in/)
         (?:
-            (?P<type>movies|sports|episode|(?P<tv>tv))/
+            (?P<type>movies|sports|episode|(?P<tv>tv|shows))/
             (?(tv)(?:[^/?#]+/){2}|[^?#]*)
         )?
         [^/?#]+/
@@ -122,6 +122,25 @@ class HotStarIE(HotStarBaseIE):
             'episode': 'Janhvi Targets Suman',
             'episode_number': 8,
         }
+    }, {
+        'url': 'https://www.hotstar.com/in/shows/anupama/1260022017/anupama-anuj-share-a-moment/1000282843',
+        'info_dict': {
+            'id': '1000282843',
+            'ext': 'mp4',
+            'title': 'Anupama, Anuj Share a Moment',
+            'season': 'Chapter 1',
+            'description': 'md5:8d74ed2248423b8b06d5c8add4d7a0c0',
+            'timestamp': 1678149000,
+            'channel': 'StarPlus',
+            'series': 'Anupama',
+            'season_number': 1,
+            'season_id': 7399,
+            'upload_date': '20230307',
+            'episode': 'Anupama, Anuj Share a Moment',
+            'episode_number': 853,
+            'duration': 1272,
+            'channel_id': 3,
+        },
     }, {
         'url': 'https://www.hotstar.com/movies/radha-gopalam/1000057157',
         'only_matching': True,
@@ -139,6 +158,7 @@ class HotStarIE(HotStarBaseIE):
         'sports': 'match',
         'episode': 'episode',
         'tv': 'episode',
+        'shows': 'episode',
         None: 'content',
     }
 
@@ -304,13 +324,16 @@ def _real_extract(self, url):
 
 class HotStarPlaylistIE(HotStarBaseIE):
     IE_NAME = 'hotstar:playlist'
-    _VALID_URL = r'https?://(?:www\.)?hotstar\.com(?:/in)?/tv(?:/[^/]+){2}/list/[^/]+/t-(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com(?:/in)?/(?:tv|shows)(?:/[^/]+){2}/list/[^/]+/t-(?P<id>\w+)'
     _TESTS = [{
         'url': 'https://www.hotstar.com/tv/savdhaan-india/s-26/list/popular-clips/t-3_2_26',
         'info_dict': {
             'id': '3_2_26',
         },
         'playlist_mincount': 20,
+    }, {
+        'url': 'https://www.hotstar.com/shows/savdhaan-india/s-26/list/popular-clips/t-3_2_26',
+        'only_matching': True,
     }, {
         'url': 'https://www.hotstar.com/tv/savdhaan-india/s-26/list/extras/t-2480',
         'only_matching': True,
@@ -327,7 +350,7 @@ def _real_extract(self, url):
 
 class HotStarSeasonIE(HotStarBaseIE):
     IE_NAME = 'hotstar:season'
-    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com(?:/in)?/tv/[^/]+/\w+)/seasons/[^/]+/ss-(?P<id>\w+)'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com(?:/in)?/(?:tv|shows)/[^/]+/\w+)/seasons/[^/]+/ss-(?P<id>\w+)'
     _TESTS = [{
         'url': 'https://www.hotstar.com/tv/radhakrishn/1260000646/seasons/season-2/ss-8028',
         'info_dict': {
@@ -346,6 +369,9 @@ class HotStarSeasonIE(HotStarBaseIE):
             'id': '8208',
         },
         'playlist_mincount': 19,
+    }, {
+        'url': 'https://www.hotstar.com/in/shows/bigg-boss/14714/seasons/season-4/ss-8208/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -356,7 +382,7 @@ def _real_extract(self, url):
 
 class HotStarSeriesIE(HotStarBaseIE):
     IE_NAME = 'hotstar:series'
-    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com(?:/in)?/tv/[^/]+/(?P<id>\d+))/?(?:[#?]|$)'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com(?:/in)?/(?:tv|shows)/[^/]+/(?P<id>\d+))/?(?:[#?]|$)'
     _TESTS = [{
         'url': 'https://www.hotstar.com/in/tv/radhakrishn/1260000646',
         'info_dict': {
@@ -375,6 +401,12 @@ class HotStarSeriesIE(HotStarBaseIE):
             'id': '435',
         },
         'playlist_mincount': 267,
+    }, {
+        'url': 'https://www.hotstar.com/in/shows/anupama/1260022017/',
+        'info_dict': {
+            'id': '1260022017',
+        },
+        'playlist_mincount': 940,
     }]
 
     def _real_extract(self, url):

From c2a1bdb00931969193f2a31ea27b9c66a07aaec2 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 4 Jun 2023 09:28:40 -0500
Subject: [PATCH 337/871] [extractor/tiktok] Extract 1080p adaptive formats
 (#7228)

Closes #7109
Authored by: bashonly
---
 yt_dlp/extractor/tiktok.py | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index 63708229e..49035e971 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -62,7 +62,7 @@ def _call_api_impl(self, ep, query, manifest_app_version, video_id, fatal=True,
         return self._download_json(
             'https://%s/aweme/v1/%s/' % (self._API_HOSTNAME, ep), video_id=video_id,
             fatal=fatal, note=note, errnote=errnote, headers={
-                'User-Agent': f'com.ss.android.ugc.{self._APP_NAME}/{manifest_app_version} (Linux; U; Android 10; en_US; Pixel 4; Build/QQ3A.200805.001; Cronet/58.0.2991.0)',
+                'User-Agent': f'com.ss.android.ugc.{self._APP_NAME}/{manifest_app_version} (Linux; U; Android 13; en_US; Pixel 7; Build/TD1A.220804.031; Cronet/58.0.2991.0)',
                 'Accept': 'application/json',
             }, query=query)
 
@@ -79,11 +79,11 @@ def _build_api_query(self, query, app_version, manifest_app_version):
             '_rticket': int(time.time() * 1000),
             'ts': int(time.time()),
             'device_brand': 'Google',
-            'device_type': 'Pixel 4',
+            'device_type': 'Pixel 7',
             'device_platform': 'android',
-            'resolution': '1080*1920',
+            'resolution': '1080*2400',
             'dpi': 420,
-            'os_version': '10',
+            'os_version': '13',
             'os_api': '29',
             'carrier_region': 'US',
             'sys_region': 'US',
@@ -624,6 +624,32 @@ class TikTokIE(TikTokBaseIE):
             'thumbnails': 'count:3',
         },
         'expected_warnings': ['Unable to find video in feed'],
+    }, {
+        # 1080p format
+        'url': 'https://www.tiktok.com/@tatemcrae/video/7107337212743830830',
+        'md5': '982512017a8a917124d5a08c8ae79621',
+        'info_dict': {
+            'id': '7107337212743830830',
+            'ext': 'mp4',
+            'title': 'new music video 4 don’t come backkkk🧸🖤 i hope u enjoy !! @musicontiktok',
+            'description': 'new music video 4 don’t come backkkk🧸🖤 i hope u enjoy !! @musicontiktok',
+            'uploader': 'tatemcrae',
+            'uploader_id': '86328792343818240',
+            'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAA-0bQT0CqebTRr6I4IkYvMDMKSRSJHLNPBo5HrSklJwyA2psXLSZG5FP-LMNpHnJd',
+            'channel_id': 'MS4wLjABAAAA-0bQT0CqebTRr6I4IkYvMDMKSRSJHLNPBo5HrSklJwyA2psXLSZG5FP-LMNpHnJd',
+            'creator': 't8',
+            'artist': 't8',
+            'track': 'original sound',
+            'upload_date': '20220609',
+            'timestamp': 1654805899,
+            'duration': 150,
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+            'thumbnail': r're:^https://.+\.webp',
+        },
+        'params': {'format': 'bytevc1_1080p_808907-0'},
     }, {
         # Auto-captions available
         'url': 'https://www.tiktok.com/@hankgreen1/video/7047596209028074758',

From ee0ed0338df328cd986f97315c8162b5a151476d Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Mon, 5 Jun 2023 10:40:48 -0500
Subject: [PATCH 338/871] [extractor/zdf] Fix formats extraction

Closes #7238, Closes #7240
Authored by: bashonly
---
 yt_dlp/extractor/zdf.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/zdf.py b/yt_dlp/extractor/zdf.py
index c863c46ed..c04d51b7e 100644
--- a/yt_dlp/extractor/zdf.py
+++ b/yt_dlp/extractor/zdf.py
@@ -24,7 +24,7 @@
 
 class ZDFBaseIE(InfoExtractor):
     _GEO_COUNTRIES = ['DE']
-    _QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh', 'hd', 'uhd')
+    _QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh', 'hd', 'fhd', 'uhd')
 
     def _call_api(self, url, video_id, item, api_token=None, referrer=None):
         headers = {}
@@ -61,6 +61,9 @@ def _extract_format(self, video_id, formats, format_urls, meta):
         elif mime_type == 'application/f4m+xml' or ext == 'f4m':
             new_formats = self._extract_f4m_formats(
                 update_url_query(format_url, {'hdcore': '3.7.0'}), video_id, f4m_id='hds', fatal=False)
+        elif ext == 'mpd':
+            new_formats = self._extract_mpd_formats(
+                format_url, video_id, mpd_id='dash', fatal=False)
         else:
             f = parse_codecs(meta.get('mimeCodec'))
             if not f and meta.get('type'):

From 59d9fe08312bbb76ee26238d207a8ca35410a48d Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Mon, 5 Jun 2023 10:52:45 -0500
Subject: [PATCH 339/871] [extractor/mgtv] Fix formats extraction (#7234)

Closes #7008
Authored by: bashonly
---
 yt_dlp/extractor/mgtv.py | 65 ++++++++++++++++++++++++----------------
 1 file changed, 40 insertions(+), 25 deletions(-)

diff --git a/yt_dlp/extractor/mgtv.py b/yt_dlp/extractor/mgtv.py
index edc92b371..06edcb396 100644
--- a/yt_dlp/extractor/mgtv.py
+++ b/yt_dlp/extractor/mgtv.py
@@ -1,17 +1,17 @@
 import base64
 import time
+import urllib.error
 import uuid
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
 from ..utils import (
     ExtractorError,
     int_or_none,
+    parse_resolution,
+    traverse_obj,
     try_get,
     url_or_none,
+    urljoin,
 )
 
 
@@ -30,16 +30,18 @@ class MGTVIE(InfoExtractor):
             'duration': 7461,
             'thumbnail': r're:^https?://.*\.jpg$',
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://w.mgtv.com/b/427837/15588271.html',
         'info_dict': {
             'id': '15588271',
             'ext': 'mp4',
-            'title': '春日迟迟再出发 沉浸版',
+            'title': '春日迟迟再出发 沉浸版第1期：陆莹结婚半年查出肾炎被离婚 吴雅婷把一半票根退给前夫',
             'description': 'md5:a7a05a05b1aa87bd50cae619b19bbca6',
             'thumbnail': r're:^https?://.+\.jpg',
             'duration': 4026,
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://w.mgtv.com/b/333652/7329822.html',
         'info_dict': {
@@ -50,6 +52,7 @@ class MGTVIE(InfoExtractor):
             'thumbnail': r're:^https?://.+\.jpg',
             'duration': 2656,
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://w.mgtv.com/b/427837/15591647.html',
         'only_matching': True,
@@ -64,6 +67,13 @@ class MGTVIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    _RESOLUTIONS = {
+        '标清': ('480p', '854x480'),
+        '高清': ('540p', '960x540'),
+        '超清': ('720p', '1280x720'),
+        '蓝光': ('1080p', '1920x1080'),
+    }
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         tk2 = base64.urlsafe_b64encode(
@@ -76,55 +86,60 @@ def _real_extract(self, url):
                     'type': 'pch5'
                 }, headers=self.geo_verification_headers())['data']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
                 error = self._parse_json(e.cause.read().decode(), None)
                 if error.get('code') == 40005:
                     self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
                 raise ExtractorError(error['msg'], expected=True)
             raise
-        info = api_data['info']
-        title = info['title'].strip()
+
         stream_data = self._download_json(
             'https://pcweb.api.mgtv.com/player/getSource', video_id, query={
-                'pm2': api_data['atc']['pm2'],
                 'tk2': tk2,
+                'pm2': api_data['atc']['pm2'],
                 'video_id': video_id,
+                'type': 'pch5',
                 'src': 'intelmgtv',
             }, headers=self.geo_verification_headers())['data']
-        stream_domain = stream_data['stream_domain'][0]
+        stream_domain = traverse_obj(stream_data, ('stream_domain', ..., {url_or_none}), get_all=False)
 
         formats = []
-        for idx, stream in enumerate(stream_data['stream']):
-            stream_path = stream.get('url')
-            if not stream_path:
-                continue
-            format_data = self._download_json(
-                stream_domain + stream_path, video_id,
-                note=f'Download video info for format #{idx}')
-            format_url = format_data.get('info')
+        for idx, stream in enumerate(traverse_obj(stream_data, ('stream', lambda _, v: v['url']))):
+            stream_name = traverse_obj(stream, 'name', 'standardName', 'barName', expected_type=str)
+            resolution = traverse_obj(
+                self._RESOLUTIONS, (stream_name, 1 if stream.get('scale') == '16:9' else 0))
+            format_url = traverse_obj(self._download_json(
+                urljoin(stream_domain, stream['url']), video_id, fatal=False,
+                note=f'Downloading video info for format {resolution or stream_name}'),
+                ('info', {url_or_none}))
             if not format_url:
                 continue
             tbr = int_or_none(stream.get('filebitrate') or self._search_regex(
                 r'_(\d+)_mp4/', format_url, 'tbr', default=None))
             formats.append({
-                'format_id': compat_str(tbr or idx),
-                'url': url_or_none(format_url),
+                'format_id': str(tbr or idx),
+                'url': format_url,
                 'ext': 'mp4',
                 'tbr': tbr,
+                'vcodec': stream.get('videoFormat'),
+                'acodec': stream.get('audioFormat'),
+                **parse_resolution(resolution),
                 'protocol': 'm3u8_native',
                 'http_headers': {
                     'Referer': url,
                 },
-                'format_note': stream.get('name'),
+                'format_note': stream_name,
             })
 
         return {
             'id': video_id,
-            'title': title,
             'formats': formats,
-            'description': info.get('desc'),
-            'duration': int_or_none(info.get('duration')),
-            'thumbnail': info.get('thumb'),
+            **traverse_obj(api_data, ('info', {
+                'title': ('title', {str.strip}),
+                'description': ('desc', {str}),
+                'duration': ('duration', {int_or_none}),
+                'thumbnail': ('thumb', {url_or_none}),
+            })),
             'subtitles': self.extract_subtitles(video_id, stream_domain),
         }
 

From c2b801fea59628d5c873e06a0727fbf2051bbd1f Mon Sep 17 00:00:00 2001
From: stanoarn <74262064+stanoarn@users.noreply.github.com>
Date: Wed, 7 Jun 2023 22:18:06 +0200
Subject: [PATCH 340/871] [extractor/rozhlas] `MujRozhlas`: Add extractor
 (#7129)

Authored by: stanoarn
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/rozhlas.py     | 164 ++++++++++++++++++++++++++++----
 2 files changed, 144 insertions(+), 21 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 7120fd37d..f54024211 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1625,6 +1625,7 @@
 from .rozhlas import (
     RozhlasIE,
     RozhlasVltavaIE,
+    MujRozhlasIE,
 )
 from .rte import RteIE, RteRadioIE
 from .rtlnl import (
diff --git a/yt_dlp/extractor/rozhlas.py b/yt_dlp/extractor/rozhlas.py
index 5cc664e00..5f83d42e8 100644
--- a/yt_dlp/extractor/rozhlas.py
+++ b/yt_dlp/extractor/rozhlas.py
@@ -1,10 +1,15 @@
+import itertools
+import urllib.error
+
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     extract_attributes,
     int_or_none,
     remove_start,
     str_or_none,
     traverse_obj,
+    unified_timestamp,
     url_or_none,
 )
 
@@ -51,7 +56,40 @@ def _real_extract(self, url):
         }
 
 
-class RozhlasVltavaIE(InfoExtractor):
+class RozhlasBaseIE(InfoExtractor):
+    def _extract_formats(self, entry, audio_id):
+        formats = []
+        for audio in traverse_obj(entry, ('audioLinks', lambda _, v: url_or_none(v['url']))):
+            ext = audio.get('variant')
+            for retry in self.RetryManager():
+                if retry.attempt > 1:
+                    self._sleep(1, audio_id)
+                try:
+                    if ext == 'dash':
+                        formats.extend(self._extract_mpd_formats(
+                            audio['url'], audio_id, mpd_id=ext))
+                    elif ext == 'hls':
+                        formats.extend(self._extract_m3u8_formats(
+                            audio['url'], audio_id, 'm4a', m3u8_id=ext))
+                    else:
+                        formats.append({
+                            'url': audio['url'],
+                            'ext': ext,
+                            'format_id': ext,
+                            'abr': int_or_none(audio.get('bitrate')),
+                            'acodec': ext,
+                            'vcodec': 'none',
+                        })
+                except ExtractorError as e:
+                    if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 429:
+                        retry.error = e.cause
+                    else:
+                        self.report_warning(e.msg)
+
+        return formats
+
+
+class RozhlasVltavaIE(RozhlasBaseIE):
     _VALID_URL = r'https?://(?:\w+\.rozhlas|english\.radio)\.cz/[\w-]+-(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://wave.rozhlas.cz/papej-masicko-porcujeme-a-bilancujeme-filmy-a-serialy-ktere-letos-zabily-8891337',
@@ -168,33 +206,14 @@ class RozhlasVltavaIE(InfoExtractor):
     }]
 
     def _extract_video(self, entry):
-        formats = []
         audio_id = entry['meta']['ga']['contentId']
-        for audio in traverse_obj(entry, ('audioLinks', lambda _, v: url_or_none(v['url']))):
-            ext = audio.get('variant')
-            if ext == 'dash':
-                formats.extend(self._extract_mpd_formats(
-                    audio['url'], audio_id, mpd_id=ext, fatal=False))
-            elif ext == 'hls':
-                formats.extend(self._extract_m3u8_formats(
-                    audio['url'], audio_id, 'm4a', m3u8_id=ext, fatal=False))
-            else:
-                formats.append({
-                    'url': audio['url'],
-                    'ext': ext,
-                    'format_id': ext,
-                    'abr': int_or_none(audio.get('bitrate')),
-                    'acodec': ext,
-                    'vcodec': 'none',
-                })
-
         chapter_number = traverse_obj(entry, ('meta', 'ga', 'contentSerialPart', {int_or_none}))
 
         return {
             'id': audio_id,
             'chapter': traverse_obj(entry, ('meta', 'ga', 'contentNameShort')) if chapter_number else None,
             'chapter_number': chapter_number,
-            'formats': formats,
+            'formats': self._extract_formats(entry, audio_id),
             **traverse_obj(entry, {
                 'title': ('meta', 'ga', 'contentName'),
                 'description': 'title',
@@ -219,3 +238,106 @@ def _real_extract(self, url):
             'title': traverse_obj(data, ('series', 'title')),
             'entries': map(self._extract_video, data['playlist']),
         }
+
+
+class MujRozhlasIE(RozhlasBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?mujrozhlas\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        # single episode extraction
+        'url': 'https://www.mujrozhlas.cz/vykopavky/ach-jo-zase-teleci-rizek-je-mnohem-min-cesky-nez-jsme-si-mysleli',
+        'md5': '6f8fd68663e64936623e67c152a669e0',
+        'info_dict': {
+            'id': '10739193',
+            'ext': 'mp3',
+            'title': 'Ach jo, zase to telecí! Řízek je mnohem míň český, než jsme si mysleli',
+            'description': 'md5:db7141e9caaedc9041ec7cefb9a62908',
+            'timestamp': 1684915200,
+            'modified_timestamp': 1684922446,
+            'series': 'Vykopávky',
+            'thumbnail': 'https://portal.rozhlas.cz/sites/default/files/images/84377046610af6ddc54d910b1dd7a22b.jpg',
+            'channel_id': 'radio-wave',
+            'upload_date': '20230524',
+            'modified_date': '20230524',
+        },
+    }, {
+        # serial extraction
+        'url': 'https://www.mujrozhlas.cz/radiokniha/jaroslava-janackova-pribeh-tajemneho-psani-o-pramenech-genezi-babicky',
+        'playlist_mincount': 7,
+        'info_dict': {
+            'id': 'bb2b5f4e-ffb4-35a6-a34a-046aa62d6f6b',
+            'title': 'Jaroslava Janáčková: Příběh tajemného psaní. O pramenech a genezi Babičky',
+            'description': 'md5:7434d8fac39ac9fee6df098e11dfb1be',
+        },
+    }, {
+        # show extraction
+        'url': 'https://www.mujrozhlas.cz/nespavci',
+        'playlist_mincount': 14,
+        'info_dict': {
+            'id': '09db9b37-d0f4-368c-986a-d3439f741f08',
+            'title': 'Nespavci',
+            'description': 'md5:c430adcbf9e2b9eac88b745881e814dc',
+        },
+    }]
+
+    def _call_api(self, path, item_id, msg='API JSON'):
+        return self._download_json(
+            f'https://api.mujrozhlas.cz/{path}/{item_id}', item_id,
+            note=f'Downloading {msg}', errnote=f'Failed to download {msg}')['data']
+
+    def _extract_audio_entry(self, entry):
+        audio_id = entry['meta']['ga']['contentId']
+
+        return {
+            'id': audio_id,
+            'formats': self._extract_formats(entry['attributes'], audio_id),
+            **traverse_obj(entry, {
+                'title': ('attributes', 'title'),
+                'description': ('attributes', 'description'),
+                'episode_number': ('attributes', 'part'),
+                'series': ('attributes', 'mirroredShow', 'title'),
+                'chapter': ('attributes', 'mirroredSerial', 'title'),
+                'artist': ('meta', 'ga', 'contentAuthor'),
+                'channel_id': ('meta', 'ga', 'contentCreator'),
+                'timestamp': ('attributes', 'since', {unified_timestamp}),
+                'modified_timestamp': ('attributes', 'updated', {unified_timestamp}),
+                'thumbnail': ('attributes', 'asset', 'url', {url_or_none}),
+            })
+        }
+
+    def _entries(self, api_url, playlist_id):
+        for page in itertools.count(1):
+            episodes = self._download_json(
+                api_url, playlist_id, note=f'Downloading episodes page {page}',
+                errnote=f'Failed to download episodes page {page}', fatal=False)
+            for episode in traverse_obj(episodes, ('data', lambda _, v: v['meta']['ga']['contentId'])):
+                yield self._extract_audio_entry(episode)
+            api_url = traverse_obj(episodes, ('links', 'next', {url_or_none}))
+            if not api_url:
+                break
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        info = self._search_json(r'\bvar\s+dl\s*=', webpage, 'info json', display_id)
+
+        entity = info['siteEntityBundle']
+
+        if entity == 'episode':
+            return self._extract_audio_entry(self._call_api(
+                'episodes', info['contentId'], 'episode info API JSON'))
+
+        elif entity in ('show', 'serial'):
+            playlist_id = info['contentShow'].split(':')[0] if entity == 'show' else info['contentId']
+            data = self._call_api(f'{entity}s', playlist_id, f'{entity} playlist JSON')
+            api_url = data['relationships']['episodes']['links']['related']
+            return self.playlist_result(
+                self._entries(api_url, playlist_id), playlist_id,
+                **traverse_obj(data, ('attributes', {
+                    'title': 'title',
+                    'description': 'description',
+                })))
+
+        else:
+            # `entity == 'person'` not implemented yet by API, ref:
+            # https://api.mujrozhlas.cz/persons/8367e456-2a57-379a-91bb-e699619bea49/participation
+            raise ExtractorError(f'Unsupported entity type "{entity}"')

From 14a14335b280766fbf5a469ae26836d6c1fe450a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 8 Jun 2023 18:58:49 +0530
Subject: [PATCH 341/871] [extractor/youtube] Misc cleanup

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 93 ++++++++++++++++++++++++++-----------
 1 file changed, 66 insertions(+), 27 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 6e7485c03..1b1266360 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -292,6 +292,7 @@ class BadgeType(enum.Enum):
     AVAILABILITY_PREMIUM = enum.auto()
     AVAILABILITY_SUBSCRIPTION = enum.auto()
     LIVE_NOW = enum.auto()
+    VERIFIED = enum.auto()
 
 
 class YoutubeBaseInfoExtractor(InfoExtractor):
@@ -791,17 +792,23 @@ def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
     def _extract_and_report_alerts(self, data, *args, **kwargs):
         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 
-    def _extract_badges(self, renderer: dict):
-        privacy_icon_map = {
+    def _extract_badges(self, badge_list: list):
+        """
+        Extract known BadgeType's from a list of badge renderers.
+        @returns [{'type': BadgeType}]
+        """
+        icon_type_map = {
             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
-            'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
+            'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC,
+            'CHECK_CIRCLE_THICK': BadgeType.VERIFIED,
+            'OFFICIAL_ARTIST_BADGE': BadgeType.VERIFIED,
         }
 
         badge_style_map = {
             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
-            'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
+            'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW,
         }
 
         label_map = {
@@ -809,13 +816,13 @@ def _extract_badges(self, renderer: dict):
             'private': BadgeType.AVAILABILITY_PRIVATE,
             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
             'live': BadgeType.LIVE_NOW,
-            'premium': BadgeType.AVAILABILITY_PREMIUM
+            'premium': BadgeType.AVAILABILITY_PREMIUM,
         }
 
         badges = []
-        for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer')):
+        for badge in traverse_obj(badge_list, (..., lambda key, _: re.search(r'[bB]adgeRenderer$', key))):
             badge_type = (
-                privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
+                icon_type_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
                 or badge_style_map.get(traverse_obj(badge, 'style'))
             )
             if badge_type:
@@ -823,11 +830,12 @@ def _extract_badges(self, renderer: dict):
                 continue
 
             # fallback, won't work in some languages
-            label = traverse_obj(badge, 'label', expected_type=str, default='')
+            label = traverse_obj(
+                badge, 'label', ('accessibilityData', 'label'), 'tooltip', 'iconTooltip', get_all=False, expected_type=str, default='')
             for match, label_badge_type in label_map.items():
                 if match in label.lower():
-                    badges.append({'type': badge_type})
-                    continue
+                    badges.append({'type': label_badge_type})
+                    break
 
         return badges
 
@@ -1020,8 +1028,7 @@ def _extract_video(self, renderer):
         overlay_style = traverse_obj(
             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
             get_all=False, expected_type=str)
-        badges = self._extract_badges(renderer)
-
+        badges = self._extract_badges(traverse_obj(renderer, 'badges'))
         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
             expected_type=str)) or ''
@@ -1079,7 +1086,7 @@ def _extract_video(self, renderer):
                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
             view_count_field: view_count,
-            'live_status': live_status
+            'live_status': live_status,
         }
 
 
@@ -1332,6 +1339,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
                 'uploader_id': '@PhilippHagemeister',
+                'heatmap': 'count:100',
             },
             'params': {
                 'skip_download': True,
@@ -1415,6 +1423,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'The Witcher',
                 'uploader_url': 'https://www.youtube.com/@thewitcher',
                 'uploader_id': '@thewitcher',
+                'comment_count': int,
+                'heatmap': 'count:100',
             },
         },
         {
@@ -1894,6 +1904,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Bernie Sanders',
                 'uploader_url': 'https://www.youtube.com/@BernieSanders',
                 'uploader_id': '@BernieSanders',
+                'heatmap': 'count:100',
             },
             'params': {
                 'skip_download': True,
@@ -1955,6 +1966,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Vsauce',
                 'uploader_url': 'https://www.youtube.com/@Vsauce',
                 'uploader_id': '@Vsauce',
+                'comment_count': int,
             },
             'params': {
                 'skip_download': True,
@@ -2147,6 +2159,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'kudvenkat',
                 'uploader_url': 'https://www.youtube.com/@Csharp-video-tutorialsBlogspot',
                 'uploader_id': '@Csharp-video-tutorialsBlogspot',
+                'heatmap': 'count:100',
             },
             'params': {
                 'skip_download': True,
@@ -2227,6 +2240,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'CBS Mornings',
                 'uploader_url': 'https://www.youtube.com/@CBSMornings',
                 'uploader_id': '@CBSMornings',
+                'comment_count': int,
             }
         },
         {
@@ -2297,6 +2311,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'colinfurze',
                 'uploader_url': 'https://www.youtube.com/@colinfurze',
                 'uploader_id': '@colinfurze',
+                'comment_count': int,
+                'heatmap': 'count:100',
             },
             'params': {
                 'format': '17',  # 3gp format available on android
@@ -2342,6 +2358,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'SciShow',
                 'uploader_url': 'https://www.youtube.com/@SciShow',
                 'uploader_id': '@SciShow',
+                'comment_count': int,
+                'heatmap': 'count:100',
             }, 'params': {'format': 'mhtml', 'skip_download': True}
         }, {
             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
@@ -2370,6 +2388,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Leon Nguyen',
                 'uploader_url': 'https://www.youtube.com/@LeonNguyen',
                 'uploader_id': '@LeonNguyen',
+                'heatmap': 'count:100',
             }
         }, {
             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
@@ -2398,6 +2417,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Leon Nguyen',
                 'uploader_url': 'https://www.youtube.com/@LeonNguyen',
                 'uploader_id': '@LeonNguyen',
+                'heatmap': 'count:100',
             },
             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
         }, {
@@ -2428,6 +2448,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Quackity',
                 'uploader_id': '@Quackity',
                 'uploader_url': 'https://www.youtube.com/@Quackity',
+                'comment_count': int,
+                'heatmap': 'count:100',
             }
         },
         {   # continuous livestream. Microformat upload date should be preferred.
@@ -2594,6 +2616,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'MrBeast',
                 'uploader_url': 'https://www.youtube.com/@MrBeast',
                 'uploader_id': '@MrBeast',
+                'comment_count': int,
+                'heatmap': 'count:100',
             },
             'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
         }, {
@@ -2655,6 +2679,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'さなちゃんねる',
                 'uploader_url': 'https://www.youtube.com/@sana_natori',
                 'uploader_id': '@sana_natori',
+                'heatmap': 'count:100',
             },
         },
         {
@@ -2684,6 +2709,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'thumbnail': r're:^https?://.*\.webp',
                 'channel_url': 'https://www.youtube.com/channel/UCxzC4EngIsMrPmbm6Nxvb-A',
                 'playable_in_embed': True,
+                'comment_count': int,
+                'heatmap': 'count:100',
             },
             'params': {
                 'extractor_args': {'youtube': {'player_client': ['android'], 'player_skip': ['webpage']}},
@@ -2720,6 +2747,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Christopher Sykes',
                 'uploader_url': 'https://www.youtube.com/@ChristopherSykesDocumentaries',
                 'uploader_id': '@ChristopherSykesDocumentaries',
+                'heatmap': 'count:100',
             },
             'params': {
                 'skip_download': True,
@@ -3312,10 +3340,9 @@ def _extract_comment(self, comment_renderer, parent=None):
         if comment_abr is not None:
             info['is_favorited'] = 'creatorHeart' in comment_abr
 
-        comment_ab_icontype = traverse_obj(
-            comment_renderer, ('authorCommentBadge', 'authorCommentBadgeRenderer', 'icon', 'iconType'))
-        if comment_ab_icontype is not None:
-            info['author_is_verified'] = comment_ab_icontype in ('CHECK_CIRCLE_THICK', 'OFFICIAL_ARTIST_BADGE')
+        badges = self._extract_badges([traverse_obj(comment_renderer, 'authorCommentBadge')])
+        if self._has_badge(badges, BadgeType.VERIFIED):
+            info['author_is_verified'] = True
 
         is_pinned = traverse_obj(comment_renderer, 'pinnedCommentBadge')
         if is_pinned:
@@ -4481,7 +4508,7 @@ def process_language(container, base_url, lang_code, sub_name, query):
             if v:
                 info[d_k] = v
 
-        badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
+        badges = self._extract_badges(traverse_obj(vpir, 'badges'))
 
         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
                       or get_first(video_details, 'isPrivate', expected_type=bool))
@@ -4554,13 +4581,14 @@ def _extract_channel_renderer(self, renderer):
         channel_id = self.ucid_or_none(renderer['channelId'])
         title = self._get_text(renderer, 'title')
         channel_url = format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None)
-        # As of 2023-03-01 YouTube doesn't use the channel handles on these renderers yet.
-        # However we can expect them to change that in the future.
         channel_handle = self.handle_from_url(
             traverse_obj(renderer, (
                 'navigationEndpoint', (('commandMetadata', 'webCommandMetadata', 'url'),
                                        ('browseEndpoint', 'canonicalBaseUrl')),
                 {str}), get_all=False))
+        if not channel_handle:
+            # As of 2023-06-01, YouTube sets subscriberCountText to the handle in search
+            channel_handle = self.handle_or_none(self._get_text(renderer, 'subscriberCountText'))
         return {
             '_type': 'url',
             'url': channel_url,
@@ -4573,9 +4601,15 @@ def _extract_channel_renderer(self, renderer):
             'title': title,
             'uploader_id': channel_handle,
             'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
-            'channel_follower_count': self._get_count(renderer, 'subscriberCountText'),
+            # See above. YouTube sets videoCountText to the subscriber text in search channel renderers.
+            # However, in feed/channels this is set correctly to the subscriber count
+            'channel_follower_count': traverse_obj(
+                renderer, 'subscriberCountText', 'videoCountText', expected_type=self._get_count),
             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
-            'playlist_count': self._get_count(renderer, 'videoCountText'),
+            'playlist_count': (
+                # videoCountText may be the subscriber count
+                self._get_count(renderer, 'videoCountText')
+                if self._get_count(renderer, 'subscriberCountText') is not None else None),
             'description': self._get_text(renderer, 'descriptionSnippet'),
         }
 
@@ -5100,7 +5134,7 @@ def _extract_availability(self, data):
         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
         player_header_privacy = playlist_header_renderer.get('privacy')
 
-        badges = self._extract_badges(sidebar_renderer)
+        badges = self._extract_badges(traverse_obj(sidebar_renderer, 'badges'))
 
         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
         privacy_setting_icon = get_first(
@@ -5350,7 +5384,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader_url': 'https://www.youtube.com/@3blue1brown',
             'uploader': '3Blue1Brown',
             'tags': ['Mathematics'],
-            'channel_follower_count': int
+            'channel_follower_count': int,
         },
     }, {
         'note': 'playlists, singlepage',
@@ -5690,7 +5724,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
     }, {
         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
         'info_dict': {
-            'id': 'AlTsmyW4auo',  # This will keep changing
+            'id': 'hGkQjiJLjWQ',  # This will keep changing
             'ext': 'mp4',
             'title': str,
             'upload_date': r're:\d{8}',
@@ -6202,7 +6236,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
                 'channel': str,
                 'uploader': str,
                 'uploader_url': str,
-                'uploader_id': str
+                'uploader_id': str,
             }
         }],
         'params': {'extract_flat': True, 'playlist_items': '1'},
@@ -6865,12 +6899,14 @@ class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
                 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
                 'title': 'Kurzgesagt – In a Nutshell',
                 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
-                'playlist_count': int,  # XXX: should have a way of saying > 1
+                # No longer available for search as it is set to the handle.
+                # 'playlist_count': int,
                 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
                 'thumbnails': list,
                 'uploader_id': '@kurzgesagt',
                 'uploader_url': 'https://www.youtube.com/@kurzgesagt',
                 'uploader': 'Kurzgesagt – In a Nutshell',
+                'channel_follower_count': int,
             }
         }],
         'params': {'extract_flat': True, 'playlist_items': '1'},
@@ -7134,6 +7170,8 @@ class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
             'live_status': 'not_live',
             'channel_follower_count': int,
             'chapters': 'count:20',
+            'comment_count': int,
+            'heatmap': 'count:100',
         }
     }]
 
@@ -7194,6 +7232,7 @@ class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
             'channel': 'さなちゃんねる',
             'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
             'uploader': 'さなちゃんねる',
+            'heatmap': 'count:100',
         },
         'add_ie': ['Youtube'],
         'params': {'skip_download': 'Youtube'},

From 8213ce28a485e200f6a7e1af1434a987c8e702bd Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Thu, 8 Jun 2023 19:50:05 +1200
Subject: [PATCH 342/871] [extractor/youtube] Extract `channel_is_verified`
 (#7213)

Authored by: coletdjnz
---
 README.md                   |  1 +
 yt_dlp/extractor/common.py  |  1 +
 yt_dlp/extractor/youtube.py | 38 +++++++++++++++++++++++++++++++++++++
 3 files changed, 40 insertions(+)

diff --git a/README.md b/README.md
index 3d89c0af9..ce555c66f 100644
--- a/README.md
+++ b/README.md
@@ -1292,6 +1292,7 @@ # OUTPUT TEMPLATE
  - `channel` (string): Full name of the channel the video is uploaded on
  - `channel_id` (string): Id of the channel
  - `channel_follower_count` (numeric): Number of followers of the channel
+ - `channel_is_verified` (boolean): Whether the channel is verified on the platform
  - `location` (string): Physical location where the video was filmed
  - `duration` (numeric): Length of the video in seconds
  - `duration_string` (string): Length of the video (HH:mm:ss)
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index fa46a5240..ca2164a5d 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -286,6 +286,7 @@ class InfoExtractor:
     channel_id:     Id of the channel.
     channel_url:    Full URL to a channel webpage.
     channel_follower_count: Number of followers of the channel.
+    channel_is_verified: Whether the channel is verified on the platform.
     location:       Physical location where the video was filmed.
     subtitles:      The available subtitles as a dictionary in the format
                     {tag: subformats}. "tag" is usually a language code, and
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 1b1266360..47ad1da76 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -803,12 +803,15 @@ def _extract_badges(self, badge_list: list):
             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC,
             'CHECK_CIRCLE_THICK': BadgeType.VERIFIED,
             'OFFICIAL_ARTIST_BADGE': BadgeType.VERIFIED,
+            'CHECK': BadgeType.VERIFIED,
         }
 
         badge_style_map = {
             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW,
+            'BADGE_STYLE_TYPE_VERIFIED': BadgeType.VERIFIED,
+            'BADGE_STYLE_TYPE_VERIFIED_ARTIST': BadgeType.VERIFIED
         }
 
         label_map = {
@@ -817,6 +820,8 @@ def _extract_badges(self, badge_list: list):
             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
             'live': BadgeType.LIVE_NOW,
             'premium': BadgeType.AVAILABILITY_PREMIUM,
+            'verified': BadgeType.VERIFIED,
+            'official artist channel': BadgeType.VERIFIED
         }
 
         badges = []
@@ -1029,6 +1034,7 @@ def _extract_video(self, renderer):
             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
             get_all=False, expected_type=str)
         badges = self._extract_badges(traverse_obj(renderer, 'badges'))
+        owner_badges = self._extract_badges(traverse_obj(renderer, 'ownerBadges'))
         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
             expected_type=str)) or ''
@@ -1087,6 +1093,7 @@ def _extract_video(self, renderer):
                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
             view_count_field: view_count,
             'live_status': live_status,
+            'channel_is_verified': True if self._has_badge(owner_badges, BadgeType.VERIFIED) else None
         }
 
 
@@ -1424,6 +1431,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': 'https://www.youtube.com/@thewitcher',
                 'uploader_id': '@thewitcher',
                 'comment_count': int,
+                'channel_is_verified': True,
                 'heatmap': 'count:100',
             },
         },
@@ -1454,6 +1462,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': 'https://www.youtube.com/@FlyingKitty900',
                 'uploader_id': '@FlyingKitty900',
                 'comment_count': int,
+                'channel_is_verified': True,
             },
         },
         {
@@ -1587,6 +1596,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Olympics',
                 'uploader_url': 'https://www.youtube.com/@Olympics',
                 'uploader_id': '@Olympics',
+                'channel_is_verified': True,
             },
             'params': {
                 'skip_download': 'requires avconv',
@@ -1904,6 +1914,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Bernie Sanders',
                 'uploader_url': 'https://www.youtube.com/@BernieSanders',
                 'uploader_id': '@BernieSanders',
+                'channel_is_verified': True,
                 'heatmap': 'count:100',
             },
             'params': {
@@ -1967,6 +1978,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': 'https://www.youtube.com/@Vsauce',
                 'uploader_id': '@Vsauce',
                 'comment_count': int,
+                'channel_is_verified': True,
             },
             'params': {
                 'skip_download': True,
@@ -2159,6 +2171,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'kudvenkat',
                 'uploader_url': 'https://www.youtube.com/@Csharp-video-tutorialsBlogspot',
                 'uploader_id': '@Csharp-video-tutorialsBlogspot',
+                'channel_is_verified': True,
                 'heatmap': 'count:100',
             },
             'params': {
@@ -2241,6 +2254,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': 'https://www.youtube.com/@CBSMornings',
                 'uploader_id': '@CBSMornings',
                 'comment_count': int,
+                'channel_is_verified': True,
             }
         },
         {
@@ -2312,6 +2326,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': 'https://www.youtube.com/@colinfurze',
                 'uploader_id': '@colinfurze',
                 'comment_count': int,
+                'channel_is_verified': True,
                 'heatmap': 'count:100',
             },
             'params': {
@@ -2359,6 +2374,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': 'https://www.youtube.com/@SciShow',
                 'uploader_id': '@SciShow',
                 'comment_count': int,
+                'channel_is_verified': True,
                 'heatmap': 'count:100',
             }, 'params': {'format': 'mhtml', 'skip_download': True}
         }, {
@@ -2449,6 +2465,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': '@Quackity',
                 'uploader_url': 'https://www.youtube.com/@Quackity',
                 'comment_count': int,
+                'channel_is_verified': True,
                 'heatmap': 'count:100',
             }
         },
@@ -2617,6 +2634,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': 'https://www.youtube.com/@MrBeast',
                 'uploader_id': '@MrBeast',
                 'comment_count': int,
+                'channel_is_verified': True,
                 'heatmap': 'count:100',
             },
             'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
@@ -2679,6 +2697,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'さなちゃんねる',
                 'uploader_url': 'https://www.youtube.com/@sana_natori',
                 'uploader_id': '@sana_natori',
+                'channel_is_verified': True,
                 'heatmap': 'count:100',
             },
         },
@@ -2710,6 +2729,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel_url': 'https://www.youtube.com/channel/UCxzC4EngIsMrPmbm6Nxvb-A',
                 'playable_in_embed': True,
                 'comment_count': int,
+                'channel_is_verified': True,
                 'heatmap': 'count:100',
             },
             'params': {
@@ -4483,6 +4503,9 @@ def process_language(container, base_url, lang_code, sub_name, query):
                         info['artist'] = mrr_contents_text
                     elif mrr_title == 'Song':
                         info['track'] = mrr_contents_text
+            owner_badges = self._extract_badges(traverse_obj(vsir, ('owner', 'videoOwnerRenderer', 'badges')))
+            if self._has_badge(owner_badges, BadgeType.VERIFIED):
+                info['channel_is_verified'] = True
 
         info.update({
             'uploader': info.get('channel'),
@@ -4611,6 +4634,8 @@ def _extract_channel_renderer(self, renderer):
                 self._get_count(renderer, 'videoCountText')
                 if self._get_count(renderer, 'subscriberCountText') is not None else None),
             'description': self._get_text(renderer, 'descriptionSnippet'),
+            'channel_is_verified': True if self._has_badge(
+                self._extract_badges(traverse_obj(renderer, 'ownerBadges')), BadgeType.VERIFIED) else None,
         }
 
     def _grid_entries(self, grid_renderer):
@@ -5026,6 +5051,10 @@ def _get_uncropped(url):
                 'uploader_id': channel_handle,
                 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
             })
+
+        channel_badges = self._extract_badges(traverse_obj(data, ('header', ..., 'badges'), get_all=False))
+        if self._has_badge(channel_badges, BadgeType.VERIFIED):
+            info['channel_is_verified'] = True
         # Playlist stats is a text runs array containing [video count, view count, last updated].
         # last updated or (view count and last updated) may be missing.
         playlist_stats = get_first(
@@ -5385,6 +5414,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader': '3Blue1Brown',
             'tags': ['Mathematics'],
             'channel_follower_count': int,
+            'channel_is_verified': True,
         },
     }, {
         'note': 'playlists, singlepage',
@@ -5561,6 +5591,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader_url': 'https://www.youtube.com/@3blue1brown',
             'uploader_id': '@3blue1brown',
             'uploader': '3Blue1Brown',
+            'channel_is_verified': True,
         },
     }, {
         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
@@ -5748,6 +5779,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader_url': 'https://www.youtube.com/@SkyNews',
             'uploader_id': '@SkyNews',
             'uploader': 'Sky News',
+            'channel_is_verified': True,
         },
         'params': {
             'skip_download': True,
@@ -6237,6 +6269,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
                 'uploader': str,
                 'uploader_url': str,
                 'uploader_id': str,
+                'channel_is_verified': bool,  # this will keep changing
             }
         }],
         'params': {'extract_flat': True, 'playlist_items': '1'},
@@ -6272,6 +6305,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
                 'uploader': 'PewDiePie',
                 'uploader_url': 'https://www.youtube.com/@PewDiePie',
                 'uploader_id': '@PewDiePie',
+                'channel_is_verified': True,
             }
         }],
         'params': {'extract_flat': True},
@@ -6290,6 +6324,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader_url': 'https://www.youtube.com/@3blue1brown',
             'uploader_id': '@3blue1brown',
             'uploader': '3Blue1Brown',
+            'channel_is_verified': True,
         },
         'playlist_count': 0,
     }, {
@@ -6324,6 +6359,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'description': 'I make music',
             'channel_url': 'https://www.youtube.com/channel/UCgFwu-j5-xNJml2FtTrrB3A',
             'channel_follower_count': int,
+            'channel_is_verified': True,
         },
         'playlist_mincount': 10,
     }]
@@ -6906,6 +6942,7 @@ class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
                 'uploader_id': '@kurzgesagt',
                 'uploader_url': 'https://www.youtube.com/@kurzgesagt',
                 'uploader': 'Kurzgesagt – In a Nutshell',
+                'channel_is_verified': True,
                 'channel_follower_count': int,
             }
         }],
@@ -7232,6 +7269,7 @@ class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
             'channel': 'さなちゃんねる',
             'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
             'uploader': 'さなちゃんねる',
+            'channel_is_verified': True,
             'heatmap': 'count:100',
         },
         'add_ie': ['Youtube'],

From 44c0d66442b568d9e1359e669d8b029b08a77fa7 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 8 Jun 2023 13:36:09 -0500
Subject: [PATCH 343/871] [extractor/lbry] Extract original quality formats
 (#7257)

Closes #7251
Authored by: bashonly
---
 yt_dlp/extractor/lbry.py | 129 ++++++++++++++++++++++++---------------
 1 file changed, 81 insertions(+), 48 deletions(-)

diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py
index b5def1e07..23d3daf13 100644
--- a/yt_dlp/extractor/lbry.py
+++ b/yt_dlp/extractor/lbry.py
@@ -1,8 +1,8 @@
 import functools
 import json
+import urllib.parse
 
 from .common import InfoExtractor
-from ..compat import compat_str, compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     HEADRequest,
@@ -12,7 +12,10 @@
     int_or_none,
     mimetype2ext,
     parse_qs,
+    traverse_obj,
     try_get,
+    url_or_none,
+    urlhandle_detect_ext,
     urljoin,
 )
 
@@ -52,38 +55,25 @@ def _permanent_url(self, url, claim_name, claim_id):
             '/%s:%s' % (claim_name, claim_id))
 
     def _parse_stream(self, stream, url):
-        stream_value = stream.get('value') or {}
-        stream_type = stream_value.get('stream_type')
-        source = stream_value.get('source') or {}
-        media = stream_value.get(stream_type) or {}
-        signing_channel = stream.get('signing_channel') or {}
-        channel_name = signing_channel.get('name')
-        channel_claim_id = signing_channel.get('claim_id')
-        channel_url = None
-        if channel_name and channel_claim_id:
-            channel_url = self._permanent_url(url, channel_name, channel_claim_id)
+        stream_type = traverse_obj(stream, ('value', 'stream_type', {str}))
+
+        info = traverse_obj(stream, {
+            'title': ('value', 'title', {str}),
+            'thumbnail': ('value', 'thumbnail', 'url', {url_or_none}),
+            'description': ('value', 'description', {str}),
+            'license': ('value', 'license', {str}),
+            'timestamp': ('timestamp', {int_or_none}),
+            'release_timestamp': ('value', 'release_time', {int_or_none}),
+            'tags': ('value', 'tags', ..., {lambda x: x or None}),
+            'duration': ('value', stream_type, 'duration', {int_or_none}),
+            'channel': ('signing_channel', 'value', 'title', {str}),
+            'channel_id': ('signing_channel', 'claim_id', {str}),
+        })
+
+        channel_name = traverse_obj(stream, ('signing_channel', 'name', {str}))
+        if channel_name and info.get('channel_id'):
+            info['channel_url'] = self._permanent_url(url, channel_name, info['channel_id'])
 
-        info = {
-            'thumbnail': try_get(stream_value, lambda x: x['thumbnail']['url'], compat_str),
-            'description': stream_value.get('description'),
-            'license': stream_value.get('license'),
-            'timestamp': int_or_none(stream.get('timestamp')),
-            'release_timestamp': int_or_none(stream_value.get('release_time')),
-            'tags': stream_value.get('tags'),
-            'duration': int_or_none(media.get('duration')),
-            'channel': try_get(signing_channel, lambda x: x['value']['title']),
-            'channel_id': channel_claim_id,
-            'channel_url': channel_url,
-            'ext': determine_ext(source.get('name')) or mimetype2ext(source.get('media_type')),
-            'filesize': int_or_none(source.get('size')),
-        }
-        if stream_type == 'audio':
-            info['vcodec'] = 'none'
-        else:
-            info.update({
-                'width': int_or_none(media.get('width')),
-                'height': int_or_none(media.get('height')),
-            })
         return info
 
 
@@ -186,6 +176,28 @@ class LBRYIE(LBRYBaseIE):
             'license': 'None',
         },
         'params': {'skip_download': True}
+    }, {
+        # original quality format w/higher resolution than HLS formats
+        'url': 'https://odysee.com/@wickedtruths:2/Biotechnological-Invasion-of-Skin-(April-2023):4',
+        'md5': '305b0b3b369bde1b984961f005b67193',
+        'info_dict': {
+            'id': '41fbfe805eb73c8d3012c0c49faa0f563274f634',
+            'ext': 'mp4',
+            'title': 'Biotechnological Invasion of Skin (April 2023)',
+            'description': 'md5:709a2f4c07bd8891cda3a7cc2d6fcf5c',
+            'channel': 'Wicked Truths',
+            'channel_id': '23d2bbf856b0ceed5b1d7c5960bcc72da5a20cb0',
+            'channel_url': 'https://odysee.com/@wickedtruths:23d2bbf856b0ceed5b1d7c5960bcc72da5a20cb0',
+            'timestamp': 1685790036,
+            'upload_date': '20230603',
+            'release_timestamp': 1685617473,
+            'release_date': '20230601',
+            'duration': 1063,
+            'thumbnail': 'https://thumbs.odycdn.com/4e6d39da4df0cfdad45f64e253a15959.webp',
+            'tags': ['smart skin surveillance', 'biotechnology invasion of skin', 'morgellons'],
+            'license': 'None',
+            'protocol': 'https',  # test for direct mp4 download
+        },
     }, {
         'url': 'https://odysee.com/@BrodieRobertson:5/apple-is-tracking-everything-you-do-on:e',
         'only_matching': True,
@@ -221,41 +233,64 @@ def _real_extract(self, url):
             display_id = display_id.split('/', 2)[-1].replace('/', ':')
         else:
             display_id = display_id.replace(':', '#')
-        display_id = compat_urllib_parse_unquote(display_id)
+        display_id = urllib.parse.unquote(display_id)
         uri = 'lbry://' + display_id
         result = self._resolve_url(uri, display_id, 'stream')
         headers = {'Referer': 'https://odysee.com/'}
-        if result['value'].get('stream_type') in self._SUPPORTED_STREAM_TYPES:
+
+        formats = []
+        stream_type = traverse_obj(result, ('value', 'stream_type', {str}))
+
+        if stream_type in self._SUPPORTED_STREAM_TYPES:
             claim_id, is_live = result['claim_id'], False
             streaming_url = self._call_api_proxy(
                 'get', claim_id, {'uri': uri}, 'streaming url')['streaming_url']
+
+            # GET request returns original video/audio file if available
+            ext = urlhandle_detect_ext(self._request_webpage(
+                streaming_url, display_id, 'Checking for original quality', headers=headers))
+            if ext != 'm3u8':
+                formats.append({
+                    'url': streaming_url,
+                    'format_id': 'original',
+                    'quality': 1,
+                    **traverse_obj(result, ('value', {
+                        'ext': ('source', (('name', {determine_ext}), ('media_type', {mimetype2ext}))),
+                        'filesize': ('source', 'size', {int_or_none}),
+                        'width': ('video', 'width', {int_or_none}),
+                        'height': ('video', 'height', {int_or_none}),
+                    }), get_all=False),
+                    'vcodec': 'none' if stream_type == 'audio' else None,
+                })
+
+            # HEAD request returns redirect response to m3u8 URL if available
             final_url = self._request_webpage(
                 HEADRequest(streaming_url), display_id, headers=headers,
                 note='Downloading streaming redirect url info').geturl()
+
         elif result.get('value_type') == 'stream':
             claim_id, is_live = result['signing_channel']['claim_id'], True
             live_data = self._download_json(
                 'https://api.odysee.live/livestream/is_live', claim_id,
                 query={'channel_claim_id': claim_id},
                 note='Downloading livestream JSON metadata')['data']
-            streaming_url = final_url = live_data.get('VideoURL')
+            final_url = live_data.get('VideoURL')
             # Upcoming videos may still give VideoURL
             if not live_data.get('Live'):
-                streaming_url = final_url = None
+                final_url = None
                 self.raise_no_formats('This stream is not live', True, claim_id)
+
         else:
             raise UnsupportedError(url)
 
-        info = self._parse_stream(result, url)
         if determine_ext(final_url) == 'm3u8':
-            info['formats'] = self._extract_m3u8_formats(
-                final_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', live=is_live, headers=headers)
-        else:
-            info['url'] = streaming_url
+            formats.extend(self._extract_m3u8_formats(
+                final_url, display_id, 'mp4', m3u8_id='hls', live=is_live, headers=headers))
+
         return {
-            **info,
+            **self._parse_stream(result, url),
             'id': claim_id,
-            'title': result['value']['title'],
+            'formats': formats,
             'is_live': is_live,
             'http_headers': headers,
         }
@@ -299,14 +334,12 @@ def _fetch_page(self, claim_id, url, params, page):
             if not (stream_claim_name and stream_claim_id):
                 continue
 
-            info = self._parse_stream(item, url)
-            info.update({
+            yield {
+                **self._parse_stream(item, url),
                 '_type': 'url',
                 'id': stream_claim_id,
-                'title': try_get(item, lambda x: x['value']['title']),
                 'url': self._permanent_url(url, stream_claim_name, stream_claim_id),
-            })
-            yield info
+            }
 
     def _real_extract(self, url):
         display_id = self._match_id(url).replace(':', '#')

From d1795f4a6af99c976c9d3ea2dabe5cf4f8965d3c Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 8 Jun 2023 13:47:13 -0500
Subject: [PATCH 344/871] [extractor/twitter] Add login support (#7258)

Closes #6951
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 213 +++++++++++++++++++++++++++++++++---
 1 file changed, 198 insertions(+), 15 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 4624ce503..f854d9c4a 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -3,7 +3,6 @@
 
 from .common import InfoExtractor
 from .periscope import PeriscopeBaseIE, PeriscopeIE
-from ..compat import functools  # isort: split
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse_unquote,
@@ -30,11 +29,67 @@
 
 
 class TwitterBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'twitter'
     _API_BASE = 'https://api.twitter.com/1.1/'
     _GRAPHQL_API_BASE = 'https://twitter.com/i/api/graphql/'
     _BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?(?:twitter\.com|twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid\.onion)/'
     _AUTH = {'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'}
     _guest_token = None
+    _flow_token = None
+
+    _LOGIN_INIT_DATA = json.dumps({
+        'input_flow_data': {
+            'flow_context': {
+                'debug_overrides': {},
+                'start_location': {
+                    'location': 'unknown'
+                }
+            }
+        },
+        'subtask_versions': {
+            'action_list': 2,
+            'alert_dialog': 1,
+            'app_download_cta': 1,
+            'check_logged_in_account': 1,
+            'choice_selection': 3,
+            'contacts_live_sync_permission_prompt': 0,
+            'cta': 7,
+            'email_verification': 2,
+            'end_flow': 1,
+            'enter_date': 1,
+            'enter_email': 2,
+            'enter_password': 5,
+            'enter_phone': 2,
+            'enter_recaptcha': 1,
+            'enter_text': 5,
+            'enter_username': 2,
+            'generic_urt': 3,
+            'in_app_notification': 1,
+            'interest_picker': 3,
+            'js_instrumentation': 1,
+            'menu_dialog': 1,
+            'notifications_permission_prompt': 2,
+            'open_account': 2,
+            'open_home_timeline': 1,
+            'open_link': 1,
+            'phone_verification': 4,
+            'privacy_options': 1,
+            'security_key': 3,
+            'select_avatar': 4,
+            'select_banner': 2,
+            'settings_list': 7,
+            'show_code': 1,
+            'sign_up': 2,
+            'sign_up_review': 4,
+            'tweet_selection_urt': 1,
+            'update_users': 1,
+            'upload_media': 1,
+            'user_recommendations_list': 4,
+            'user_recommendations_urt': 1,
+            'wait_spinner': 3,
+            'web_modal': 1
+        }
+    }, separators=(',', ':')).encode()
 
     def _extract_variant_formats(self, variant, video_id):
         variant_url = variant.get('url')
@@ -86,18 +141,151 @@ def _search_dimensions_in_video_url(a_format, video_url):
                 'height': int(m.group('height')),
             })
 
-    @functools.cached_property
+    @property
     def is_logged_in(self):
         return bool(self._get_cookies(self._API_BASE).get('auth_token'))
 
-    def _call_api(self, path, video_id, query={}, graphql=False):
-        cookies = self._get_cookies(self._API_BASE)
+    def _fetch_guest_token(self, headers, display_id):
+        headers.pop('x-guest-token', None)
+        self._guest_token = traverse_obj(self._download_json(
+            f'{self._API_BASE}guest/activate.json', display_id,
+            'Downloading guest token', data=b'', headers=headers), 'guest_token')
+        if not self._guest_token:
+            raise ExtractorError('Could not retrieve guest token')
+
+    def _set_base_headers(self):
         headers = self._AUTH.copy()
+        csrf_token = try_call(lambda: self._get_cookies(self._API_BASE)['ct0'].value)
+        if csrf_token:
+            headers['x-csrf-token'] = csrf_token
+        return headers
 
-        csrf_cookie = cookies.get('ct0')
-        if csrf_cookie:
-            headers['x-csrf-token'] = csrf_cookie.value
+    def _call_login_api(self, note, headers, query={}, data=None):
+        response = self._download_json(
+            f'{self._API_BASE}onboarding/task.json', None, note,
+            headers=headers, query=query, data=data, expected_status=400)
+        error = traverse_obj(response, ('errors', 0, 'message', {str}))
+        if error:
+            raise ExtractorError(f'Login failed, Twitter API says: {error}', expected=True)
+        elif traverse_obj(response, 'status') != 'success':
+            raise ExtractorError('Login was unsuccessful')
 
+        subtask = traverse_obj(
+            response, ('subtasks', ..., 'subtask_id', {str}), get_all=False)
+        if not subtask:
+            raise ExtractorError('Twitter API did not return next login subtask')
+
+        self._flow_token = response['flow_token']
+
+        return subtask
+
+    def _perform_login(self, username, password):
+        if self.is_logged_in:
+            return
+
+        self._request_webpage('https://twitter.com/', None, 'Requesting cookies')
+        headers = self._set_base_headers()
+        self._fetch_guest_token(headers, None)
+        headers.update({
+            'content-type': 'application/json',
+            'x-guest-token': self._guest_token,
+            'x-twitter-client-language': 'en',
+            'x-twitter-active-user': 'yes',
+            'Referer': 'https://twitter.com/',
+            'Origin': 'https://twitter.com',
+        })
+
+        def build_login_json(*subtask_inputs):
+            return json.dumps({
+                'flow_token': self._flow_token,
+                'subtask_inputs': subtask_inputs
+            }, separators=(',', ':')).encode()
+
+        def input_dict(subtask_id, text):
+            return {
+                'subtask_id': subtask_id,
+                'enter_text': {
+                    'text': text,
+                    'link': 'next_link'
+                }
+            }
+
+        next_subtask = self._call_login_api(
+            'Downloading flow token', headers, query={'flow_name': 'login'}, data=self._LOGIN_INIT_DATA)
+
+        while not self.is_logged_in:
+            if next_subtask == 'LoginJsInstrumentationSubtask':
+                next_subtask = self._call_login_api(
+                    'Submitting JS instrumentation response', headers, data=build_login_json({
+                        'subtask_id': next_subtask,
+                        'js_instrumentation': {
+                            'response': '{}',
+                            'link': 'next_link'
+                        }
+                    }))
+
+            elif next_subtask == 'LoginEnterUserIdentifierSSO':
+                next_subtask = self._call_login_api(
+                    'Submitting username', headers, data=build_login_json({
+                        'subtask_id': next_subtask,
+                        'settings_list': {
+                            'setting_responses': [{
+                                'key': 'user_identifier',
+                                'response_data': {
+                                    'text_data': {
+                                        'result': username
+                                    }
+                                }
+                            }],
+                            'link': 'next_link'
+                        }
+                    }))
+
+            elif next_subtask == 'LoginEnterAlternateIdentifierSubtask':
+                next_subtask = self._call_login_api(
+                    'Submitting alternate identifier', headers,
+                    data=build_login_json(input_dict(next_subtask, self._get_tfa_info(
+                        'one of username, phone number or email that was not used as --username'))))
+
+            elif next_subtask == 'LoginEnterPassword':
+                next_subtask = self._call_login_api(
+                    'Submitting password', headers, data=build_login_json({
+                        'subtask_id': next_subtask,
+                        'enter_password': {
+                            'password': password,
+                            'link': 'next_link'
+                        }
+                    }))
+
+            elif next_subtask == 'AccountDuplicationCheck':
+                next_subtask = self._call_login_api(
+                    'Submitting account duplication check', headers, data=build_login_json({
+                        'subtask_id': next_subtask,
+                        'check_logged_in_account': {
+                            'link': 'AccountDuplicationCheck_false'
+                        }
+                    }))
+
+            elif next_subtask == 'LoginTwoFactorAuthChallenge':
+                next_subtask = self._call_login_api(
+                    'Submitting 2FA token', headers, data=build_login_json(input_dict(
+                        next_subtask, self._get_tfa_info('two-factor authentication token'))))
+
+            elif next_subtask == 'LoginAcid':
+                next_subtask = self._call_login_api(
+                    'Submitting confirmation code', headers, data=build_login_json(input_dict(
+                        next_subtask, self._get_tfa_info('confirmation code sent to your email or phone'))))
+
+            elif next_subtask == 'LoginSuccessSubtask':
+                raise ExtractorError('Twitter API did not grant auth token cookie')
+
+            else:
+                raise ExtractorError(f'Unrecognized subtask ID "{next_subtask}"')
+
+        self.report_login()
+
+    def _call_api(self, path, video_id, query={}, graphql=False):
+        headers = self._set_base_headers()
         if self.is_logged_in:
             headers.update({
                 'x-twitter-auth-type': 'OAuth2Session',
@@ -106,15 +294,10 @@ def _call_api(self, path, video_id, query={}, graphql=False):
             })
 
         for first_attempt in (True, False):
-            if not self.is_logged_in and not self._guest_token:
-                headers.pop('x-guest-token', None)
-                self._guest_token = traverse_obj(self._download_json(
-                    f'{self._API_BASE}guest/activate.json', video_id,
-                    'Downloading guest token', data=b'', headers=headers), 'guest_token')
-            if self._guest_token:
+            if not self.is_logged_in:
+                if not self._guest_token:
+                    self._fetch_guest_token(headers, video_id)
                 headers['x-guest-token'] = self._guest_token
-            elif not self.is_logged_in:
-                raise ExtractorError('Could not retrieve guest token')
 
             allowed_status = {400, 401, 403, 404} if graphql else {403}
             result = self._download_json(

From 4f7b11cc1c1cebf598107e00cd7295588ed484da Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 10 Jun 2023 15:43:22 -0500
Subject: [PATCH 345/871] [extractor/voot] Fix extractor (#7227)

Closes #6715
Authored by: bashonly
---
 yt_dlp/extractor/voot.py | 177 ++++++++++++++++++++++++++-------------
 1 file changed, 119 insertions(+), 58 deletions(-)

diff --git a/yt_dlp/extractor/voot.py b/yt_dlp/extractor/voot.py
index b709b74e2..dd41647aa 100644
--- a/yt_dlp/extractor/voot.py
+++ b/yt_dlp/extractor/voot.py
@@ -1,14 +1,86 @@
+import json
+import time
+import urllib.error
+import uuid
+
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     ExtractorError,
+    float_or_none,
     int_or_none,
+    jwt_decode_hs256,
+    parse_age_limit,
+    traverse_obj,
+    try_call,
     try_get,
-    unified_timestamp,
+    unified_strdate,
 )
 
 
-class VootIE(InfoExtractor):
+class VootBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'voot'
+    _GEO_BYPASS = False
+    _LOGIN_HINT = 'Log in with "-u <email_address> -p <password>", or use "-u token -p <auth_token>" to login with auth token.'
+    _TOKEN = None
+    _EXPIRY = 0
+    _API_HEADERS = {'Origin': 'https://www.voot.com', 'Referer': 'https://www.voot.com/'}
+
+    def _perform_login(self, username, password):
+        if self._TOKEN and self._EXPIRY:
+            return
+
+        if username.lower() == 'token' and try_call(lambda: jwt_decode_hs256(password)):
+            VootBaseIE._TOKEN = password
+            VootBaseIE._EXPIRY = jwt_decode_hs256(password)['exp']
+            self.report_login()
+
+        # Mobile number as username is not supported
+        elif not username.isdigit():
+            check_username = self._download_json(
+                'https://userauth.voot.com/usersV3/v3/checkUser', None, data=json.dumps({
+                    'type': 'email',
+                    'email': username
+                }, separators=(',', ':')).encode(), headers={
+                    **self._API_HEADERS,
+                    'Content-Type': 'application/json;charset=utf-8',
+                }, note='Checking username', expected_status=403)
+            if not traverse_obj(check_username, ('isExist', {bool})):
+                if traverse_obj(check_username, ('status', 'code', {int})) == 9999:
+                    self.raise_geo_restricted(countries=['IN'])
+                raise ExtractorError('Incorrect username', expected=True)
+            auth_token = traverse_obj(self._download_json(
+                'https://userauth.voot.com/usersV3/v3/login', None, data=json.dumps({
+                    'type': 'traditional',
+                    'deviceId': str(uuid.uuid4()),
+                    'deviceBrand': 'PC/MAC',
+                    'data': {
+                        'email': username,
+                        'password': password
+                    }
+                }, separators=(',', ':')).encode(), headers={
+                    **self._API_HEADERS,
+                    'Content-Type': 'application/json;charset=utf-8',
+                }, note='Logging in', expected_status=400), ('data', 'authToken', {dict}))
+            if not auth_token:
+                raise ExtractorError('Incorrect password', expected=True)
+            VootBaseIE._TOKEN = auth_token['accessToken']
+            VootBaseIE._EXPIRY = auth_token['expirationTime']
+
+        else:
+            raise ExtractorError(self._LOGIN_HINT, expected=True)
+
+    def _check_token_expiry(self):
+        if int(time.time()) >= self._EXPIRY:
+            raise ExtractorError('Access token has expired', expected=True)
+
+    def _real_initialize(self):
+        if not self._TOKEN:
+            self.raise_login_required(self._LOGIN_HINT, method=None)
+        self._check_token_expiry()
+
+
+class VootIE(VootBaseIE):
     _VALID_URL = r'''(?x)
                     (?:
                         voot:|
@@ -20,27 +92,25 @@ class VootIE(InfoExtractor):
                      )
                     (?P<id>\d{3,})
                     '''
-    _GEO_COUNTRIES = ['IN']
     _TESTS = [{
         'url': 'https://www.voot.com/shows/ishq-ka-rang-safed/1/360558/is-this-the-end-of-kamini-/441353',
         'info_dict': {
-            'id': '0_8ledb18o',
+            'id': '441353',
             'ext': 'mp4',
-            'title': 'Ishq Ka Rang Safed - Season 01 - Episode 340',
+            'title': 'Is this the end of Kamini?',
             'description': 'md5:06291fbbbc4dcbe21235c40c262507c1',
-            'timestamp': 1472162937,
+            'timestamp': 1472103000,
             'upload_date': '20160825',
             'series': 'Ishq Ka Rang Safed',
             'season_number': 1,
             'episode': 'Is this the end of Kamini?',
             'episode_number': 340,
-            'view_count': int,
-            'like_count': int,
+            'release_date': '20160825',
+            'season': 'Season 1',
+            'age_limit': 13,
+            'duration': 1146.0,
         },
-        'params': {
-            'skip_download': True,
-        },
-        'expected_warnings': ['Failed to download m3u8 information'],
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://www.voot.com/kids/characters/mighty-cat-masked-niyander-e-/400478/school-bag-disappears/440925',
         'only_matching': True,
@@ -55,59 +125,50 @@ class VootIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         media_info = self._download_json(
-            'https://wapi.voot.com/ws/ott/getMediaInfo.json', video_id,
-            query={
-                'platform': 'Web',
-                'pId': 2,
-                'mediaId': video_id,
-            })
+            'https://psapi.voot.com/jio/voot/v1/voot-web/content/query/asset-details', video_id,
+            query={'ids': f'include:{video_id}', 'responseType': 'common'}, headers={'accesstoken': self._TOKEN})
 
-        status_code = try_get(media_info, lambda x: x['status']['code'], int)
-        if status_code != 0:
-            raise ExtractorError(media_info['status']['message'], expected=True)
+        try:
+            m3u8_url = self._download_json(
+                'https://vootapi.media.jio.com/playback/v1/playbackrights', video_id,
+                'Downloading playback JSON', data=b'{}', headers={
+                    **self.geo_verification_headers(),
+                    **self._API_HEADERS,
+                    'Content-Type': 'application/json;charset=utf-8',
+                    'platform': 'androidwebdesktop',
+                    'vootid': video_id,
+                    'voottoken': self._TOKEN,
+                })['m3u8']
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 400:
+                self._check_token_expiry()
+            raise
 
-        media = media_info['assets']
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls')
+        self._remove_duplicate_formats(formats)
 
-        entry_id = media['EntryId']
-        title = media['MediaName']
-        formats = self._extract_m3u8_formats(
-            'https://cdnapisec.kaltura.com/p/1982551/playManifest/pt/https/f/applehttp/t/web/e/' + entry_id,
-            video_id, 'mp4', m3u8_id='hls')
-
-        description, series, season_number, episode, episode_number = [None] * 5
-
-        for meta in try_get(media, lambda x: x['Metas'], list) or []:
-            key, value = meta.get('Key'), meta.get('Value')
-            if not key or not value:
-                continue
-            if key == 'ContentSynopsis':
-                description = value
-            elif key == 'RefSeriesTitle':
-                series = value
-            elif key == 'RefSeriesSeason':
-                season_number = int_or_none(value)
-            elif key == 'EpisodeMainTitle':
-                episode = value
-            elif key == 'EpisodeNo':
-                episode_number = int_or_none(value)
         return {
-            'extractor_key': 'Kaltura',
-            'id': entry_id,
-            'title': title,
-            'description': description,
-            'series': series,
-            'season_number': season_number,
-            'episode': episode,
-            'episode_number': episode_number,
-            'timestamp': unified_timestamp(media.get('CreationDate')),
-            'duration': int_or_none(media.get('Duration')),
-            'view_count': int_or_none(media.get('ViewCounter')),
-            'like_count': int_or_none(media.get('like_counter')),
-            'formats': formats,
+            'id': video_id,
+            # '/_definst_/smil:vod/' m3u8 manifests claim to have 720p+ formats but max out at 480p
+            'formats': traverse_obj(formats, (
+                lambda _, v: '/_definst_/smil:vod/' not in v['url'] or v['height'] <= 480)),
+            'http_headers': self._API_HEADERS,
+            **traverse_obj(media_info, ('result', 0, {
+                'title': ('fullTitle', {str}),
+                'description': ('fullSynopsis', {str}),
+                'series': ('showName', {str}),
+                'season_number': ('season', {int_or_none}),
+                'episode': ('fullTitle', {str}),
+                'episode_number': ('episode', {int_or_none}),
+                'timestamp': ('uploadTime', {int_or_none}),
+                'release_date': ('telecastDate', {unified_strdate}),
+                'age_limit': ('ageNemonic', {parse_age_limit}),
+                'duration': ('duration', {float_or_none}),
+            })),
         }
 
 
-class VootSeriesIE(InfoExtractor):
+class VootSeriesIE(VootBaseIE):
     _VALID_URL = r'https?://(?:www\.)?voot\.com/shows/[^/]+/(?P<id>\d{3,})'
     _TESTS = [{
         'url': 'https://www.voot.com/shows/chakravartin-ashoka-samrat/100002',

From b4a252fba81f53631c07ca40ce7583f5d19a8a36 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 10 Jun 2023 17:49:12 -0500
Subject: [PATCH 346/871] [jsinterp] Fix division (#7279)

* Fixes nsig decryption for Youtube JS player `8c7583ff`

Authored by: bashonly
---
 test/test_jsinterp.py          | 7 +++++++
 test/test_youtube_signature.py | 4 ++++
 yt_dlp/jsinterp.py             | 2 +-
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 4d44e6efe..b01477e6f 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -28,6 +28,13 @@ def test_basic(self):
     def test_calc(self):
         self._test('function f(a){return 2*a+1;}', 7, args=[3])
 
+    def test_div(self):
+        jsi = JSInterpreter('function f(a, b){return a / b;}')
+        self.assertTrue(math.isnan(jsi.call_function('f', 0, 0)))
+        self.assertTrue(math.isnan(jsi.call_function('f', JS_Undefined, 1)))
+        self.assertTrue(math.isinf(jsi.call_function('f', 2, 0)))
+        self.assertEqual(jsi.call_function('f', 0, 3), 0)
+
     def test_empty_return(self):
         self._test('function f(){return; y()}', None)
 
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 13120d97f..01f09de88 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -150,6 +150,10 @@
         'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js',
         'aCi3iElgd2kq0bxVbQ', 'QX1y8jGb2IbZ0w',
     ),
+    (
+        'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js',
+        '1wWCVpRR96eAmMI87L', 'KSkWAVv1ZQxC3A',
+    ),
 ]
 
 
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index 7c7940efd..d6d555733 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -44,7 +44,7 @@ def wrapped(a, b):
 
 
 def _js_div(a, b):
-    if JS_Undefined in (a, b) or not (a and b):
+    if JS_Undefined in (a, b) or not (a or b):
         return float('nan')
     return (a or 0) / b if b else float('inf')
 

From f8ae441501596733e2b967430471643a1d7cacb8 Mon Sep 17 00:00:00 2001
From: DataGhost <site.github@dataghost.com>
Date: Sun, 11 Jun 2023 17:17:26 +0200
Subject: [PATCH 347/871] [extractor/Dumpert] Fix m3u8 and support new URL
 pattern (#6091)

Authored by: DataGhost, pukkandan
Closes #5032
---
 yt_dlp/extractor/dumpert.py | 49 +++++++++++++++++++++++++++++++------
 1 file changed, 42 insertions(+), 7 deletions(-)
 mode change 100644 => 100755 yt_dlp/extractor/dumpert.py

diff --git a/yt_dlp/extractor/dumpert.py b/yt_dlp/extractor/dumpert.py
old mode 100644
new mode 100755
index 010c2d092..0cf84263c
--- a/yt_dlp/extractor/dumpert.py
+++ b/yt_dlp/extractor/dumpert.py
@@ -1,12 +1,17 @@
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     int_or_none,
     qualities,
 )
 
 
 class DumpertIE(InfoExtractor):
-    _VALID_URL = r'(?P<protocol>https?)://(?:(?:www|legacy)\.)?dumpert\.nl/(?:mediabase|embed|item)/(?P<id>[0-9]+[/_][0-9a-zA-Z]+)'
+    _VALID_URL = r'''(?x)
+        (?P<protocol>https?)://(?:(?:www|legacy)\.)?dumpert\.nl(?:
+            /(?:mediabase|embed|item)/|
+            (?:/toppers|/latest|/?)\?selectedId=
+        )(?P<id>[0-9]+[/_][0-9a-zA-Z]+)'''
     _TESTS = [{
         'url': 'https://www.dumpert.nl/item/6646981_951bc60f',
         'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
@@ -16,6 +21,9 @@ class DumpertIE(InfoExtractor):
             'title': 'Ik heb nieuws voor je',
             'description': 'Niet schrikken hoor',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 9,
+            'view_count': int,
+            'like_count': int,
         }
     }, {
         'url': 'https://www.dumpert.nl/embed/6675421_dc440fe7',
@@ -26,6 +34,28 @@ class DumpertIE(InfoExtractor):
     }, {
         'url': 'http://legacy.dumpert.nl/embed/6675421/dc440fe7',
         'only_matching': True,
+    }, {
+        'url': 'https://www.dumpert.nl/item/100031688_b317a185',
+        'info_dict': {
+            'id': '100031688/b317a185',
+            'ext': 'mp4',
+            'title': 'Epic schijnbeweging',
+            'description': '<p>Die zag je niet eh</p>',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
+            'duration': 12,
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {'skip_download': 'm3u8'}
+    }, {
+        'url': 'https://www.dumpert.nl/toppers?selectedId=100031688_b317a185',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.dumpert.nl/latest?selectedId=100031688_b317a185',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.dumpert.nl/?selectedId=100031688_b317a185',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -36,18 +66,23 @@ def _real_extract(self, url):
         title = item['title']
         media = next(m for m in item['media'] if m.get('mediatype') == 'VIDEO')
 
-        quality = qualities(['flv', 'mobile', 'tablet', '720p'])
+        quality = qualities(['flv', 'mobile', 'tablet', '720p', '1080p'])
         formats = []
         for variant in media.get('variants', []):
             uri = variant.get('uri')
             if not uri:
                 continue
             version = variant.get('version')
-            formats.append({
-                'url': uri,
-                'format_id': version,
-                'quality': quality(version),
-            })
+            preference = quality(version)
+            if determine_ext(uri) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    uri, video_id, 'mp4', m3u8_id=version, quality=preference))
+            else:
+                formats.append({
+                    'url': uri,
+                    'format_id': version,
+                    'quality': preference,
+                })
 
         thumbnails = []
         stills = item.get('stills') or {}

From 1a2eb5bda51d8b7a78a65acebf72a0dcf9da196b Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 11 Jun 2023 12:06:34 -0500
Subject: [PATCH 348/871] [extractor/odnoklassniki] Fix formats extraction
 (#7217)

Closes #2959, Closes #4462, Closes #7201
Authored by: bashonly
---
 yt_dlp/extractor/odnoklassniki.py | 56 ++++++++++++++++++++++++-------
 1 file changed, 43 insertions(+), 13 deletions(-)

diff --git a/yt_dlp/extractor/odnoklassniki.py b/yt_dlp/extractor/odnoklassniki.py
index 4b73eed37..0d0ad0bb8 100644
--- a/yt_dlp/extractor/odnoklassniki.py
+++ b/yt_dlp/extractor/odnoklassniki.py
@@ -1,3 +1,5 @@
+import urllib.parse
+
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_fromstring,
@@ -7,6 +9,7 @@
 )
 from ..utils import (
     ExtractorError,
+    HEADRequest,
     float_or_none,
     int_or_none,
     qualities,
@@ -15,6 +18,7 @@
     unescapeHTML,
     unified_strdate,
     unsmuggle_url,
+    url_or_none,
     urlencode_postdata,
 )
 
@@ -41,7 +45,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'ext': 'mp4',
             'timestamp': 1545580896,
             'view_count': int,
-            'thumbnail': 'https://coub-attachments.akamaized.net/coub_storage/coub/simple/cw_image/c5ac87553bd/608e806a1239c210ab692/1545580913_00026.jpg',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'title': 'Народная забава',
             'uploader': 'Nevata',
             'upload_date': '20181223',
@@ -65,13 +69,14 @@ class OdnoklassnikiIE(InfoExtractor):
             'title': str,
             'uploader': str,
         },
+        'skip': 'vk extractor error',
     }, {
-        # metadata in JSON
+        # metadata in JSON, webm_dash with Firefox UA
         'url': 'http://ok.ru/video/20079905452',
-        'md5': '5d2b64756e2af296e3b383a0bc02a6aa',
+        'md5': '8f477d8931c531374a3e36daec617b2c',
         'info_dict': {
             'id': '20079905452',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'Культура меняет нас (прекрасный ролик!))',
             'thumbnail': str,
             'duration': 100,
@@ -81,10 +86,14 @@ class OdnoklassnikiIE(InfoExtractor):
             'like_count': int,
             'age_limit': 0,
         },
+        'params': {
+            'format': 'bv[ext=webm]',
+            'http_headers': {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0'},
+        },
     }, {
         # metadataUrl
         'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
-        'md5': 'f8c951122516af72e6e6ffdd3c41103b',
+        'md5': '2bae2f58eefe1b3d26f3926c4a64d2f3',
         'info_dict': {
             'id': '63567059965189-0',
             'ext': 'mp4',
@@ -98,10 +107,11 @@ class OdnoklassnikiIE(InfoExtractor):
             'age_limit': 0,
             'start_time': 5,
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
         'url': 'https://ok.ru/video/3952212382174',
-        'md5': '91749d0bd20763a28d083fa335bbd37a',
+        'md5': '5fb5f83ce16cb212d6bf887282b5da53',
         'info_dict': {
             'id': '5axVgHHDBvU',
             'ext': 'mp4',
@@ -116,7 +126,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'live_status': 'not_live',
             'view_count': int,
             'thumbnail': 'https://i.mycdn.me/i?r=AEHujHvw2RjEbemUCNEorZbxYpb_p_9AcN2FmGik64Krkcmz37YtlY093oAM5-HIEAt7Zi9s0CiBOSDmbngC-I-k&fn=external_8',
-            'uploader_url': 'http://www.youtube.com/user/MrKewlkid94',
+            'uploader_url': 'https://www.youtube.com/@MrKewlkid94',
             'channel_follower_count': int,
             'tags': ['youtube-dl', 'youtube playlists', 'download videos', 'download audio'],
             'channel_id': 'UCVGtvURtEURYHtJFUegdSug',
@@ -145,7 +155,6 @@ class OdnoklassnikiIE(InfoExtractor):
         },
         'skip': 'Video has not been found',
     }, {
-        # TODO: HTTP Error 400: Bad Request, it only works if there's no cookies when downloading
         'note': 'Only available in mobile webpage',
         'url': 'https://m.ok.ru/video/2361249957145',
         'info_dict': {
@@ -153,8 +162,8 @@ class OdnoklassnikiIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Быковское крещение',
             'duration': 3038.181,
+            'thumbnail': r're:^https?://i\.mycdn\.me/videoPreview\?.+',
         },
-        'skip': 'HTTP Error 400',
     }, {
         'note': 'subtitles',
         'url': 'https://ok.ru/video/4249587550747',
@@ -226,6 +235,14 @@ class OdnoklassnikiIE(InfoExtractor):
         'skip': 'Site no longer embeds',
     }]
 
+    def _clear_cookies(self, cdn_url):
+        # Direct http downloads will fail if CDN cookies are set
+        # so we need to reset them after each format extraction
+        if self._get_cookies('https://notarealsubdomain.mycdn.me/'):
+            self.cookiejar.clear(domain='.mycdn.me')
+        if self._get_cookies(cdn_url):
+            self.cookiejar.clear(domain=urllib.parse.urlparse(cdn_url).hostname)
+
     @classmethod
     def _extract_embed_urls(cls, url, webpage):
         for x in super()._extract_embed_urls(url, webpage):
@@ -364,14 +381,22 @@ def _extract_desktop(self, url):
         formats = [{
             'url': f['url'],
             'ext': 'mp4',
-            'format_id': f['name'],
-        } for f in metadata['videos']]
+            'format_id': f.get('name'),
+        } for f in traverse_obj(metadata, ('videos', lambda _, v: url_or_none(v['url'])))]
 
-        m3u8_url = metadata.get('hlsManifestUrl')
+        m3u8_url = traverse_obj(metadata, 'hlsManifestUrl', 'ondemandHls')
         if m3u8_url:
             formats.extend(self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', 'm3u8_native',
                 m3u8_id='hls', fatal=False))
+            self._clear_cookies(m3u8_url)
+
+        for mpd_id, mpd_key in [('dash', 'ondemandDash'), ('webm', 'metadataWebmUrl')]:
+            mpd_url = metadata.get(mpd_key)
+            if mpd_url:
+                formats.extend(self._extract_mpd_formats(
+                    mpd_url, video_id, mpd_id=mpd_id, fatal=False))
+                self._clear_cookies(mpd_url)
 
         dash_manifest = metadata.get('metadataEmbedded')
         if dash_manifest:
@@ -390,6 +415,7 @@ def _extract_desktop(self, url):
         if m3u8_url:
             formats.extend(self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+            self._clear_cookies(m3u8_url)
         rtmp_url = metadata.get('rtmpUrl')
         if rtmp_url:
             formats.append({
@@ -423,6 +449,10 @@ def _extract_mobile(self, url):
             r'data-video="(.+?)"', webpage, 'json data')
         json_data = self._parse_json(unescapeHTML(json_data), video_id) or {}
 
+        redirect_url = self._request_webpage(HEADRequest(
+            json_data['videoSrc']), video_id, 'Requesting download URL').geturl()
+        self._clear_cookies(redirect_url)
+
         return {
             'id': video_id,
             'title': json_data.get('videoName'),
@@ -430,7 +460,7 @@ def _extract_mobile(self, url):
             'thumbnail': json_data.get('videoPosterSrc'),
             'formats': [{
                 'format_id': 'mobile',
-                'url': json_data.get('videoSrc'),
+                'url': redirect_url,
                 'ext': 'mp4',
             }]
         }

From 9d7fde89a40360396f0baa2ee8bf507f92108b32 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 11 Jun 2023 12:15:05 -0500
Subject: [PATCH 349/871] [extractor/zee5] Fix extraction of new content
 (#7280)

Authored by: bashonly
---
 yt_dlp/extractor/zee5.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/extractor/zee5.py b/yt_dlp/extractor/zee5.py
index a64eb9ed0..b4734cc8f 100644
--- a/yt_dlp/extractor/zee5.py
+++ b/yt_dlp/extractor/zee5.py
@@ -1,14 +1,16 @@
 import json
-import random
-import string
+import time
+import uuid
 
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     int_or_none,
+    jwt_decode_hs256,
     parse_age_limit,
     str_or_none,
+    try_call,
     try_get,
     unified_strdate,
     unified_timestamp,
@@ -94,12 +96,12 @@ class Zee5IE(InfoExtractor):
         'url': 'https://www.zee5.com/music-videos/details/adhento-gaani-vunnapaatuga-jersey-nani-shraddha-srinath/0-0-56973',
         'only_matching': True
     }]
-    _DETAIL_API_URL = 'https://spapi.zee5.com/singlePlayback/getDetails/secure?content_id={}&device_id={}&platform_name=desktop_web&country=IN&check_parental_control=false'
-    _DEVICE_ID = ''.join(random.choices(string.ascii_letters + string.digits, k=20)).ljust(32, '0')
+    _DEVICE_ID = str(uuid.uuid4())
     _USER_TOKEN = None
     _LOGIN_HINT = 'Use "--username <mobile_number>" to login using otp or "--username token" and "--password <user_token>" to login using user token.'
     _NETRC_MACHINE = 'zee5'
     _GEO_COUNTRIES = ['IN']
+    _USER_COUNTRY = None
 
     def _perform_login(self, username, password):
         if len(username) == 10 and username.isdigit() and self._USER_TOKEN is None:
@@ -118,11 +120,16 @@ def _perform_login(self, username, password):
             self._USER_TOKEN = otp_verify_json.get('token')
             if not self._USER_TOKEN:
                 raise ExtractorError(otp_request_json['message'], expected=True)
-        elif username.lower() == 'token' and len(password) > 1198:
+        elif username.lower() == 'token' and try_call(lambda: jwt_decode_hs256(password)):
             self._USER_TOKEN = password
         else:
             raise ExtractorError(self._LOGIN_HINT, expected=True)
 
+        token = jwt_decode_hs256(self._USER_TOKEN)
+        if token.get('exp', 0) <= int(time.time()):
+            raise ExtractorError('User token has expired', expected=True)
+        self._USER_COUNTRY = token.get('current_country')
+
     def _real_extract(self, url):
         video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
         access_token_request = self._download_json(
@@ -137,8 +144,13 @@ def _real_extract(self, url):
             data['X-Z5-Guest-Token'] = self._DEVICE_ID
 
         json_data = self._download_json(
-            self._DETAIL_API_URL.format(video_id, self._DEVICE_ID),
-            video_id, headers={'content-type': 'application/json'}, data=json.dumps(data).encode('utf-8'))
+            'https://spapi.zee5.com/singlePlayback/getDetails/secure', video_id, query={
+                'content_id': video_id,
+                'device_id': self._DEVICE_ID,
+                'platform_name': 'desktop_web',
+                'country': self._USER_COUNTRY or self.get_param('geo_bypass_country') or 'IN',
+                'check_parental_control': False,
+            }, headers={'content-type': 'application/json'}, data=json.dumps(data).encode('utf-8'))
         asset_data = json_data['assetDetails']
         show_data = json_data.get('showDetails', {})
         if 'premium' in asset_data['business_type']:

From ab6057ec80aa75db6303b8206916d00c376c622c Mon Sep 17 00:00:00 2001
From: puc9 <51006296+puc9@users.noreply.github.com>
Date: Sun, 11 Jun 2023 11:57:59 -0700
Subject: [PATCH 350/871] [extractor/tiktok] Fix resolution extraction (#7237)

Authored by: puc9
---
 yt_dlp/extractor/tiktok.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index 49035e971..9c6d74007 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -218,8 +218,8 @@ def mp3_meta(url):
         def extract_addr(addr, add_meta={}):
             parsed_meta, res = parse_url_key(addr.get('url_key', ''))
             if res:
-                known_resolutions.setdefault(res, {}).setdefault('height', add_meta.get('height'))
-                known_resolutions[res].setdefault('width', add_meta.get('width'))
+                known_resolutions.setdefault(res, {}).setdefault('height', add_meta.get('height') or addr.get('height'))
+                known_resolutions[res].setdefault('width', add_meta.get('width') or addr.get('width'))
                 parsed_meta.update(known_resolutions.get(res, {}))
                 add_meta.setdefault('height', int_or_none(res[:-1]))
             return [{

From 8790ea7b2536332777bce68590386b1aa935fac7 Mon Sep 17 00:00:00 2001
From: linsui <36977733+linsui@users.noreply.github.com>
Date: Mon, 12 Jun 2023 08:02:50 +0000
Subject: [PATCH 351/871] [extractor/ximalaya] Sort playlist entries (#7292)

Authored by: linsui
---
 yt_dlp/extractor/ximalaya.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/ximalaya.py b/yt_dlp/extractor/ximalaya.py
index ff18ba697..3d5e6cf90 100644
--- a/yt_dlp/extractor/ximalaya.py
+++ b/yt_dlp/extractor/ximalaya.py
@@ -158,7 +158,7 @@ def _fetch_page(self, playlist_id, page_idx):
         return self._download_json(
             'https://www.ximalaya.com/revision/album/v1/getTracksList',
             playlist_id, note=f'Downloading tracks list page {page_idx}',
-            query={'albumId': playlist_id, 'pageNum': page_idx, 'sort': 1})['data']
+            query={'albumId': playlist_id, 'pageNum': page_idx})['data']
 
     def _get_entries(self, page_data):
         for e in page_data['tracks']:

From 345b4c0aedd9d19898ce00d5cef35fe0d277a052 Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Mon, 12 Jun 2023 14:12:09 -0400
Subject: [PATCH 352/871] [extractor/zaiko] Add extractor (#7254)

Closes #7196
Authored by: c-basalt
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/zaiko.py       | 92 +++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 yt_dlp/extractor/zaiko.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index f54024211..921b7dee9 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2441,6 +2441,7 @@
 from .youporn import YouPornIE
 from .yourporn import YourPornIE
 from .yourupload import YourUploadIE
+from .zaiko import ZaikoIE
 from .zapiks import ZapiksIE
 from .zattoo import (
     BBVTVIE,
diff --git a/yt_dlp/extractor/zaiko.py b/yt_dlp/extractor/zaiko.py
new file mode 100644
index 000000000..59fc64c5a
--- /dev/null
+++ b/yt_dlp/extractor/zaiko.py
@@ -0,0 +1,92 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    extract_attributes,
+    int_or_none,
+    str_or_none,
+    traverse_obj,
+    unescapeHTML,
+    url_or_none,
+)
+
+
+class ZaikoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[\w-]+\.)?zaiko\.io/event/(?P<id>\d+)/stream(?:/\d+)+'
+    _TESTS = [{
+        'url': 'https://zaiko.io/event/324868/stream/20571/20571',
+        'info_dict': {
+            'id': '324868',
+            'ext': 'mp4',
+            'title': 'ZAIKO STREAMING TEST',
+            'alt_title': '[VOD] ZAIKO STREAMING TEST_20210603(Do Not Delete)',
+            'uploader_id': '454',
+            'uploader': 'ZAIKO ZERO',
+            'release_timestamp': 1583809200,
+            'thumbnail': r're:https://[a-z0-9]+.cloudfront.net/[a-z0-9_]+/[a-z0-9_]+',
+            'release_date': '20200310',
+            'categories': ['Tech House'],
+            'live_status': 'was_live',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _parse_vue_element_attr(self, name, string, video_id):
+        page_elem = self._search_regex(rf'(<{name}[^>]+>)', string, name)
+        attrs = {}
+        for key, value in extract_attributes(page_elem).items():
+            if key.startswith(':'):
+                attrs[key[1:]] = self._parse_json(
+                    value, video_id, transform_source=unescapeHTML, fatal=False)
+        return attrs
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage, urlh = self._download_webpage_handle(url, video_id)
+        final_url = urlh.geturl()
+        if 'zaiko.io/login' in final_url:
+            self.raise_login_required()
+        elif '/_buy/' in final_url:
+            raise ExtractorError('Your account does not have tickets to this event', expected=True)
+        stream_meta = self._parse_vue_element_attr('stream-page', webpage, video_id)
+
+        player_page = self._download_webpage(
+            stream_meta['stream-access']['video_source'], video_id,
+            'Downloading player page', headers={'referer': 'https://zaiko.io/'})
+        player_meta = self._parse_vue_element_attr('player', player_page, video_id)
+        status = traverse_obj(player_meta, ('initial_event_info', 'status', {str}))
+        live_status, msg, expected = {
+            'vod': ('was_live', 'No VOD stream URL was found', False),
+            'archiving': ('post_live', 'Event VOD is still being processed', True),
+            'deleting': ('post_live', 'This event has ended', True),
+            'deleted': ('post_live', 'This event has ended', True),
+            'error': ('post_live', 'This event has ended', True),
+            'disconnected': ('post_live', 'Stream has been disconnected', True),
+            'live_to_disconnected': ('post_live', 'Stream has been disconnected', True),
+            'live': ('is_live', 'No livestream URL found was found', False),
+            'waiting': ('is_upcoming', 'Live event has not yet started', True),
+            'cancelled': ('not_live', 'Event has been cancelled', True),
+        }.get(status) or ('not_live', f'Unknown event status "{status}"', False)
+
+        stream_url = traverse_obj(player_meta, ('initial_event_info', 'endpoint', {url_or_none}))
+        formats = self._extract_m3u8_formats(
+            stream_url, video_id, live=True, fatal=False) if stream_url else []
+        if not formats:
+            self.raise_no_formats(msg, expected=expected)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'live_status': live_status,
+            **traverse_obj(stream_meta, {
+                'title': ('event', 'name', {str}),
+                'uploader': ('profile', 'name', {str}),
+                'uploader_id': ('profile', 'id', {str_or_none}),
+                'release_timestamp': ('stream', 'start', 'timestamp', {int_or_none}),
+                'categories': ('event', 'genres', ..., {lambda x: x or None}),
+            }),
+            **traverse_obj(player_meta, ('initial_event_info', {
+                'alt_title': ('title', {str}),
+                'thumbnail': ('poster_url', {url_or_none}),
+            })),
+        }

From cab94a0cd8b6d3fffed5a6faff030274adbed182 Mon Sep 17 00:00:00 2001
From: Cyberes <64224601+Cyberes@users.noreply.github.com>
Date: Mon, 12 Jun 2023 21:23:17 -0600
Subject: [PATCH 353/871] [extractor/funker530] Add extractor (#7291)

Authored by: Cyberes
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/funker530.py   | 79 +++++++++++++++++++++++++++++++++
 yt_dlp/extractor/rumble.py      |  2 +-
 3 files changed, 81 insertions(+), 1 deletion(-)
 create mode 100644 yt_dlp/extractor/funker530.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 921b7dee9..69c7a9e90 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -664,6 +664,7 @@
     FunimationShowIE,
 )
 from .funk import FunkIE
+from .funker530 import Funker530IE
 from .fusion import FusionIE
 from .fuyintv import FuyinTVIE
 from .gab import (
diff --git a/yt_dlp/extractor/funker530.py b/yt_dlp/extractor/funker530.py
new file mode 100644
index 000000000..ba5ab7d4e
--- /dev/null
+++ b/yt_dlp/extractor/funker530.py
@@ -0,0 +1,79 @@
+from .common import InfoExtractor
+from .rumble import RumbleEmbedIE
+from .youtube import YoutubeIE
+from ..utils import ExtractorError, clean_html, get_element_by_class, strip_or_none
+
+
+class Funker530IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?funker530\.com/video/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://funker530.com/video/azov-patrol-caught-in-open-under-automatic-grenade-launcher-fire/',
+        'md5': '085f50fea27523a388bbc22e123e09c8',
+        'info_dict': {
+            'id': 'v2qbmu4',
+            'ext': 'mp4',
+            'title': 'Azov Patrol Caught In Open Under Automatic Grenade Launcher Fire',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'Funker530',
+            'channel': 'Funker530',
+            'channel_url': 'https://rumble.com/c/c-1199543',
+            'width': 1280,
+            'height': 720,
+            'fps': 25,
+            'duration': 27,
+            'upload_date': '20230608',
+            'timestamp': 1686241321,
+            'live_status': 'not_live',
+            'description': 'md5:bea2e1f458095414e04b5ac189c2f980',
+        }
+    }, {
+        'url': 'https://funker530.com/video/my-friends-joined-the-russians-civdiv/',
+        'md5': 'a42c2933391210662e93e867d7124b70',
+        'info_dict': {
+            'id': 'k-pk4bOvoac',
+            'ext': 'mp4',
+            'view_count': int,
+            'channel': 'Civ Div',
+            'comment_count': int,
+            'channel_follower_count': int,
+            'thumbnail': 'https://i.ytimg.com/vi/k-pk4bOvoac/maxresdefault.jpg',
+            'uploader_id': '@CivDiv',
+            'duration': 357,
+            'channel_url': 'https://www.youtube.com/channel/UCgsCiwJ88up-YyMHo7hL5-A',
+            'tags': [],
+            'uploader_url': 'https://www.youtube.com/@CivDiv',
+            'channel_id': 'UCgsCiwJ88up-YyMHo7hL5-A',
+            'like_count': int,
+            'description': 'md5:aef75ec3f59c07a0e39400f609b24429',
+            'live_status': 'not_live',
+            'age_limit': 0,
+            'uploader': 'Civ Div',
+            'categories': ['People & Blogs'],
+            'title': 'My “Friends” joined the Russians.',
+            'availability': 'public',
+            'upload_date': '20230608',
+            'playable_in_embed': True,
+            'heatmap': 'count:100',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        rumble_url = list(RumbleEmbedIE._extract_embed_urls(url, webpage))
+        if rumble_url:
+            info = {'url': rumble_url[0], 'ie_key': RumbleEmbedIE.ie_key()}
+        else:
+            youtube_url = list(YoutubeIE._extract_embed_urls(url, webpage))
+            if youtube_url:
+                info = {'url': youtube_url[0], 'ie_key': YoutubeIE.ie_key()}
+        if not info:
+            raise ExtractorError('No videos found on webpage', expected=True)
+
+        return {
+            **info,
+            '_type': 'url_transparent',
+            'description': strip_or_none(self._search_regex(
+                r'(?s)(.+)About the Author', clean_html(get_element_by_class('video-desc-paragraph', webpage)),
+                'description', default=None))
+        }
diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py
index 98f660f8b..82f3f0f8c 100644
--- a/yt_dlp/extractor/rumble.py
+++ b/yt_dlp/extractor/rumble.py
@@ -144,7 +144,7 @@ def _extract_embed_urls(cls, url, webpage):
         if embeds:
             return embeds
         return [f'https://rumble.com/embed/{mobj.group("id")}' for mobj in re.finditer(
-            r'<script>\s*Rumble\(\s*"play"\s*,\s*{\s*[\'"]video[\'"]\s*:\s*[\'"](?P<id>[0-9a-z]+)[\'"]', webpage)]
+            r'<script>[^<]*\bRumble\(\s*"play"\s*,\s*{\s*[\'"]?video[\'"]?\s*:\s*[\'"](?P<id>[0-9a-z]+)[\'"]', webpage)]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c8561c6d03f025268d6d3972abeb47987c8d7cbb Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Tue, 13 Jun 2023 15:49:18 -0500
Subject: [PATCH 354/871] [extractor/wrestleuniverse] Fix cookies support

Closes #7298
Authored by: bashonly
---
 yt_dlp/extractor/wrestleuniverse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/wrestleuniverse.py b/yt_dlp/extractor/wrestleuniverse.py
index 946edf20a..b12b0f0a9 100644
--- a/yt_dlp/extractor/wrestleuniverse.py
+++ b/yt_dlp/extractor/wrestleuniverse.py
@@ -41,7 +41,7 @@ def _TOKEN(self):
             token = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
             if not token and not self._REFRESH_TOKEN:
                 self.raise_login_required()
-            self._REAL_TOKEN = token
+            self._TOKEN = token
 
         if not self._REAL_TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
             if not self._REFRESH_TOKEN:

From 7bcd4813215ac98daa4949af2ffc677c78307a38 Mon Sep 17 00:00:00 2001
From: hoaluvn <hoaluvn@users.noreply.github.com>
Date: Wed, 14 Jun 2023 17:52:17 +0200
Subject: [PATCH 355/871] [extractor/urplay] Extract all subtitles (#7309)

Authored by: hoaluvn
---
 yt_dlp/extractor/urplay.py | 25 +++++++++++++------------
 yt_dlp/utils/_utils.py     |  1 +
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/urplay.py b/yt_dlp/extractor/urplay.py
index 5d69dadd6..7f97fc95f 100644
--- a/yt_dlp/extractor/urplay.py
+++ b/yt_dlp/extractor/urplay.py
@@ -112,18 +112,19 @@ def parse_lang_code(code):
                 lang = ISO639Utils.short2long(lang)
             return lang or None
 
-        for k, v in (urplayer_data['streamingInfo'].get('sweComplete') or {}).items():
-            if (k in ('sd', 'hd') or not isinstance(v, dict)):
-                continue
-            lang, sttl_url = (v.get(kk) for kk in ('language', 'location', ))
-            if not sttl_url:
-                continue
-            lang = parse_lang_code(lang)
-            if not lang:
-                continue
-            sttl = subtitles.get(lang) or []
-            sttl.append({'ext': k, 'url': sttl_url, })
-            subtitles[lang] = sttl
+        for stream in urplayer_data['streamingInfo'].values():
+            for k, v in stream.items():
+                if (k in ('sd', 'hd') or not isinstance(v, dict)):
+                    continue
+                lang, sttl_url = (v.get(kk) for kk in ('language', 'location', ))
+                if not sttl_url:
+                    continue
+                lang = parse_lang_code(lang)
+                if not lang:
+                    continue
+                sttl = subtitles.get(lang) or []
+                sttl.append({'ext': k, 'url': sttl_url, })
+                subtitles[lang] = sttl
 
         image = urplayer_data.get('image') or {}
         thumbnails = []
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 4179d58c1..646210116 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -4147,6 +4147,7 @@ class ISO639Utils:
         'or': 'ori',
         'os': 'oss',
         'pa': 'pan',
+        'pe': 'per',
         'pi': 'pli',
         'pl': 'pol',
         'ps': 'pus',

From 6daaf21092888beff11b807cd46f832f1f9c46a0 Mon Sep 17 00:00:00 2001
From: RjY <rjy@users.sourceforge.net>
Date: Wed, 14 Jun 2023 19:40:06 +0100
Subject: [PATCH 356/871] [extractor/discogs] Add extractor (#6624)

Authored by: rjy
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/discogs.py     | 35 +++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 yt_dlp/extractor/discogs.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 69c7a9e90..e4fd944e7 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -520,6 +520,7 @@
     DeuxMNewsIE
 )
 from .digitalconcerthall import DigitalConcertHallIE
+from .discogs import DiscogsReleasePlaylistIE
 from .discovery import DiscoveryIE
 from .disney import DisneyIE
 from .dispeak import DigitallySpeakingIE
diff --git a/yt_dlp/extractor/discogs.py b/yt_dlp/extractor/discogs.py
new file mode 100644
index 000000000..048c62288
--- /dev/null
+++ b/yt_dlp/extractor/discogs.py
@@ -0,0 +1,35 @@
+from .common import InfoExtractor
+from .youtube import YoutubeIE
+from ..utils import traverse_obj
+
+
+class DiscogsReleasePlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?discogs\.com/(?P<type>release|master)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.discogs.com/release/1-The-Persuader-Stockholm',
+        'info_dict': {
+            'id': 'release1',
+            'title': 'Stockholm',
+        },
+        'playlist_mincount': 7,
+    }, {
+        'url': 'https://www.discogs.com/master/113-Vince-Watson-Moments-In-Time',
+        'info_dict': {
+            'id': 'master113',
+            'title': 'Moments In Time',
+        },
+        'playlist_mincount': 53,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id, playlist_type = self._match_valid_url(url).group('id', 'type')
+
+        display_id = f'{playlist_type}{playlist_id}'
+        response = self._download_json(
+            f'https://api.discogs.com/{playlist_type}s/{playlist_id}', display_id)
+
+        entries = [
+            self.url_result(video['uri'], YoutubeIE, video_title=video.get('title'))
+            for video in traverse_obj(response, ('videos', lambda _, v: YoutubeIE.suitable(v['uri'])))]
+
+        return self.playlist_result(entries, display_id, response.get('title'))

From 83465fc4100a2fb2c188898fbc2f3021f6a9b4dd Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Wed, 14 Jun 2023 12:54:06 -0600
Subject: [PATCH 357/871] [extractor/ettutv] Add extractor (#6579)

Closes #6359
Authored by: elyse0
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/ettutv.py      | 60 +++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 yt_dlp/extractor/ettutv.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index e4fd944e7..10e132b4b 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -579,6 +579,7 @@
     ESPNCricInfoIE,
 )
 from .esri import EsriVideoIE
+from .ettutv import EttuTvIE
 from .europa import EuropaIE, EuroParlWebstreamIE
 from .europeantour import EuropeanTourIE
 from .eurosport import EurosportIE
diff --git a/yt_dlp/extractor/ettutv.py b/yt_dlp/extractor/ettutv.py
new file mode 100644
index 000000000..46d725543
--- /dev/null
+++ b/yt_dlp/extractor/ettutv.py
@@ -0,0 +1,60 @@
+from .common import InfoExtractor
+from ..utils import bool_or_none, traverse_obj, unified_timestamp, url_or_none
+
+
+class EttuTvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ettu\.tv/[^?#]+/playerpage/(?P<id>[0-9]+)'
+
+    _TESTS = [{
+        'url': 'https://www.ettu.tv/en-int/playerpage/1573849',
+        'md5': '5874b7639a2aa866d1f6c3a4037c7c09',
+        'info_dict': {
+            'id': '1573849',
+            'title': 'Ni Xia Lian - Shao Jieni',
+            'description': 'ITTF Europe Top 16 Cup',
+            'timestamp': 1677348600,
+            'upload_date': '20230225',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'ext': 'mp4',
+        },
+    }, {
+        'url': 'https://www.ettu.tv/en-int/playerpage/1573753',
+        'md5': '1fc094bf96cf2d5ec0f434d3a6dec9aa',
+        'info_dict': {
+            'id': '1573753',
+            'title': 'Qiu Dang - Jorgic Darko',
+            'description': 'ITTF Europe Top 16 Cup',
+            'timestamp': 1677423600,
+            'upload_date': '20230226',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'ext': 'mp4',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        player_settings = self._download_json(
+            f'https://www.ettu.tv/api/v3/contents/{video_id}/player-settings', video_id, query={
+                'language': 'en',
+                'showTitle': 'true',
+                'device': 'desktop',
+            })
+
+        stream_response = self._download_json(player_settings['streamAccess'], video_id, data={})
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            stream_response['data']['stream'], video_id, 'mp4')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(player_settings, {
+                'title': 'title',
+                'description': ('metaInformation', 'competition'),
+                'thumbnail': ('image', {url_or_none}),
+                'timestamp': ('date', {unified_timestamp}),
+                'is_live': ('isLivestream', {bool_or_none}),
+            })
+        }

From fdd69db38924c38194ef236b26325d66ac815c88 Mon Sep 17 00:00:00 2001
From: "Jeong, Heon" <blmarket@gmail.com>
Date: Wed, 14 Jun 2023 15:01:18 -0400
Subject: [PATCH 358/871] [extractor/afreecatv] Fix extractor (#6283)

Closes #6133
Authored by: blmarket
---
 yt_dlp/extractor/afreecatv.py | 103 +++++++++-------------------------
 1 file changed, 27 insertions(+), 76 deletions(-)

diff --git a/yt_dlp/extractor/afreecatv.py b/yt_dlp/extractor/afreecatv.py
index 9276fe799..3d26d9c25 100644
--- a/yt_dlp/extractor/afreecatv.py
+++ b/yt_dlp/extractor/afreecatv.py
@@ -76,59 +76,6 @@ class AfreecaTVIE(InfoExtractor):
             },
         }],
         'skip': 'Video is gone',
-    }, {
-        'url': 'http://vod.afreecatv.com/PLAYER/STATION/18650793',
-        'info_dict': {
-            'id': '18650793',
-            'ext': 'mp4',
-            'title': '오늘은 다르다! 쏘님의 우월한 위아래~ 댄스리액션!',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': '윈아디',
-            'uploader_id': 'badkids',
-            'duration': 107,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://vod.afreecatv.com/PLAYER/STATION/10481652',
-        'info_dict': {
-            'id': '10481652',
-            'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
-            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
-            'uploader': 'dailyapril',
-            'uploader_id': 'dailyapril',
-            'duration': 6492,
-        },
-        'playlist_count': 2,
-        'playlist': [{
-            'md5': 'd8b7c174568da61d774ef0203159bf97',
-            'info_dict': {
-                'id': '20160502_c4c62b9d_174361386_1',
-                'ext': 'mp4',
-                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 1)",
-                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
-                'uploader': 'dailyapril',
-                'uploader_id': 'dailyapril',
-                'upload_date': '20160502',
-                'duration': 3601,
-            },
-        }, {
-            'md5': '58f2ce7f6044e34439ab2d50612ab02b',
-            'info_dict': {
-                'id': '20160502_39e739bb_174361386_2',
-                'ext': 'mp4',
-                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 2)",
-                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
-                'uploader': 'dailyapril',
-                'uploader_id': 'dailyapril',
-                'upload_date': '20160502',
-                'duration': 2891,
-            },
-        }],
-        'params': {
-            'skip_download': True,
-        },
     }, {
         # non standard key
         'url': 'http://vod.afreecatv.com/PLAYER/STATION/20515605',
@@ -146,8 +93,8 @@ class AfreecaTVIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
-        # PARTIAL_ADULT
-        'url': 'http://vod.afreecatv.com/PLAYER/STATION/32028439',
+        # adult content
+        'url': 'https://vod.afreecatv.com/player/97267690',
         'info_dict': {
             'id': '20180327_27901457_202289533_1',
             'ext': 'mp4',
@@ -161,16 +108,25 @@ class AfreecaTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
-        'expected_warnings': ['adult content'],
+        'skip': 'The VOD does not exist',
     }, {
         'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
         'only_matching': True,
     }, {
-        'url': 'http://vod.afreecatv.com/PLAYER/STATION/15055030',
-        'only_matching': True,
-    }, {
-        'url': 'http://vod.afreecatv.com/player/15055030',
-        'only_matching': True,
+        'url': 'https://vod.afreecatv.com/player/96753363',
+        'info_dict': {
+            'id': '20230108_9FF5BEE1_244432674_1',
+            'ext': 'mp4',
+            'uploader_id': 'rlantnghks',
+            'uploader': '페이즈으',
+            'duration': 10840,
+            'thumbnail': 'http://videoimg.afreecatv.com/php/SnapshotLoad.php?rowKey=20230108_9FF5BEE1_244432674_1_r',
+            'upload_date': '20230108',
+            'title': '젠지 페이즈',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     @staticmethod
@@ -223,26 +179,21 @@ def _perform_login(self, username, password):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-
-        if re.search(r'alert\(["\']This video has been deleted', webpage):
-            raise ExtractorError(
-                'Video %s has been deleted' % video_id, expected=True)
-
-        station_id = self._search_regex(
-            r'nStationNo\s*=\s*(\d+)', webpage, 'station')
-        bbs_id = self._search_regex(
-            r'nBbsNo\s*=\s*(\d+)', webpage, 'bbs')
-        video_id = self._search_regex(
-            r'nTitleNo\s*=\s*(\d+)', webpage, 'title', default=video_id)
-
         partial_view = False
         adult_view = False
         for _ in range(2):
+            data = self._download_json(
+                'https://api.m.afreecatv.com/station/video/a/view',
+                video_id, headers={'Referer': url}, data=urlencode_postdata({
+                    'nTitleNo': video_id,
+                    'nApiLevel': 10,
+                }))['data']
+            if traverse_obj(data, ('code', {int})) == -6221:
+                raise ExtractorError('The VOD does not exist', expected=True)
             query = {
                 'nTitleNo': video_id,
-                'nStationNo': station_id,
-                'nBbsNo': bbs_id,
+                'nStationNo': data['station_no'],
+                'nBbsNo': data['bbs_no'],
             }
             if partial_view:
                 query['partialView'] = 'SKIP_ADULT'

From f9213f8a2d7ba46b912afe1dd3ce6bb700a33d72 Mon Sep 17 00:00:00 2001
From: foreignBlade <136548235+foreignBlade@users.noreply.github.com>
Date: Thu, 15 Jun 2023 06:56:26 -0400
Subject: [PATCH 359/871] [extractor/stripchat] Fix extractor (#7306)

Closes #7305
Authored by: foreignBlade
---
 yt_dlp/extractor/stripchat.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/stripchat.py b/yt_dlp/extractor/stripchat.py
index 4d2fb0608..b9523c865 100644
--- a/yt_dlp/extractor/stripchat.py
+++ b/yt_dlp/extractor/stripchat.py
@@ -42,14 +42,13 @@ def _real_extract(self, url):
         elif not traverse_obj(data, ('viewCam', 'model', 'isLive'), expected_type=bool):
             raise UserNotLive(video_id=video_id)
 
-        server = traverse_obj(data, ('viewCam', 'viewServers', 'flashphoner-hls'), expected_type=str)
         model_id = traverse_obj(data, ('viewCam', 'model', 'id'), expected_type=int)
 
         formats = []
         for host in traverse_obj(data, ('config', 'data', (
                 (('features', 'featuresV2'), 'hlsFallback', 'fallbackDomains', ...), 'hlsStreamHost'))):
             formats = self._extract_m3u8_formats(
-                f'https://b-{server}.{host}/hls/{model_id}/master/{model_id}_auto.m3u8',
+                f'https://edge-hls.{host}/hls/{model_id}/master/{model_id}_auto.m3u8',
                 video_id, ext='mp4', m3u8_id='hls', fatal=False, live=True)
             if formats:
                 break

From 125ffaa1737dd04716f2f6fbb0595ad3eb7a4b1c Mon Sep 17 00:00:00 2001
From: TxI5 <92522534+TxI5@users.noreply.github.com>
Date: Thu, 15 Jun 2023 19:57:25 +0200
Subject: [PATCH 360/871] [extractor/tv4] Fix extractor (#5649)

Closes #5535
Authored by: TxI5, dirkf
---
 yt_dlp/extractor/tv4.py | 81 ++++++++++++++++++++++++-----------------
 1 file changed, 47 insertions(+), 34 deletions(-)

diff --git a/yt_dlp/extractor/tv4.py b/yt_dlp/extractor/tv4.py
index 1378a6f57..10a2fe6e2 100644
--- a/yt_dlp/extractor/tv4.py
+++ b/yt_dlp/extractor/tv4.py
@@ -2,8 +2,11 @@
 
 from .common import InfoExtractor
 from ..utils import (
+    bool_or_none,
     int_or_none,
     parse_iso8601,
+    traverse_obj,
+    url_or_none,
 )
 
 
@@ -20,19 +23,25 @@ class TV4IE(InfoExtractor):
                 sport/|
             )
         )(?P<id>[0-9]+)'''
-    _GEO_COUNTRIES = ['SE']
+    _GEO_BYPASS = False
     _TESTS = [
         {
+            # not geo-restricted
             'url': 'http://www.tv4.se/kalla-fakta/klipp/kalla-fakta-5-english-subtitles-2491650',
             'md5': 'cb837212f342d77cec06e6dad190e96d',
             'info_dict': {
                 'id': '2491650',
                 'ext': 'mp4',
                 'title': 'Kalla Fakta 5 (english subtitles)',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'timestamp': int,
+                'description': '2491650',
+                'series': 'Kalla fakta',
+                'duration': 1335,
+                'thumbnail': r're:^https?://[^/?#]+/api/v2/img/',
+                'timestamp': 1385373240,
                 'upload_date': '20131125',
             },
+            'params': {'skip_download': 'm3u8'},
+            'expected_warnings': ['Unable to download f4m manifest'],
         },
         {
             'url': 'http://www.tv4play.se/iframe/video/3054113',
@@ -46,6 +55,7 @@ class TV4IE(InfoExtractor):
                 'timestamp': int,
                 'upload_date': '20150130',
             },
+            'skip': '404 Not Found',
         },
         {
             'url': 'http://www.tv4play.se/sport/3060959',
@@ -69,29 +79,28 @@ class TV4IE(InfoExtractor):
         }
     ]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        info = self._download_json(
-            'https://playback-api.b17g.net/asset/%s' % video_id,
-            video_id, 'Downloading video info JSON', query={
-                'service': 'tv4',
-                'device': 'browser',
-                'protocol': 'hls,dash',
-                'drm': 'widevine',
-            })['metadata']
-
-        title = info['title']
-
-        manifest_url = self._download_json(
-            'https://playback-api.b17g.net/media/' + video_id,
-            video_id, query={
+    def _call_api(self, endpoint, video_id, headers=None, query={}):
+        return self._download_json(
+            f'https://playback2.a2d.tv/{endpoint}/{video_id}', video_id,
+            f'Downloading {endpoint} API JSON', headers=headers, query={
                 'service': 'tv4',
                 'device': 'browser',
                 'protocol': 'hls',
-            })['playbackItem']['manifestUrl']
-        formats = []
-        subtitles = {}
+                **query,
+            })
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        info = traverse_obj(self._call_api('asset', video_id, query={
+            'protocol': 'hls,dash',
+            'drm': 'widevine',
+        }), ('metadata', {dict})) or {}
+
+        manifest_url = self._call_api(
+            'play', video_id, headers=self.geo_verification_headers())['playbackItem']['manifestUrl']
+
+        formats, subtitles = [], {}
 
         fmts, subs = self._extract_m3u8_formats_and_subtitles(
             manifest_url, video_id, 'mp4',
@@ -117,20 +126,24 @@ def _real_extract(self, url):
         subtitles = self._merge_subtitles(subtitles, subs)
 
         if not formats and info.get('is_geo_restricted'):
-            self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
+            self.raise_geo_restricted(
+                'This video is not available from your location due to geo-restriction, or not being authenticated',
+                countries=['SE'])
 
         return {
             'id': video_id,
-            'title': title,
             'formats': formats,
             'subtitles': subtitles,
-            'description': info.get('description'),
-            'timestamp': parse_iso8601(info.get('broadcast_date_time')),
-            'duration': int_or_none(info.get('duration')),
-            'thumbnail': info.get('image'),
-            'is_live': info.get('isLive') is True,
-            'series': info.get('seriesTitle'),
-            'season_number': int_or_none(info.get('seasonNumber')),
-            'episode': info.get('episodeTitle'),
-            'episode_number': int_or_none(info.get('episodeNumber')),
+            **traverse_obj(info, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'timestamp': (('broadcast_date_time', 'broadcastDateTime'), {parse_iso8601}),
+                'duration': ('duration', {int_or_none}),
+                'thumbnail': ('image', {url_or_none}),
+                'is_live': ('isLive', {bool_or_none}),
+                'series': ('seriesTitle', {str}),
+                'season_number': ('seasonNumber', {int_or_none}),
+                'episode': ('episodeTitle', {str}),
+                'episode_number': ('episodeNumber', {int_or_none}),
+            }, get_all=False),
         }

From 0a5d7c39e17bb9bd50c9db42bcad40eb82d7f784 Mon Sep 17 00:00:00 2001
From: toomyzoom <52140413+toomyzoom@users.noreply.github.com>
Date: Thu, 15 Jun 2023 16:23:01 -0700
Subject: [PATCH 361/871] [extractor/iwara] Fix authentication (#7137)

Closes #7035, Closes #7207
Authored by: toomyzoom
---
 yt_dlp/extractor/iwara.py | 149 ++++++++++++++++++++++----------------
 1 file changed, 85 insertions(+), 64 deletions(-)

diff --git a/yt_dlp/extractor/iwara.py b/yt_dlp/extractor/iwara.py
index bdc39a7dd..e23fdfd6a 100644
--- a/yt_dlp/extractor/iwara.py
+++ b/yt_dlp/extractor/iwara.py
@@ -1,68 +1,83 @@
 import functools
 import urllib.parse
+import urllib.error
 import hashlib
 import json
+import time
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
     int_or_none,
+    jwt_decode_hs256,
     mimetype2ext,
     qualities,
     traverse_obj,
+    try_call,
     unified_timestamp,
 )
 
 
-# https://github.com/yt-dlp/yt-dlp/issues/6671
 class IwaraBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'iwara'
     _USERTOKEN = None
     _MEDIATOKEN = None
-    _NETRC_MACHINE = 'iwara'
 
-    def _get_user_token(self, invalidate=False):
-        if not invalidate and self._USERTOKEN:
-            return self._USERTOKEN
+    def _is_token_expired(self, token, token_type):
+        # User token TTL == ~3 weeks, Media token TTL == ~1 hour
+        if (try_call(lambda: jwt_decode_hs256(token)['exp']) or 0) <= int(time.time() - 120):
+            self.to_screen(f'{token_type} token has expired')
+            return True
 
+    def _get_user_token(self):
         username, password = self._get_login_info()
-        IwaraBaseIE._USERTOKEN = username and self.cache.load(self._NETRC_MACHINE, username)
-        if not IwaraBaseIE._USERTOKEN or invalidate:
-            IwaraBaseIE._USERTOKEN = self._download_json(
+        if not username or not password:
+            return
+
+        user_token = IwaraBaseIE._USERTOKEN or self.cache.load(self._NETRC_MACHINE, username)
+        if not user_token or self._is_token_expired(user_token, 'User'):
+            response = self._download_json(
                 'https://api.iwara.tv/user/login', None, note='Logging in',
-                data=json.dumps({
+                headers={'Content-Type': 'application/json'}, data=json.dumps({
                     'email': username,
                     'password': password
-                }).encode('utf-8'),
-                headers={
+                }).encode(), expected_status=lambda x: True)
+            user_token = traverse_obj(response, ('token', {str}))
+            if not user_token:
+                error = traverse_obj(response, ('message', {str}))
+                if 'invalidLogin' in error:
+                    raise ExtractorError('Invalid login credentials', expected=True)
+                else:
+                    raise ExtractorError(f'Iwara API said: {error or "nothing"}')
+
+            self.cache.store(self._NETRC_MACHINE, username, user_token)
+
+        IwaraBaseIE._USERTOKEN = user_token
+
+    def _get_media_token(self):
+        self._get_user_token()
+        if not IwaraBaseIE._USERTOKEN:
+            return  # user has not passed credentials
+
+        if not IwaraBaseIE._MEDIATOKEN or self._is_token_expired(IwaraBaseIE._MEDIATOKEN, 'Media'):
+            IwaraBaseIE._MEDIATOKEN = self._download_json(
+                'https://api.iwara.tv/user/token', None, note='Fetching media token',
+                data=b'', headers={
+                    'Authorization': f'Bearer {IwaraBaseIE._USERTOKEN}',
                     'Content-Type': 'application/json'
-                })['token']
+                })['accessToken']
 
-            self.cache.store(self._NETRC_MACHINE, username, IwaraBaseIE._USERTOKEN)
+        return {'Authorization': f'Bearer {IwaraBaseIE._MEDIATOKEN}'}
 
-        return self._USERTOKEN
-
-    def _get_media_token(self, invalidate=False):
-        if not invalidate and self._MEDIATOKEN:
-            return self._MEDIATOKEN
-
-        IwaraBaseIE._MEDIATOKEN = self._download_json(
-            'https://api.iwara.tv/user/token', None, note='Fetching media token',
-            data=b'',  # Need to have some data here, even if it's empty
-            headers={
-                'Authorization': f'Bearer {self._get_user_token()}',
-                'Content-Type': 'application/json'
-            })['accessToken']
-
-        return self._MEDIATOKEN
+    def _perform_login(self, username, password):
+        self._get_media_token()
 
 
 class IwaraIE(IwaraBaseIE):
     IE_NAME = 'iwara'
     _VALID_URL = r'https?://(?:www\.|ecchi\.)?iwara\.tv/videos?/(?P<id>[a-zA-Z0-9]+)'
     _TESTS = [{
-        # this video cannot be played because of migration
-        'only_matching': True,
         'url': 'https://www.iwara.tv/video/k2ayoueezfkx6gvq',
         'info_dict': {
             'id': 'k2ayoueezfkx6gvq',
@@ -79,25 +94,29 @@ class IwaraIE(IwaraBaseIE):
             'timestamp': 1677843869,
             'modified_timestamp': 1679056362,
         },
+        'skip': 'this video cannot be played because of migration',
     }, {
         'url': 'https://iwara.tv/video/1ywe1sbkqwumpdxz5/',
-        'md5': '20691ce1473ec2766c0788e14c60ce66',
+        'md5': '7645f966f069b8ec9210efd9130c9aad',
         'info_dict': {
             'id': '1ywe1sbkqwumpdxz5',
             'ext': 'mp4',
             'age_limit': 18,
-            'title': 'Aponia 阿波尼亚SEX  Party Tonight 手动脱衣 大奶 裸腿',
-            'description': 'md5:0c4c310f2e0592d68b9f771d348329ca',
-            'uploader': '龙也zZZ',
+            'title': 'Aponia アポニア SEX  Party Tonight 手の脱衣 巨乳 ',
+            'description': 'md5:3f60016fff22060eef1ef26d430b1f67',
+            'uploader': 'Lyu ya',
             'uploader_id': 'user792540',
             'tags': [
                 'uncategorized'
             ],
-            'like_count': 1809,
-            'view_count': 25156,
-            'comment_count': 1,
+            'like_count': int,
+            'view_count': int,
+            'comment_count': int,
             'timestamp': 1678732213,
-            'modified_timestamp': 1679110271,
+            'modified_timestamp': int,
+            'thumbnail': 'https://files.iwara.tv/image/thumbnail/581d12b5-46f4-4f15-beb2-cfe2cde5d13d/thumbnail-00.jpg',
+            'modified_date': '20230614',
+            'upload_date': '20230313',
         },
     }, {
         'url': 'https://iwara.tv/video/blggmfno8ghl725bg',
@@ -112,12 +131,15 @@ class IwaraIE(IwaraBaseIE):
             'tags': [
                 'pee'
             ],
-            'like_count': 192,
-            'view_count': 12119,
-            'comment_count': 0,
+            'like_count': int,
+            'view_count': int,
+            'comment_count': int,
             'timestamp': 1598880567,
-            'modified_timestamp': 1598908995,
-            'availability': 'needs_auth',
+            'modified_timestamp': int,
+            'upload_date': '20200831',
+            'modified_date': '20230605',
+            'thumbnail': 'https://files.iwara.tv/image/thumbnail/7693e881-d302-42a4-a780-f16d66b5dadd/thumbnail-00.jpg',
+            # 'availability': 'needs_auth',
         },
     }]
 
@@ -142,17 +164,16 @@ def _extract_formats(self, video_id, fileurl):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        username, password = self._get_login_info()
-        headers = {
-            'Authorization': f'Bearer {self._get_media_token()}',
-        } if username and password else None
-        video_data = self._download_json(f'https://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True, headers=headers)
+        username, _ = self._get_login_info()
+        video_data = self._download_json(
+            f'https://api.iwara.tv/video/{video_id}', video_id,
+            expected_status=lambda x: True, headers=self._get_media_token())
         errmsg = video_data.get('message')
         # at this point we can actually get uploaded user info, but do we need it?
         if errmsg == 'errors.privateVideo':
-            self.raise_login_required('Private video. Login if you have permissions to watch')
+            self.raise_login_required('Private video. Login if you have permissions to watch', method='password')
         elif errmsg == 'errors.notFound' and not username:
-            self.raise_login_required('Video may need login to view')
+            self.raise_login_required('Video may need login to view', method='password')
         elif errmsg:  # None if success
             raise ExtractorError(f'Iwara says: {errmsg}')
 
@@ -181,15 +202,6 @@ def _real_extract(self, url):
             'formats': list(self._extract_formats(video_id, video_data.get('fileUrl'))),
         }
 
-    def _perform_login(self, username, password):
-        if self.cache.load(self._NETRC_MACHINE, username) and self._get_media_token():
-            self.write_debug('Skipping logging in')
-            return
-
-        IwaraBaseIE._USERTOKEN = self._get_user_token(True)
-        self._get_media_token(True)
-        self.cache.store(self._NETRC_MACHINE, username, IwaraBaseIE._USERTOKEN)
-
 
 class IwaraUserIE(IwaraBaseIE):
     _VALID_URL = r'https?://(?:www\.)?iwara\.tv/profile/(?P<id>[^/?#&]+)'
@@ -200,12 +212,14 @@ class IwaraUserIE(IwaraBaseIE):
         'url': 'https://iwara.tv/profile/user792540/videos',
         'info_dict': {
             'id': 'user792540',
+            'title': 'Lyu ya',
         },
-        'playlist_mincount': 80,
+        'playlist_mincount': 70,
     }, {
         'url': 'https://iwara.tv/profile/theblackbirdcalls/videos',
         'info_dict': {
             'id': 'theblackbirdcalls',
+            'title': 'TheBlackbirdCalls',
         },
         'playlist_mincount': 723,
     }, {
@@ -214,6 +228,13 @@ class IwaraUserIE(IwaraBaseIE):
     }, {
         'url': 'https://iwara.tv/profile/theblackbirdcalls',
         'only_matching': True,
+    }, {
+        'url': 'https://www.iwara.tv/profile/lumymmd',
+        'info_dict': {
+            'id': 'lumymmd',
+            'title': 'Lumy MMD',
+        },
+        'playlist_mincount': 1,
     }]
 
     def _entries(self, playlist_id, user_id, page):
@@ -225,7 +246,7 @@ def _entries(self, playlist_id, user_id, page):
                 'sort': 'date',
                 'user': user_id,
                 'limit': self._PER_PAGE,
-            })
+            }, headers=self._get_media_token())
         for x in traverse_obj(videos, ('results', ..., 'id')):
             yield self.url_result(f'https://iwara.tv/video/{x}')
 
@@ -244,7 +265,6 @@ def _real_extract(self, url):
 
 
 class IwaraPlaylistIE(IwaraBaseIE):
-    # the ID is an UUID but I don't think it's necessary to write concrete regex
     _VALID_URL = r'https?://(?:www\.)?iwara\.tv/playlist/(?P<id>[0-9a-f-]+)'
     IE_NAME = 'iwara:playlist'
     _PER_PAGE = 32
@@ -260,7 +280,8 @@ class IwaraPlaylistIE(IwaraBaseIE):
     def _entries(self, playlist_id, first_page, page):
         videos = self._download_json(
             'https://api.iwara.tv/videos', playlist_id, f'Downloading page {page}',
-            query={'page': page, 'limit': self._PER_PAGE}) if page else first_page
+            query={'page': page, 'limit': self._PER_PAGE},
+            headers=self._get_media_token()) if page else first_page
         for x in traverse_obj(videos, ('results', ..., 'id')):
             yield self.url_result(f'https://iwara.tv/video/{x}')
 
@@ -268,7 +289,7 @@ def _real_extract(self, url):
         playlist_id = self._match_id(url)
         page_0 = self._download_json(
             f'https://api.iwara.tv/playlist/{playlist_id}?page=0&limit={self._PER_PAGE}', playlist_id,
-            note='Requesting playlist info')
+            note='Requesting playlist info', headers=self._get_media_token())
 
         return self.playlist_result(
             OnDemandPagedList(

From ff9b0e071ffae5543cc309e6f9e647ac51e5846e Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 14 Jun 2023 19:08:46 +0530
Subject: [PATCH 362/871] [extractor/youtube] Determine audio language using
 automatic captions

---
 yt_dlp/extractor/youtube.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 47ad1da76..606f24d04 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4312,9 +4312,13 @@ def process_language(container, base_url, lang_code, sub_name, query):
                             continue
                         trans_code += f'-{lang_code}'
                         trans_name += format_field(lang_name, None, ' from %s')
-                    # Add an "-orig" label to the original language so that it can be distinguished.
-                    # The subs are returned without "-orig" as well for compatibility
                     if lang_code == f'a-{orig_trans_code}':
+                        # Set audio language based on original subtitles
+                        for f in formats:
+                            if f.get('acodec') != 'none' and not f.get('language'):
+                                f['language'] = orig_trans_code
+                        # Add an "-orig" label to the original language so that it can be distinguished.
+                        # The subs are returned without "-orig" as well for compatibility
                         process_language(
                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
                     # Setting tlang=lang returns damaged subtitles.

From 13ff78095372fd98900a32572cf817994c07ccb5 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 14 Jun 2023 19:09:53 +0530
Subject: [PATCH 363/871] [postprocessor] Print newline for
 `--progress-template`

Closes #7193
---
 yt_dlp/postprocessor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/postprocessor/common.py b/yt_dlp/postprocessor/common.py
index 537792b07..08b0fe1ff 100644
--- a/yt_dlp/postprocessor/common.py
+++ b/yt_dlp/postprocessor/common.py
@@ -187,7 +187,7 @@ def report_progress(self, s):
         tmpl = progress_template.get('postprocess')
         if tmpl:
             self._downloader.to_screen(
-                self._downloader.evaluate_outtmpl(tmpl, progress_dict), skip_eol=True, quiet=False)
+                self._downloader.evaluate_outtmpl(tmpl, progress_dict), quiet=False)
 
         self._downloader.to_console_title(self._downloader.evaluate_outtmpl(
             progress_template.get('postprocess-title') or 'yt-dlp %(progress._default_template)s',

From 01aba2519a0884ef17d5f85608dbd2a455577147 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 18 Jun 2023 04:04:52 +0530
Subject: [PATCH 364/871] [jsinterp] Fix global object extraction

Closes #7327
---
 test/test_youtube_signature.py | 7 ++++++-
 yt_dlp/jsinterp.py             | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 01f09de88..6759d2c46 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -62,6 +62,11 @@
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
         '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
         '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
+    ),
+    (
+        'https://www.youtube.com/s/player/6ed0d907/player_ias.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        'AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
     )
 ]
 
@@ -230,7 +235,7 @@ def n_sig(jscode, sig_input):
 
 
 make_sig_test = t_factory(
-    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+    'signature', signature, re.compile(r'.*(?:-|/player/)(?P<id>[a-zA-Z0-9_-]+)(?:/.+\.js|(?:/watch_as3|/html5player)?\.[a-z]+)$'))
 for test_spec in _SIG_TESTS:
     make_sig_test(*test_spec)
 
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index d6d555733..9c280fb86 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -779,7 +779,7 @@ def extract_object(self, objname):
         obj = {}
         obj_m = re.search(
             r'''(?x)
-                (?<!this\.)%s\s*=\s*{\s*
+                (?<!\.)%s\s*=\s*{\s*
                     (?P<fields>(%s\s*:\s*function\s*\(.*?\)\s*{.*?}(?:,\s*)?)*)
                 }\s*;
             ''' % (re.escape(objname), _FUNC_NAME_RE),

From 81c8b9bdd9841b72cbfc1bbff9dab5fb4aa038b0 Mon Sep 17 00:00:00 2001
From: garret <76261416+garret1317@users.noreply.github.com>
Date: Mon, 19 Jun 2023 14:25:27 +0100
Subject: [PATCH 365/871] [extractor/nhk] `NhkRadiruLive`: Add extractor
 (#7332)

Authored by: garret1317
---
 README.md                       |  3 ++
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/nhk.py         | 75 ++++++++++++++++++++++++++++++++-
 3 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ce555c66f..659730410 100644
--- a/README.md
+++ b/README.md
@@ -1850,6 +1850,9 @@ ### wrestleuniverse
 #### twitchstream (Twitch)
 * `client_id`: Client ID value to be sent with GraphQL requests, e.g. `twitchstream:client_id=kimne78kx3ncx6brgo4mv6wki5h1ko`
 
+#### nhkradirulive (NHK らじる★らじる LIVE)
+* `area`: Which regional variation to extract. Valid areas are: `sapporo`, `sendai`, `tokyo`, `nagoya`, `osaka`, `hiroshima`, `matsuyama`, `fukuoka`. Defaults to `tokyo`
+
 **Note**: These options may be changed/removed in the future without concern for backward compatibility
 
 <!-- MANPAGE: MOVE "INSTALLATION" SECTION HERE -->
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 10e132b4b..394f3c29d 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1260,6 +1260,7 @@
     NhkForSchoolProgramListIE,
     NhkRadioNewsPageIE,
     NhkRadiruIE,
+    NhkRadiruLiveIE,
 )
 from .nhl import NHLIE
 from .nick import (
diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py
index a3efa326a..fbd6a18f6 100644
--- a/yt_dlp/extractor/nhk.py
+++ b/yt_dlp/extractor/nhk.py
@@ -2,12 +2,15 @@
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
+    int_or_none,
+    join_nonempty,
     parse_duration,
     traverse_obj,
     unescapeHTML,
     unified_timestamp,
+    url_or_none,
     urljoin,
-    url_or_none
 )
 
 
@@ -492,3 +495,73 @@ class NhkRadioNewsPageIE(InfoExtractor):
 
     def _real_extract(self, url):
         return self.url_result('https://www.nhk.or.jp/radio/ondemand/detail.html?p=F261_01', NhkRadiruIE)
+
+
+class NhkRadiruLiveIE(InfoExtractor):
+    _GEO_COUNTRIES = ['JP']
+    _VALID_URL = r'https?://www\.nhk\.or\.jp/radio/player/\?ch=(?P<id>r[12]|fm)'
+    _TESTS = [{
+        # radio 1, no area specified
+        'url': 'https://www.nhk.or.jp/radio/player/?ch=r1',
+        'info_dict': {
+            'id': 'r1-tokyo',
+            'title': 're:^ＮＨＫネットラジオ第1 東京.+$',
+            'ext': 'm4a',
+            'thumbnail': 'https://www.nhk.or.jp/common/img/media/r1-200x200.png',
+            'live_status': 'is_live',
+        },
+    }, {
+        # radio 2, area specified
+        # (the area doesnt actually matter, r2 is national)
+        'url': 'https://www.nhk.or.jp/radio/player/?ch=r2',
+        'params': {'extractor_args': {'nhkradirulive': {'area': ['fukuoka']}}},
+        'info_dict': {
+            'id': 'r2-fukuoka',
+            'title': 're:^ＮＨＫネットラジオ第2 福岡.+$',
+            'ext': 'm4a',
+            'thumbnail': 'https://www.nhk.or.jp/common/img/media/r2-200x200.png',
+            'live_status': 'is_live',
+        },
+    }, {
+        # fm, area specified
+        'url': 'https://www.nhk.or.jp/radio/player/?ch=fm',
+        'params': {'extractor_args': {'nhkradirulive': {'area': ['sapporo']}}},
+        'info_dict': {
+            'id': 'fm-sapporo',
+            'title': 're:^ＮＨＫネットラジオＦＭ 札幌.+$',
+            'ext': 'm4a',
+            'thumbnail': 'https://www.nhk.or.jp/common/img/media/fm-200x200.png',
+            'live_status': 'is_live',
+        }
+    }]
+
+    _NOA_STATION_IDS = {'r1': 'n1', 'r2': 'n2', 'fm': 'n3'}
+
+    def _real_extract(self, url):
+        station = self._match_id(url)
+        area = self._configuration_arg('area', ['tokyo'])[0]
+
+        config = self._download_xml(
+            'https://www.nhk.or.jp/radio/config/config_web.xml', station, 'Downloading area information')
+        data = config.find(f'.//data//area[.="{area}"]/..')
+
+        if not data:
+            raise ExtractorError('Invalid area. Valid areas are: %s' % ', '.join(
+                [i.text for i in config.findall('.//data//area')]), expected=True)
+
+        noa_info = self._download_json(
+            f'https:{config.find(".//url_program_noa").text}'.format(area=data.find('areakey').text),
+            station, note=f'Downloading {area} station metadata')
+        present_info = traverse_obj(noa_info, ('nowonair_list', self._NOA_STATION_IDS.get(station), 'present'))
+
+        return {
+            'title': ' '.join(traverse_obj(present_info, (('service', 'area',), 'name', {str}))),
+            'id': join_nonempty(station, area),
+            'thumbnails': traverse_obj(present_info, ('service', 'images', ..., {
+                'url': 'url',
+                'width': ('width', {int_or_none}),
+                'height': ('height', {int_or_none}),
+            })),
+            'formats': self._extract_m3u8_formats(data.find(f'{station}hls').text, station),
+            'is_live': True,
+        }

From 6f69101dc912690338d32e2aab085c32e44eba3f Mon Sep 17 00:00:00 2001
From: Vladislav <117850688+7vlad7@users.noreply.github.com>
Date: Mon, 19 Jun 2023 23:43:35 +0300
Subject: [PATCH 366/871] [extractor/yappy] YappyProfile: Add extractor (#7346)

Authored by: 7vlad7
---
 yt_dlp/extractor/_extractors.py |  5 ++++-
 yt_dlp/extractor/yappy.py       | 30 +++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 394f3c29d..7e1fa4a0d 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2427,7 +2427,10 @@
     ZenYandexChannelIE,
 )
 from .yapfiles import YapFilesIE
-from .yappy import YappyIE
+from .yappy import (
+    YappyIE,
+    YappyProfileIE,
+)
 from .yesjapan import YesJapanIE
 from .yinyuetai import YinYueTaiIE
 from .yle_areena import YleAreenaIE
diff --git a/yt_dlp/extractor/yappy.py b/yt_dlp/extractor/yappy.py
index f168bdbf9..7b3d0cb81 100644
--- a/yt_dlp/extractor/yappy.py
+++ b/yt_dlp/extractor/yappy.py
@@ -1,9 +1,10 @@
 from .common import InfoExtractor
 from ..utils import (
+    OnDemandPagedList,
     int_or_none,
     traverse_obj,
     unified_timestamp,
-    url_or_none
+    url_or_none,
 )
 
 
@@ -97,3 +98,30 @@ def _real_extract(self, url):
             'categories': traverse_obj(media_data, ('categories', ..., 'name')) or None,
             'repost_count': int_or_none(media_data.get('sharingCount'))
         }
+
+
+class YappyProfileIE(InfoExtractor):
+    _VALID_URL = r'https?://yappy\.media/profile/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://yappy.media/profile/59a0c8c485e5410b9c43474bf4c6a373',
+        'info_dict': {
+            'id': '59a0c8c485e5410b9c43474bf4c6a373',
+        },
+        'playlist_mincount': 527,
+    }]
+
+    def _real_extract(self, url):
+        profile_id = self._match_id(url)
+
+        def fetch_page(page_num):
+            page_num += 1
+            videos = self._download_json(
+                f'https://yappy.media/api/video/list/{profile_id}?page={page_num}',
+                profile_id, f'Downloading profile page {page_num} JSON')
+
+            for video in traverse_obj(videos, ('results', lambda _, v: v['uuid'])):
+                yield self.url_result(
+                    f'https://yappy.media/video/{video["uuid"]}', YappyIE,
+                    video['uuid'], video.get('description'))
+
+        return self.playlist_result(OnDemandPagedList(fetch_page, 15), profile_id)

From 5cc09c004bd5edbbada9b041c08a720cadc4f4df Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Tue, 20 Jun 2023 12:22:36 +0800
Subject: [PATCH 367/871] [extractor/zaiko] ZaikoETicket: Add extractor (#7347)

Authored by: pzhlkj6612
---
 yt_dlp/extractor/_extractors.py |  5 ++-
 yt_dlp/extractor/zaiko.py       | 70 +++++++++++++++++++++++++--------
 2 files changed, 58 insertions(+), 17 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 7e1fa4a0d..ff659a7a2 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2448,7 +2448,10 @@
 from .youporn import YouPornIE
 from .yourporn import YourPornIE
 from .yourupload import YourUploadIE
-from .zaiko import ZaikoIE
+from .zaiko import (
+    ZaikoIE,
+    ZaikoETicketIE,
+)
 from .zapiks import ZapiksIE
 from .zattoo import (
     BBVTVIE,
diff --git a/yt_dlp/extractor/zaiko.py b/yt_dlp/extractor/zaiko.py
index 59fc64c5a..84cee4445 100644
--- a/yt_dlp/extractor/zaiko.py
+++ b/yt_dlp/extractor/zaiko.py
@@ -1,3 +1,5 @@
+import base64
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -5,12 +7,33 @@
     int_or_none,
     str_or_none,
     traverse_obj,
+    try_call,
     unescapeHTML,
     url_or_none,
 )
 
 
-class ZaikoIE(InfoExtractor):
+class ZaikoBaseIE(InfoExtractor):
+    def _download_real_webpage(self, url, video_id):
+        webpage, urlh = self._download_webpage_handle(url, video_id)
+        final_url = urlh.geturl()
+        if 'zaiko.io/login' in final_url:
+            self.raise_login_required()
+        elif '/_buy/' in final_url:
+            raise ExtractorError('Your account does not have tickets to this event', expected=True)
+        return webpage
+
+    def _parse_vue_element_attr(self, name, string, video_id):
+        page_elem = self._search_regex(rf'(<{name}[^>]+>)', string, name)
+        attrs = {}
+        for key, value in extract_attributes(page_elem).items():
+            if key.startswith(':'):
+                attrs[key[1:]] = self._parse_json(
+                    value, video_id, transform_source=unescapeHTML, fatal=False)
+        return attrs
+
+
+class ZaikoIE(ZaikoBaseIE):
     _VALID_URL = r'https?://(?:[\w-]+\.)?zaiko\.io/event/(?P<id>\d+)/stream(?:/\d+)+'
     _TESTS = [{
         'url': 'https://zaiko.io/event/324868/stream/20571/20571',
@@ -30,24 +53,10 @@ class ZaikoIE(InfoExtractor):
         'params': {'skip_download': 'm3u8'},
     }]
 
-    def _parse_vue_element_attr(self, name, string, video_id):
-        page_elem = self._search_regex(rf'(<{name}[^>]+>)', string, name)
-        attrs = {}
-        for key, value in extract_attributes(page_elem).items():
-            if key.startswith(':'):
-                attrs[key[1:]] = self._parse_json(
-                    value, video_id, transform_source=unescapeHTML, fatal=False)
-        return attrs
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage, urlh = self._download_webpage_handle(url, video_id)
-        final_url = urlh.geturl()
-        if 'zaiko.io/login' in final_url:
-            self.raise_login_required()
-        elif '/_buy/' in final_url:
-            raise ExtractorError('Your account does not have tickets to this event', expected=True)
+        webpage = self._download_real_webpage(url, video_id)
         stream_meta = self._parse_vue_element_attr('stream-page', webpage, video_id)
 
         player_page = self._download_webpage(
@@ -90,3 +99,32 @@ def _real_extract(self, url):
                 'thumbnail': ('poster_url', {url_or_none}),
             })),
         }
+
+
+class ZaikoETicketIE(ZaikoBaseIE):
+    _VALID_URL = r'https?://(?:www.)?zaiko\.io/account/eticket/(?P<id>[\w=-]{49})'
+    _TESTS = [{
+        'url': 'https://zaiko.io/account/eticket/TZjMwMzQ2Y2EzMXwyMDIzMDYwNzEyMTMyNXw1MDViOWU2Mw==',
+        'playlist_count': 1,
+        'info_dict': {
+            'id': 'f30346ca31-20230607121325-505b9e63',
+            'title': 'ZAIKO STREAMING TEST',
+            'thumbnail': 'https://media.zkocdn.net/pf_1/1_3wdyjcjyupseatkwid34u',
+        },
+        'skip': 'Only available with the ticketholding account',
+    }]
+
+    def _real_extract(self, url):
+        ticket_id = self._match_id(url)
+        ticket_id = try_call(
+            lambda: base64.urlsafe_b64decode(ticket_id[1:]).decode().replace('|', '-')) or ticket_id
+
+        webpage = self._download_real_webpage(url, ticket_id)
+        eticket = self._parse_vue_element_attr('eticket', webpage, ticket_id)
+
+        return self.playlist_result(
+            [self.url_result(stream, ZaikoIE) for stream in traverse_obj(eticket, ('streams', ..., 'url'))],
+            ticket_id, **traverse_obj(eticket, ('ticket-details', {
+                'title': 'event_name',
+                'thumbnail': 'event_img_url',
+            })))

From eedda5252c05327748dede204a8fccafa0288118 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 19 Jun 2023 14:06:39 +0530
Subject: [PATCH 368/871] [utils] `FormatSorter`: Improve `size` and `br`

Closes #1596

Previously, when some formats have accurate size and some approximate,
the ones with accurate size was always prioritized

For formats with known tbr and unknown vbr/abr, we were setting
(vbr=tbr, abr=0) for sorting to work. This is no longer needed.

Authored by pukkandan, u-spec-png
---
 test/test_InfoExtractor.py | 10 ----------
 yt_dlp/utils/_utils.py     | 25 +++++++++++++++----------
 2 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 1f60abfd2..b7dee496a 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -917,8 +917,6 @@ def test_parse_m3u8_formats(self):
                     'acodec': 'mp4a.40.2',
                     'video_ext': 'mp4',
                     'audio_ext': 'none',
-                    'vbr': 263.851,
-                    'abr': 0,
                 }, {
                     'format_id': '577',
                     'format_index': None,
@@ -936,8 +934,6 @@ def test_parse_m3u8_formats(self):
                     'acodec': 'mp4a.40.2',
                     'video_ext': 'mp4',
                     'audio_ext': 'none',
-                    'vbr': 577.61,
-                    'abr': 0,
                 }, {
                     'format_id': '915',
                     'format_index': None,
@@ -955,8 +951,6 @@ def test_parse_m3u8_formats(self):
                     'acodec': 'mp4a.40.2',
                     'video_ext': 'mp4',
                     'audio_ext': 'none',
-                    'vbr': 915.905,
-                    'abr': 0,
                 }, {
                     'format_id': '1030',
                     'format_index': None,
@@ -974,8 +968,6 @@ def test_parse_m3u8_formats(self):
                     'acodec': 'mp4a.40.2',
                     'video_ext': 'mp4',
                     'audio_ext': 'none',
-                    'vbr': 1030.138,
-                    'abr': 0,
                 }, {
                     'format_id': '1924',
                     'format_index': None,
@@ -993,8 +985,6 @@ def test_parse_m3u8_formats(self):
                     'acodec': 'mp4a.40.2',
                     'video_ext': 'mp4',
                     'audio_ext': 'none',
-                    'vbr': 1924.009,
-                    'abr': 0,
                 }],
                 {
                     'en': [{
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 646210116..1fd6f44af 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -5669,6 +5669,7 @@ def orderedSet_from_options(options, alias_dict, *, use_regex=False, start=None)
     return orderedSet(requested)
 
 
+# TODO: Rewrite
 class FormatSorter:
     regex = r' *((?P<reverse>\+)?(?P<field>[a-zA-Z0-9_]+)((?P<separator>[~:])(?P<limit>.*?))?)? *$'
 
@@ -5717,8 +5718,10 @@ class FormatSorter:
         'source': {'convert': 'float', 'field': 'source_preference', 'default': -1},
 
         'codec': {'type': 'combined', 'field': ('vcodec', 'acodec')},
-        'br': {'type': 'combined', 'field': ('tbr', 'vbr', 'abr'), 'same_limit': True},
-        'size': {'type': 'combined', 'same_limit': True, 'field': ('filesize', 'fs_approx')},
+        'br': {'type': 'multiple', 'field': ('tbr', 'vbr', 'abr'),
+               'function': lambda it: next(filter(None, it), None)},
+        'size': {'type': 'multiple', 'field': ('filesize', 'fs_approx'),
+                 'function': lambda it: next(filter(None, it), None)},
         'ext': {'type': 'combined', 'field': ('vext', 'aext')},
         'res': {'type': 'multiple', 'field': ('height', 'width'),
                 'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))},
@@ -5949,13 +5952,15 @@ def calculate_preference(self, format):
             format['preference'] = -100
 
         # Determine missing bitrates
-        if format.get('tbr') is None:
-            if format.get('vbr') is not None and format.get('abr') is not None:
-                format['tbr'] = format.get('vbr', 0) + format.get('abr', 0)
-        else:
-            if format.get('vcodec') != 'none' and format.get('vbr') is None:
-                format['vbr'] = format.get('tbr') - format.get('abr', 0)
-            if format.get('acodec') != 'none' and format.get('abr') is None:
-                format['abr'] = format.get('tbr') - format.get('vbr', 0)
+        if format.get('vcodec') == 'none':
+            format['vbr'] = 0
+        if format.get('acodec') == 'none':
+            format['abr'] = 0
+        if not format.get('vbr') and format.get('vcodec') != 'none':
+            format['vbr'] = try_call(lambda: format['tbr'] - format['abr']) or None
+        if not format.get('abr') and format.get('acodec') != 'none':
+            format['abr'] = try_call(lambda: format['tbr'] - format['vbr']) or None
+        if not format.get('tbr'):
+            format['tbr'] = try_call(lambda: format['vbr'] + format['abr']) or None
 
         return tuple(self._calculate_field_preference(format, field) for field in self._order)

From 51a07b0dca4c079d58311c19b6d1c097c24bb021 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 19 Jun 2023 14:09:26 +0530
Subject: [PATCH 369/871] [extractor/youtube] Prioritize premium formats

Closes #7283
---
 yt_dlp/extractor/youtube.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 606f24d04..4cbf0115c 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3848,6 +3848,7 @@ def build_fragments(f):
                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
 
             client_name = fmt.get(STREAMING_DATA_CLIENT_NAME)
+            name = fmt.get('qualityLabel') or quality.replace('audio_quality_', '') or ''
             dct = {
                 'asr': int_or_none(fmt.get('audioSampleRate')),
                 'filesize': int_or_none(fmt.get('contentLength')),
@@ -3855,15 +3856,15 @@ def build_fragments(f):
                 'format_note': join_nonempty(
                     join_nonempty(audio_track.get('displayName'),
                                   language_preference > 0 and ' (default)', delim=''),
-                    fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
-                    fmt.get('isDrc') and 'DRC',
+                    name, fmt.get('isDrc') and 'DRC',
                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
                     throttled and 'THROTTLED', is_damaged and 'DAMAGED',
                     (self.get_param('verbose') or all_formats) and client_name,
                     delim=', '),
                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
-                'source_preference': -10 if throttled else -5 if itag == '22' else -1,
+                'source_preference': ((-10 if throttled else -5 if itag == '22' else -1)
+                                      + (100 if 'Premium' in name else 0)),
                 'fps': int_or_none(fmt.get('fps')) or None,
                 'audio_channels': fmt.get('audioChannels'),
                 'height': height,

From 2e023649ea4e11151545a34dc1360c114981a236 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 19 Jun 2023 14:45:59 +0530
Subject: [PATCH 370/871] [cookies] Revert compatibility breakage in
 b38d4c941d1993ab27e4c0f8e024e23c2ec0f8f8

---
 README.md         |  2 +-
 yt_dlp/cookies.py | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 659730410..c39e151d4 100644
--- a/README.md
+++ b/README.md
@@ -728,7 +728,7 @@ ## Filesystem Options:
                                     By default, all containers of the most
                                     recently accessed profile are used.
                                     Currently supported keyrings are: basictext,
-                                    gnomekeyring, kwallet
+                                    gnomekeyring, kwallet, kwallet5, kwallet6
     --no-cookies-from-browser       Do not load cookies from browser (default)
     --cache-dir DIR                 Location in the filesystem where yt-dlp can
                                     store some downloaded information (such as
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index e46d19341..8693e0b4a 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -705,11 +705,11 @@ class _LinuxKeyring(Enum):
     https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_util_linux.h
     SelectedLinuxBackend
     """
-    KWALLET4 = auto()  # this value is just called KWALLET in the chromium source but it is for KDE4 only
+    KWALLET = auto()  # KDE4
     KWALLET5 = auto()
     KWALLET6 = auto()
-    GNOME_KEYRING = auto()
-    BASIC_TEXT = auto()
+    GNOMEKEYRING = auto()
+    BASICTEXT = auto()
 
 
 SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
@@ -803,7 +803,7 @@ def _choose_linux_keyring(logger):
     desktop_environment = _get_linux_desktop_environment(os.environ, logger)
     logger.debug(f'detected desktop environment: {desktop_environment.name}')
     if desktop_environment == _LinuxDesktopEnvironment.KDE4:
-        linux_keyring = _LinuxKeyring.KWALLET4
+        linux_keyring = _LinuxKeyring.KWALLET
     elif desktop_environment == _LinuxDesktopEnvironment.KDE5:
         linux_keyring = _LinuxKeyring.KWALLET5
     elif desktop_environment == _LinuxDesktopEnvironment.KDE6:
@@ -811,9 +811,9 @@ def _choose_linux_keyring(logger):
     elif desktop_environment in (
         _LinuxDesktopEnvironment.KDE3, _LinuxDesktopEnvironment.LXQT, _LinuxDesktopEnvironment.OTHER
     ):
-        linux_keyring = _LinuxKeyring.BASIC_TEXT
+        linux_keyring = _LinuxKeyring.BASICTEXT
     else:
-        linux_keyring = _LinuxKeyring.GNOME_KEYRING
+        linux_keyring = _LinuxKeyring.GNOMEKEYRING
     return linux_keyring
 
 
@@ -828,7 +828,7 @@ def _get_kwallet_network_wallet(keyring, logger):
     """
     default_wallet = 'kdewallet'
     try:
-        if keyring == _LinuxKeyring.KWALLET4:
+        if keyring == _LinuxKeyring.KWALLET:
             service_name = 'org.kde.kwalletd'
             wallet_path = '/modules/kwalletd'
         elif keyring == _LinuxKeyring.KWALLET5:
@@ -929,11 +929,11 @@ def _get_linux_keyring_password(browser_keyring_name, keyring, logger):
     keyring = _LinuxKeyring[keyring] if keyring else _choose_linux_keyring(logger)
     logger.debug(f'Chosen keyring: {keyring.name}')
 
-    if keyring in (_LinuxKeyring.KWALLET4, _LinuxKeyring.KWALLET5, _LinuxKeyring.KWALLET6):
+    if keyring in (_LinuxKeyring.KWALLET, _LinuxKeyring.KWALLET5, _LinuxKeyring.KWALLET6):
         return _get_kwallet_password(browser_keyring_name, keyring, logger)
-    elif keyring == _LinuxKeyring.GNOME_KEYRING:
+    elif keyring == _LinuxKeyring.GNOMEKEYRING:
         return _get_gnome_keyring_password(browser_keyring_name, logger)
-    elif keyring == _LinuxKeyring.BASIC_TEXT:
+    elif keyring == _LinuxKeyring.BASICTEXT:
         # when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
         return None
     assert False, f'Unknown keyring {keyring}'

From 97afb093d4cbe5df889145afa5f9ede4535e93e4 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 05:11:14 +0530
Subject: [PATCH 371/871] [extractor/youtube] Ignore wrong fps of some formats

---
 yt_dlp/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 4cbf0115c..4daa4f50e 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3849,6 +3849,7 @@ def build_fragments(f):
 
             client_name = fmt.get(STREAMING_DATA_CLIENT_NAME)
             name = fmt.get('qualityLabel') or quality.replace('audio_quality_', '') or ''
+            fps = int_or_none(fmt.get('fps')) or 0
             dct = {
                 'asr': int_or_none(fmt.get('audioSampleRate')),
                 'filesize': int_or_none(fmt.get('contentLength')),
@@ -3865,7 +3866,7 @@ def build_fragments(f):
                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
                 'source_preference': ((-10 if throttled else -5 if itag == '22' else -1)
                                       + (100 if 'Premium' in name else 0)),
-                'fps': int_or_none(fmt.get('fps')) or None,
+                'fps': fps if fps > 1 else None,  # For some formats, fps is wrongly returned as 1
                 'audio_channels': fmt.get('audioChannels'),
                 'height': height,
                 'quality': q(quality) - bool(fmt.get('isDrc')) / 2,
@@ -3936,6 +3937,8 @@ def process_manifest_format(f, proto, client_name, itag):
                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
             if self.get_param('verbose'):
                 f['format_note'] = join_nonempty(f.get('format_note'), client_name, delim=', ')
+            if f.get('fps') and f['fps'] <= 1:
+                del f['fps']
             return True
 
         subtitles = {}

From 93b39cdbd9dcf351bfa0c4ee252805b4617fdca9 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 04:31:39 +0530
Subject: [PATCH 372/871] Add `--compat-option playlist-match-filter`

Closes #6073
---
 README.md           | 7 ++++---
 yt_dlp/YoutubeDL.py | 2 +-
 yt_dlp/options.py   | 8 ++++----
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index c39e151d4..578f84956 100644
--- a/README.md
+++ b/README.md
@@ -157,14 +157,15 @@ ### Differences in default behavior
 * `certifi` will be used for SSL root certificates, if installed. If you want to use system certificates (e.g. self-signed), use `--compat-options no-certifi`
 * yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
 * yt-dlp tries to parse the external downloader outputs into the standard progress output if possible (Currently implemented: [~~aria2c~~](https://github.com/yt-dlp/yt-dlp/issues/5931)). You can use `--compat-options no-external-downloader-progress` to get the downloader output as-is
+* yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filter` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this
 
 For ease of use, a few more compat options are available:
 
 * `--compat-options all`: Use all compat options (Do NOT use)
-* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams`
-* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect`
+* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter`
+* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter`
 * `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date`
-* `--compat-options 2022`: Same as `--compat-options no-external-downloader-progress`. Use this to enable all future compat options
+* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress`. Use this to enable all future compat options
 
 
 # INSTALLATION
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index b4923920f..077a37b30 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1403,7 +1403,7 @@ def prepare_filename(self, info_dict, dir_type='', *, outtmpl=None, warn=False):
 
     def _match_entry(self, info_dict, incomplete=False, silent=False):
         """Returns None if the file should be downloaded"""
-        _type = info_dict.get('_type', 'video')
+        _type = 'video' if 'playlist-match-filter' in self.params['compat_opts'] else info_dict.get('_type', 'video')
         assert incomplete or _type == 'video', 'Only video result can be considered complete'
 
         video_title = info_dict.get('title', info_dict.get('id', 'entry'))
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index fecc27403..1c8d73f16 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -467,15 +467,15 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         callback_kwargs={
             'allowed_values': {
                 'filename', 'filename-sanitization', 'format-sort', 'abort-on-error', 'format-spec', 'no-playlist-metafiles',
-                'multistreams', 'no-live-chat', 'playlist-index', 'list-formats', 'no-direct-merge',
+                'multistreams', 'no-live-chat', 'playlist-index', 'list-formats', 'no-direct-merge', 'playlist-match-filter',
                 'no-attach-info-json', 'embed-thumbnail-atomicparsley', 'no-external-downloader-progress',
                 'embed-metadata', 'seperate-video-versions', 'no-clean-infojson', 'no-keep-subs', 'no-certifi',
                 'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-youtube-prefer-utc-upload-date',
             }, 'aliases': {
-                'youtube-dl': ['all', '-multistreams'],
-                'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat'],
+                'youtube-dl': ['all', '-multistreams', '-playlist-match-filter'],
+                'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter'],
                 '2021': ['2022', 'no-certifi', 'filename-sanitization', 'no-youtube-prefer-utc-upload-date'],
-                '2022': ['no-external-downloader-progress'],
+                '2022': ['no-external-downloader-progress', 'playlist-match-filter'],
             }
         }, help=(
             'Options that can help keep compatibility with youtube-dl or youtube-dlc '

From a35af4306d24c56c6358f89cdf204860d1cd62b4 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 05:18:03 +0530
Subject: [PATCH 373/871] [utils] `strftime_or_none`: Handle negative
 timestamps

Closes #6706
Authored by pukkandan, dirkf
---
 yt_dlp/utils/_utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 1fd6f44af..256e2db5a 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -2452,7 +2452,10 @@ def strftime_or_none(timestamp, date_format, default=None):
         if isinstance(timestamp, (int, float)):  # unix timestamp
             # Using naive datetime here can break timestamp() in Windows
             # Ref: https://github.com/yt-dlp/yt-dlp/issues/5185, https://github.com/python/cpython/issues/94414
-            datetime_object = datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc)
+            # Also, datetime.datetime.fromtimestamp breaks for negative timestamps
+            # Ref: https://github.com/yt-dlp/yt-dlp/issues/6706#issuecomment-1496842642
+            datetime_object = (datetime.datetime.fromtimestamp(0, datetime.timezone.utc)
+                               + datetime.timedelta(seconds=timestamp))
         elif isinstance(timestamp, str):  # assume YYYYMMDD
             datetime_object = datetime.datetime.strptime(timestamp, '%Y%m%d')
         date_format = re.sub(  # Support %s on windows

From ebe1b4e34f43c3acad30e4bcb8484681a030c114 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 05:15:03 +0530
Subject: [PATCH 374/871] [outtmpl] Fix some minor bugs

Closes #7164
---
 test/test_YoutubeDL.py |  2 +-
 yt_dlp/YoutubeDL.py    | 10 +++++-----
 yt_dlp/utils/_utils.py |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index ee6c52713..ccc9e36f3 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -755,7 +755,7 @@ def expect_same_infodict(out):
         test('%(id)d %(id)r', "1234 '1234'")
         test('%(id)r %(height)r', "'1234' 1080")
         test('%(ext)s-%(ext|def)d', 'mp4-def')
-        test('%(width|0)04d', '0000')
+        test('%(width|0)04d', '0')
         test('a%(width|b)d', 'ab', outtmpl_na_placeholder='none')
 
         FORMATS = self.outtmpl_info['formats']
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 077a37b30..a546ce65b 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1286,17 +1286,17 @@ def create_key(outer_mobj):
             if fmt == 's' and value is not None and key in field_size_compat_map.keys():
                 fmt = f'0{field_size_compat_map[key]:d}d'
 
-            if value is None:
-                value = default
-            elif replacement is not None:
+            if None not in (value, replacement):
                 try:
                     value = replacement_formatter.format(replacement, value)
                 except ValueError:
-                    value = na
+                    value, default = None, na
 
             flags = outer_mobj.group('conversion') or ''
             str_fmt = f'{fmt[:-1]}s'
-            if fmt[-1] == 'l':  # list
+            if value is None:
+                value, fmt = default, 's'
+            elif fmt[-1] == 'l':  # list
                 delim = '\n' if '#' in flags else ', '
                 value, fmt = delim.join(map(str, variadic(value, allowed_types=(str, bytes)))), str_fmt
             elif fmt[-1] == 'j':  # json
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 256e2db5a..d10d621d5 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -3302,7 +3302,7 @@ def q(qid):
 '''
 
 
-STR_FORMAT_TYPES = 'diouxXeEfFgGcrs'
+STR_FORMAT_TYPES = 'diouxXeEfFgGcrsa'
 
 
 def limit_length(s, length):

From 424f3bf03305088df6e01d62f7311be8601ad3f4 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 02:43:10 +0530
Subject: [PATCH 375/871] [downloader/fragment] Do not sleep between fragments

Closes #6599
---
 yt_dlp/downloader/fragment.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 53b4b604e..458167216 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -173,6 +173,9 @@ def _prepare_frag_download(self, ctx):
             **self.params,
             'noprogress': True,
             'test': False,
+            'sleep_interval': 0,
+            'max_sleep_interval': 0,
+            'sleep_interval_subtitles': 0,
         })
         tmpfilename = self.temp_name(ctx['filename'])
         open_mode = 'wb'

From 02948a17d903f544363bb20b51a6d8baed7bba08 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 04:12:01 +0530
Subject: [PATCH 376/871] [update] Do not restart into versions without
 `--update-to`

---
 yt_dlp/update.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index 6c9bdaf1c..4790075eb 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -149,7 +149,7 @@ def __init__(self, ydl, target=None):
                     f'You are switching to an {self.ydl._format_err("unofficial", "red")} executable '
                     f'from {self.ydl._format_err(self._target_repo, self.ydl.Styles.EMPHASIS)}. '
                     f'Run {self.ydl._format_err("at your own risk", "light red")}')
-                self.restart = self._blocked_restart
+                self._block_restart('Automatically restarting into custom builds is disabled for security reasons')
         else:
             self._target_repo = UPDATE_SOURCES.get(self.target_channel)
             if not self._target_repo:
@@ -294,6 +294,7 @@ def update(self):
         if (_VERSION_RE.fullmatch(self.target_tag[5:])
                 and version_tuple(self.target_tag[5:]) < (2023, 3, 2)):
             self.ydl.report_warning('You are downgrading to a version without --update-to')
+            self._block_restart('Cannot automatically restart to a version without --update-to')
 
         directory = os.path.dirname(self.filename)
         if not os.access(self.filename, os.W_OK):
@@ -381,11 +382,11 @@ def restart(self):
         _, _, returncode = Popen.run(self.cmd)
         return returncode
 
-    def _blocked_restart(self):
-        self._report_error(
-            'Automatically restarting into custom builds is disabled for security reasons. '
-            'Restart yt-dlp to use the updated version', expected=True)
-        return self.ydl._download_retcode
+    def _block_restart(self, msg):
+        def wrapper():
+            self._report_error(f'{msg}. Restart yt-dlp to use the updated version', expected=True)
+            return self.ydl._download_retcode
+        self.restart = wrapper
 
 
 def run_update(ydl):

From af7585c824a1e405bd8afa46d87b4be322edc93c Mon Sep 17 00:00:00 2001
From: MMM <flashdagger@googlemail.com>
Date: Wed, 21 Jun 2023 04:44:12 +0200
Subject: [PATCH 377/871] [extractor/tagesschau] Fix single audio urls (#6626)

Authored by: flashdagger
---
 yt_dlp/extractor/tagesschau.py | 58 +++++++++++++++++++++-------------
 1 file changed, 36 insertions(+), 22 deletions(-)

diff --git a/yt_dlp/extractor/tagesschau.py b/yt_dlp/extractor/tagesschau.py
index ea0532c24..e23b490b0 100644
--- a/yt_dlp/extractor/tagesschau.py
+++ b/yt_dlp/extractor/tagesschau.py
@@ -2,10 +2,12 @@
 
 from .common import InfoExtractor
 from ..utils import (
-    js_to_json,
+    UnsupportedError,
     extract_attributes,
-    try_get,
     int_or_none,
+    js_to_json,
+    parse_iso8601,
+    try_get,
 )
 
 
@@ -14,36 +16,38 @@ class TagesschauIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
-        'md5': '7a7287612fa881a1ae1d087df45c2fd6',
+        'md5': 'ccb9359bf8c4795836e43759f3408a93',
         'info_dict': {
             'id': 'video-102143-1',
             'ext': 'mp4',
             'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
+            'duration': 138,
         },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
-        'md5': '3c54c1f6243d279b706bde660ceec633',
+        'md5': '5c15e8f3da049e48829ec9786d835536',
         'info_dict': {
             'id': 'ts-5727-1',
             'ext': 'mp4',
             'title': 'Ganze Sendung',
+            'duration': 932,
         },
     }, {
         # exclusive audio
         'url': 'http://www.tagesschau.de/multimedia/audio/audio-29417.html',
-        'md5': '4cf22023c285f35e99c24d290ba58cc9',
+        'md5': '4bff8f23504df56a0d86ed312d654182',
         'info_dict': {
             'id': 'audio-29417-1',
             'ext': 'mp3',
-            'title': 'Brasilianischer Präsident Bolsonaro unter Druck: Corona-Bericht wird vorgestellt',
+            'title': 'EU-Gipfel: Im Verbrennerstreit hat Deutschland maximalen Schaden angerichtet',
         },
     }, {
         'url': 'http://www.tagesschau.de/inland/bnd-303.html',
-        'md5': '12cfb212d9325b5ba0d52b625f1aa61c',
+        'md5': 'f049fa1698d7564e9ca4c3325108f034',
         'info_dict': {
             'id': 'bnd-303-1',
-            'ext': 'mp4',
-            'title': 'SPD-Gruppenbild mit Bärbel Bas nach der Fraktionssitzung | dpa',
+            'ext': 'mp3',
+            'title': 'Das Siegel des Bundesnachrichtendienstes | dpa',
         },
     }, {
         'url': 'http://www.tagesschau.de/inland/afd-parteitag-135.html',
@@ -51,13 +55,24 @@ class TagesschauIE(InfoExtractor):
             'id': 'afd-parteitag-135',
             'title': 'AfD',
         },
-        'playlist_count': 20,
+        'playlist_mincount': 15,
     }, {
         'url': 'https://www.tagesschau.de/multimedia/audio/audio-29417~player.html',
         'info_dict': {
             'id': 'audio-29417-1',
             'ext': 'mp3',
-            'title': 'Brasilianischer Präsident Bolsonaro unter Druck: Corona-Bericht wird vorgestellt',
+            'title': 'EU-Gipfel: Im Verbrennerstreit hat Deutschland maximalen Schaden angerichtet',
+        },
+    }, {
+        'url': 'https://www.tagesschau.de/multimedia/audio/podcast-11km-327.html',
+        'info_dict': {
+            'id': 'podcast-11km-327',
+            'ext': 'mp3',
+            'title': 'Gewalt in der Kita – Wenn Erzieher:innen schweigen',
+            'upload_date': '20230322',
+            'timestamp': 1679482808,
+            'thumbnail': 'https://www.tagesschau.de/multimedia/audio/podcast-11km-329~_v-original.jpg',
+            'description': 'md5:dad059931fe4b3693e3656e93a249848',
         },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
@@ -117,7 +132,7 @@ def _real_extract(self, url):
                 formats = []
                 if media_url.endswith('master.m3u8'):
                     formats = self._extract_m3u8_formats(media_url, video_id, 'mp4', m3u8_id='hls')
-                elif media_url.endswith('.hi.mp3') and media_url.startswith('https://download'):
+                elif media_url.endswith('.mp3'):
                     formats = [{
                         'url': media_url,
                         'vcodec': 'none',
@@ -130,20 +145,19 @@ def _real_extract(self, url):
                     'duration': int_or_none(try_get(video, lambda x: x['mc']['_duration'])),
                     'formats': formats
                 })
+
+        if not entries:
+            raise UnsupportedError(url)
+
         if len(entries) > 1:
             return self.playlist_result(entries, display_id, title)
-        formats = entries[0]['formats']
-        video_info = self._search_json_ld(webpage, video_id)
-        description = video_info.get('description')
-        thumbnail = self._og_search_thumbnail(webpage) or video_info.get('thumbnail')
-        timestamp = video_info.get('timestamp')
-        title = title or video_info.get('description')
 
         return {
             'id': display_id,
             'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-            'timestamp': timestamp,
-            'description': description,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'formats': entries[0]['formats'],
+            'timestamp': parse_iso8601(self._html_search_meta('date', webpage)),
+            'description': self._og_search_description(webpage),
+            'duration': entries[0]['duration'],
         }

From db3ad8a67661d7b234a6954d9c6a4a9b1749f5eb Mon Sep 17 00:00:00 2001
From: Nicolai Dagestad <nicolai.github@dagestad.fr>
Date: Wed, 21 Jun 2023 05:07:42 +0200
Subject: [PATCH 378/871] Add option `--netrc-cmd` (#6682)

Authored by: NDagestad, pukkandan
Closes #1706
---
 README.md                  | 15 +++++++++--
 yt_dlp/YoutubeDL.py        |  1 +
 yt_dlp/__init__.py         |  5 ++--
 yt_dlp/extractor/common.py | 53 +++++++++++++++++++++-----------------
 yt_dlp/options.py          |  4 +++
 yt_dlp/utils/_utils.py     |  8 ++++++
 6 files changed, 58 insertions(+), 28 deletions(-)

diff --git a/README.md b/README.md
index 578f84956..9a00da903 100644
--- a/README.md
+++ b/README.md
@@ -49,7 +49,7 @@
     * [Extractor Options](#extractor-options)
 * [CONFIGURATION](#configuration)
     * [Configuration file encoding](#configuration-file-encoding)
-    * [Authentication with .netrc file](#authentication-with-netrc-file)
+    * [Authentication with netrc](#authentication-with-netrc)
     * [Notes about environment variables](#notes-about-environment-variables)
 * [OUTPUT TEMPLATE](#output-template)
     * [Output template examples](#output-template-examples)
@@ -910,6 +910,8 @@ ## Authentication Options:
     --netrc-location PATH           Location of .netrc authentication data;
                                     either the path or its containing directory.
                                     Defaults to ~/.netrc
+    --netrc-cmd NETRC_CMD           Command to execute to get the credentials
+                                    credentials for an extractor.
     --video-password PASSWORD       Video password (vimeo, youku)
     --ap-mso MSO                    Adobe Pass multiple-system operator (TV
                                     provider) identifier, use --ap-list-mso for
@@ -1203,7 +1205,7 @@ ### Configuration file encoding
 
 If you want your file to be decoded differently, add `# coding: ENCODING` to the beginning of the file (e.g. `# coding: shift-jis`). There must be no characters before that, even spaces or BOM.
 
-### Authentication with `.netrc` file
+### Authentication with netrc
 
 You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every yt-dlp execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](https://stackoverflow.com/tags/.netrc/info) on a per-extractor basis. For that you will need to create a `.netrc` file in `--netrc-location` and restrict permissions to read/write by only you:
 ```
@@ -1223,6 +1225,15 @@ ### Authentication with `.netrc` file
 
 The default location of the .netrc file is `~` (see below).
 
+As an alternative to using the `.netrc` file, which has the disadvantage of keeping your passwords in a plain text file, you can configure a custom shell command to provide the credentials for an extractor. This is done by providing the `--netrc-cmd` parameter, it shall output the credentials in the netrc format and return `0` on success, other values will be treated as an error. `{}` in the command will be replaced by the name of the extractor to make it possible to select the credentials for the right extractor.
+To use braces in the command, they need to be escaped by doubling them. (see example bellow)
+
+E.g. To use an encrypted `.netrc` file stored as `.authinfo.gpg`
+```
+yt-dlp --netrc-cmd 'gpg --decrypt ~/.authinfo.gpg' https://www.youtube.com/watch?v=BaW_jenozKc
+```
+
+
 ### Notes about environment variables
 * Environment variables are normally specified as `${VARIABLE}`/`$VARIABLE` on UNIX and `%VARIABLE%` on Windows; but is always shown as `${VARIABLE}` in this documentation
 * yt-dlp also allow using UNIX-style variables on Windows for path-like options; e.g. `--output`, `--config-location`
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index a546ce65b..e51bceef3 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -190,6 +190,7 @@ class YoutubeDL:
     ap_password:       Multiple-system operator account password.
     usenetrc:          Use netrc for authentication instead.
     netrc_location:    Location of the netrc file. Defaults to ~/.netrc.
+    netrc_cmd:         Use a shell command to get credentials
     verbose:           Print additional info to stdout.
     quiet:             Do not print messages to stdout.
     no_warnings:       Do not print out anything for warnings.
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 137c9503f..46edd88d3 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -188,8 +188,8 @@ def validate_minmax(min_val, max_val, min_name, max_name=None):
         raise ValueError(f'{max_name} "{max_val}" must be must be greater than or equal to {min_name} "{min_val}"')
 
     # Usernames and passwords
-    validate(not opts.usenetrc or (opts.username is None and opts.password is None),
-             '.netrc', msg='using {name} conflicts with giving username/password')
+    validate(sum(map(bool, (opts.usenetrc, opts.netrc_cmd, opts.username))) <= 1, '.netrc',
+             msg='{name}, netrc command and username/password are mutually exclusive options')
     validate(opts.password is None or opts.username is not None, 'account username', msg='{name} missing')
     validate(opts.ap_password is None or opts.ap_username is not None,
              'TV Provider account username', msg='{name} missing')
@@ -741,6 +741,7 @@ def parse_options(argv=None):
     return ParsedOptions(parser, opts, urls, {
         'usenetrc': opts.usenetrc,
         'netrc_location': opts.netrc_location,
+        'netrc_cmd': opts.netrc_cmd,
         'username': opts.username,
         'password': opts.password,
         'twofactor': opts.twofactor,
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index ca2164a5d..f11a67358 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -13,6 +13,7 @@
 import os
 import random
 import re
+import subprocess
 import sys
 import time
 import types
@@ -34,6 +35,7 @@
     GeoUtils,
     HEADRequest,
     LenientJSONDecoder,
+    Popen,
     RegexNotFoundError,
     RetryManager,
     UnsupportedError,
@@ -70,6 +72,7 @@
     smuggle_url,
     str_or_none,
     str_to_int,
+    netrc_from_content,
     strip_or_none,
     traverse_obj,
     truncate_string,
@@ -535,7 +538,7 @@ class InfoExtractor:
     _EMBED_REGEX = []
 
     def _login_hint(self, method=NO_DEFAULT, netrc=None):
-        password_hint = f'--username and --password, or --netrc ({netrc or self._NETRC_MACHINE}) to provide account credentials'
+        password_hint = f'--username and --password, --netrc-cmd, or --netrc ({netrc or self._NETRC_MACHINE}) to provide account credentials'
         return {
             None: '',
             'any': f'Use --cookies, --cookies-from-browser, {password_hint}',
@@ -1291,45 +1294,47 @@ def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=Tr
         return clean_html(res)
 
     def _get_netrc_login_info(self, netrc_machine=None):
-        username = None
-        password = None
         netrc_machine = netrc_machine or self._NETRC_MACHINE
 
-        if self.get_param('usenetrc', False):
-            try:
-                netrc_file = compat_expanduser(self.get_param('netrc_location') or '~')
-                if os.path.isdir(netrc_file):
-                    netrc_file = os.path.join(netrc_file, '.netrc')
-                info = netrc.netrc(file=netrc_file).authenticators(netrc_machine)
-                if info is not None:
-                    username = info[0]
-                    password = info[2]
-                else:
-                    raise netrc.NetrcParseError(
-                        'No authenticators for %s' % netrc_machine)
-            except (OSError, netrc.NetrcParseError) as err:
-                self.report_warning(
-                    'parsing .netrc: %s' % error_to_compat_str(err))
+        cmd = self.get_param('netrc_cmd', '').format(netrc_machine)
+        if cmd:
+            self.to_screen(f'Executing command: {cmd}')
+            stdout, _, ret = Popen.run(cmd, text=True, shell=True, stdout=subprocess.PIPE)
+            if ret != 0:
+                raise OSError(f'Command returned error code {ret}')
+            info = netrc_from_content(stdout).authenticators(netrc_machine)
 
-        return username, password
+        elif self.get_param('usenetrc', False):
+            netrc_file = compat_expanduser(self.get_param('netrc_location') or '~')
+            if os.path.isdir(netrc_file):
+                netrc_file = os.path.join(netrc_file, '.netrc')
+            info = netrc.netrc(netrc_file).authenticators(netrc_machine)
+
+        else:
+            return None, None
+        if not info:
+            raise netrc.NetrcParseError(f'No authenticators for {netrc_machine}')
+        return info[0], info[2]
 
     def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None):
         """
         Get the login info as (username, password)
         First look for the manually specified credentials using username_option
         and password_option as keys in params dictionary. If no such credentials
-        available look in the netrc file using the netrc_machine or _NETRC_MACHINE
-        value.
+        are available try the netrc_cmd if it is defined or look in the
+        netrc file using the netrc_machine or _NETRC_MACHINE value.
         If there's no info available, return (None, None)
         """
 
-        # Attempt to use provided username and password or .netrc data
         username = self.get_param(username_option)
         if username is not None:
             password = self.get_param(password_option)
         else:
-            username, password = self._get_netrc_login_info(netrc_machine)
-
+            try:
+                username, password = self._get_netrc_login_info(netrc_machine)
+            except (OSError, netrc.NetrcParseError) as err:
+                self.report_warning(f'Failed to parse .netrc: {err}')
+                return None, None
         return username, password
 
     def _get_tfa_info(self, note='two-factor verification code'):
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 1c8d73f16..b174a24af 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -720,6 +720,10 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         '--netrc-location',
         dest='netrc_location', metavar='PATH',
         help='Location of .netrc authentication data; either the path or its containing directory. Defaults to ~/.netrc')
+    authentication.add_option(
+        '--netrc-cmd',
+        dest='netrc_cmd', metavar='NETRC_CMD',
+        help='Command to execute to get the credentials for an extractor.')
     authentication.add_option(
         '--video-password',
         dest='videopassword', metavar='PASSWORD',
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index d10d621d5..28c2785cb 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -25,6 +25,7 @@
 import locale
 import math
 import mimetypes
+import netrc
 import operator
 import os
 import platform
@@ -864,6 +865,13 @@ def escapeHTML(text):
     )
 
 
+class netrc_from_content(netrc.netrc):
+    def __init__(self, content):
+        self.hosts, self.macros = {}, {}
+        with io.StringIO(content) as stream:
+            self._parse('-', stream, False)
+
+
 def process_communicate_or_kill(p, *args, **kwargs):
     deprecation_warning(f'"{__name__}.process_communicate_or_kill" is deprecated and may be removed '
                         f'in a future version. Use "{__name__}.Popen.communicate_or_kill" instead')

From ad54c9130e793ce433bf9da334fa80df9f3aee58 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 09:21:20 +0530
Subject: [PATCH 379/871] [cleanup] Misc

Closes #6288, Closes #7197, Closes #7265, Closes #7353, Closes #5773
Authored by: mikf, freezboltz, pukkandan
---
 .github/workflows/potential-duplicates.yml |  2 +-
 README.md                                  | 28 +++++----
 devscripts/changelog_override.json         | 27 +++++++++
 devscripts/cli_to_api.py                   |  4 +-
 devscripts/make_changelog.py               | 14 ++---
 test/test_YoutubeDL.py                     | 16 +++---
 test/test_jsinterp.py                      | 67 ++++++++++++----------
 test/test_youtube_signature.py             |  2 +-
 yt_dlp/YoutubeDL.py                        | 20 ++++---
 yt_dlp/cookies.py                          |  4 ++
 yt_dlp/downloader/common.py                |  1 -
 yt_dlp/downloader/niconico.py              |  4 +-
 yt_dlp/extractor/ciscowebex.py             |  4 +-
 yt_dlp/extractor/common.py                 |  3 +-
 yt_dlp/extractor/dumpert.py                |  0
 yt_dlp/extractor/globalplayer.py           |  0
 yt_dlp/extractor/odnoklassniki.py          |  6 +-
 yt_dlp/extractor/tvp.py                    |  4 +-
 yt_dlp/extractor/vidio.py                  |  2 +-
 yt_dlp/extractor/youtube.py                | 10 ++--
 yt_dlp/options.py                          |  6 +-
 yt_dlp/utils/_legacy.py                    |  6 +-
 yt_dlp/utils/_utils.py                     | 10 +---
 23 files changed, 138 insertions(+), 102 deletions(-)
 mode change 100755 => 100644 yt_dlp/extractor/dumpert.py
 mode change 100755 => 100644 yt_dlp/extractor/globalplayer.py

diff --git a/.github/workflows/potential-duplicates.yml b/.github/workflows/potential-duplicates.yml
index 1521ae20c..cfc583186 100644
--- a/.github/workflows/potential-duplicates.yml
+++ b/.github/workflows/potential-duplicates.yml
@@ -12,7 +12,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           label: potential-duplicate
           state: all
-          threshold: 0.7
+          threshold: 0.3
           comment: |
             This issue is potentially a duplicate of one of the following issues:
             {{#issues}}
diff --git a/README.md b/README.md
index 9a00da903..d9a5e6cef 100644
--- a/README.md
+++ b/README.md
@@ -152,7 +152,7 @@ ### Differences in default behavior
 * The upload dates extracted from YouTube are in UTC [when available](https://github.com/yt-dlp/yt-dlp/blob/89e4d86171c7b7c997c77d4714542e0383bf0db0/yt_dlp/extractor/youtube.py#L3898-L3900). Use `--compat-options no-youtube-prefer-utc-upload-date` to prefer the non-UTC upload date.
 * If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this
 * Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead
-* Some private fields such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this
+* Some internal metadata such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this
 * When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the separate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this
 * `certifi` will be used for SSL root certificates, if installed. If you want to use system certificates (e.g. self-signed), use `--compat-options no-certifi`
 * yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
@@ -251,7 +251,7 @@ #### Misc
 ```
 <!-- MANPAGE: END EXCLUDED SECTION -->
 
-**Note**: The manpages, shell completion files etc. are available inside the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
+**Note**: The manpages, shell completion (autocomplete) files etc. are available inside the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
 
 ## DEPENDENCIES
 Python versions 3.7+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly.
@@ -699,9 +699,8 @@ ## Filesystem Options:
                                     --write-description etc. (default)
     --no-write-playlist-metafiles   Do not write playlist metadata when using
                                     --write-info-json, --write-description etc.
-    --clean-info-json               Remove some private fields such as filenames
-                                    from the infojson. Note that it could still
-                                    contain some personal information (default)
+    --clean-info-json               Remove some internal metadata such as
+                                    filenames from the infojson (default)
     --no-clean-info-json            Write all fields to the infojson
     --write-comments                Retrieve video comments to be placed in the
                                     infojson. The comments are fetched even
@@ -1041,13 +1040,10 @@ ## Post-Processing Options:
                                     that of --use-postprocessor (default:
                                     after_move). Same syntax as the output
                                     template can be used to pass any field as
-                                    arguments to the command. After download, an
-                                    additional field "filepath" that contains
-                                    the final path of the downloaded file is
-                                    also available, and if no fields are passed,
-                                    %(filepath,_filename|)q is appended to the
-                                    end of the command. This option can be used
-                                    multiple times
+                                    arguments to the command. If no fields are
+                                    passed, %(filepath,_filename|)q is appended
+                                    to the end of the command. This option can
+                                    be used multiple times
     --no-exec                       Remove any previously defined --exec
     --convert-subs FORMAT           Convert the subtitles to another format
                                     (currently supported: ass, lrc, srt, vtt)
@@ -1225,8 +1221,7 @@ ### Authentication with netrc
 
 The default location of the .netrc file is `~` (see below).
 
-As an alternative to using the `.netrc` file, which has the disadvantage of keeping your passwords in a plain text file, you can configure a custom shell command to provide the credentials for an extractor. This is done by providing the `--netrc-cmd` parameter, it shall output the credentials in the netrc format and return `0` on success, other values will be treated as an error. `{}` in the command will be replaced by the name of the extractor to make it possible to select the credentials for the right extractor.
-To use braces in the command, they need to be escaped by doubling them. (see example bellow)
+As an alternative to using the `.netrc` file, which has the disadvantage of keeping your passwords in a plain text file, you can configure a custom shell command to provide the credentials for an extractor. This is done by providing the `--netrc-cmd` parameter, it shall output the credentials in the netrc format and return `0` on success, other values will be treated as an error. `{}` in the command will be replaced by the name of the extractor to make it possible to select the credentials for the right extractor (To use literal braces, double them like `{{}}`).
 
 E.g. To use an encrypted `.netrc` file stored as `.authinfo.gpg`
 ```
@@ -1389,7 +1384,10 @@ # OUTPUT TEMPLATE
  - `subtitles_table` (table): The subtitle format table as printed by `--list-subs`
  - `automatic_captions_table` (table): The automatic subtitle format table as printed by `--list-subs`
  
+ Available only after the video is downloaded (`post_process`/`after_move`):
  
+ - `filepath`: Actual path of downloaded video file
+
 Available only in `--sponsorblock-chapter-title`:
 
  - `start_time` (numeric): Start time of the chapter in seconds
@@ -1435,7 +1433,7 @@ # Download YouTube playlist videos in separate directories according to their up
 $ yt-dlp -o "%(upload_date>%Y)s/%(title)s.%(ext)s" "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re"
 
 # Prefix playlist index with " - " separator, but only if it is available
-$ yt-dlp -o '%(playlist_index|)s%(playlist_index& - |)s%(title)s.%(ext)s' BaW_jenozKc "https://www.youtube.com/user/TheLinuxFoundation/playlists"
+$ yt-dlp -o "%(playlist_index&{} - |)s%(title)s.%(ext)s" BaW_jenozKc "https://www.youtube.com/user/TheLinuxFoundation/playlists"
 
 # Download all playlists of YouTube channel/user keeping each playlist in separate directory:
 $ yt-dlp -o "%(uploader)s/%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s" "https://www.youtube.com/user/TheLinuxFoundation/playlists"
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index e5c9d1aa2..73225bdb9 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -8,5 +8,32 @@
         "action": "add",
         "when": "776d1c3f0c9b00399896dd2e40e78e9a43218109",
         "short": "[priority] **YouTube throttling fixes!**"
+    },
+    {
+        "action": "remove",
+        "when": "2e023649ea4e11151545a34dc1360c114981a236"
+    },
+    {
+        "action": "add",
+        "when": "01aba2519a0884ef17d5f85608dbd2a455577147",
+        "short": "[priority] YouTube: Improved throttling and signature fixes"
+    },
+    {
+        "action": "change",
+        "when": "c86e433c35fe5da6cb29f3539eef97497f84ed38",
+        "short": "[extractor/niconico:series] Fix extraction (#6898)",
+        "authors": ["sqrtNOT"]
+    },
+    {
+        "action": "change",
+        "when": "69a40e4a7f6caa5662527ebd2f3c4e8aa02857a2",
+        "short": "[extractor/youtube:music_search_url] Extract title (#7102)",
+        "authors": ["kangalio"]
+    },
+    {
+        "action": "change",
+        "when": "8417f26b8a819cd7ffcd4e000ca3e45033e670fb",
+        "short": "Add option `--color` (#6904)",
+        "authors": ["Grub4K"]
     }
 ]
diff --git a/devscripts/cli_to_api.py b/devscripts/cli_to_api.py
index b8b7cbcf1..2aa51eb6e 100644
--- a/devscripts/cli_to_api.py
+++ b/devscripts/cli_to_api.py
@@ -19,11 +19,11 @@ def parse_patched_options(opts):
         'extract_flat': False,
         'concat_playlist': 'never',
     })
-    yt_dlp.options.__dict__['create_parser'] = lambda: patched_parser
+    yt_dlp.options.create_parser = lambda: patched_parser
     try:
         return yt_dlp.parse_options(opts)
     finally:
-        yt_dlp.options.__dict__['create_parser'] = create_parser
+        yt_dlp.options.create_parser = create_parser
 
 
 default_opts = parse_patched_options([]).ydl_opts
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index 1b7e251ee..2fcdc06d7 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -44,7 +44,7 @@ def commit_lookup(cls):
         return {
             name: group
             for group, names in {
-                cls.PRIORITY: {''},
+                cls.PRIORITY: {'priority'},
                 cls.CORE: {
                     'aes',
                     'cache',
@@ -68,7 +68,7 @@ def commit_lookup(cls):
                     'misc',
                     'test',
                 },
-                cls.EXTRACTOR: {'extractor', 'extractors'},
+                cls.EXTRACTOR: {'extractor'},
                 cls.DOWNLOADER: {'downloader'},
                 cls.POSTPROCESSOR: {'postprocessor'},
             }.items()
@@ -323,7 +323,7 @@ def apply_overrides(self, overrides):
                 logger.debug(f'Ignored {when!r}, not in commits {self._start!r}')
                 continue
 
-            override_hash = override.get('hash')
+            override_hash = override.get('hash') or when
             if override['action'] == 'add':
                 commit = Commit(override.get('hash'), override['short'], override.get('authors') or [])
                 logger.info(f'ADD    {commit}')
@@ -337,7 +337,7 @@ def apply_overrides(self, overrides):
             elif override['action'] == 'change':
                 if override_hash not in self._commits:
                     continue
-                commit = Commit(override_hash, override['short'], override['authors'])
+                commit = Commit(override_hash, override['short'], override.get('authors') or [])
                 logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}')
                 self._commits[commit.hash] = commit
 
@@ -348,7 +348,7 @@ def groups(self):
         for commit in self:
             upstream_re = self.UPSTREAM_MERGE_RE.search(commit.short)
             if upstream_re:
-                commit.short = f'[upstream] Merged with youtube-dl {upstream_re.group(1)}'
+                commit.short = f'[core/upstream] Merged with youtube-dl {upstream_re.group(1)}'
 
             match = self.MESSAGE_RE.fullmatch(commit.short)
             if not match:
@@ -394,10 +394,10 @@ def details_from_prefix(prefix):
             return CommitGroup.CORE, None, ()
 
         prefix, _, details = prefix.partition('/')
-        prefix = prefix.strip().lower()
+        prefix = prefix.strip()
         details = details.strip()
 
-        group = CommitGroup.get(prefix)
+        group = CommitGroup.get(prefix.lower())
         if group is CommitGroup.PRIORITY:
             prefix, _, details = details.partition('/')
 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index ccc9e36f3..05dd3ed41 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -668,7 +668,7 @@ def test(tmpl, expected, *, info=None, **params):
             for (name, got), expect in zip((('outtmpl', out), ('filename', fname)), expected):
                 if callable(expect):
                     self.assertTrue(expect(got), f'Wrong {name} from {tmpl}')
-                else:
+                elif expect is not None:
                     self.assertEqual(got, expect, f'Wrong {name} from {tmpl}')
 
         # Side-effects
@@ -759,15 +759,17 @@ def expect_same_infodict(out):
         test('a%(width|b)d', 'ab', outtmpl_na_placeholder='none')
 
         FORMATS = self.outtmpl_info['formats']
-        sanitize = lambda x: x.replace(':', '：').replace('"', "＂").replace('\n', ' ')
 
         # Custom type casting
         test('%(formats.:.id)l', 'id 1, id 2, id 3')
         test('%(formats.:.id)#l', ('id 1\nid 2\nid 3', 'id 1 id 2 id 3'))
         test('%(ext)l', 'mp4')
         test('%(formats.:.id) 18l', '  id 1, id 2, id 3')
-        test('%(formats)j', (json.dumps(FORMATS), sanitize(json.dumps(FORMATS))))
-        test('%(formats)#j', (json.dumps(FORMATS, indent=4), sanitize(json.dumps(FORMATS, indent=4))))
+        test('%(formats)j', (json.dumps(FORMATS), None))
+        test('%(formats)#j', (
+            json.dumps(FORMATS, indent=4),
+            json.dumps(FORMATS, indent=4).replace(':', '：').replace('"', "＂").replace('\n', ' ')
+        ))
         test('%(title5).3B', 'á')
         test('%(title5)U', 'áéí 𝐀')
         test('%(title5)#U', 'a\u0301e\u0301i\u0301 𝐀')
@@ -792,8 +794,8 @@ def expect_same_infodict(out):
         test('%(title|%)s %(title|%%)s', '% %%')
         test('%(id+1-height+3)05d', '00158')
         test('%(width+100)05d', 'NA')
-        test('%(formats.0) 15s', ('% 15s' % FORMATS[0], '% 15s' % sanitize(str(FORMATS[0]))))
-        test('%(formats.0)r', (repr(FORMATS[0]), sanitize(repr(FORMATS[0]))))
+        test('%(formats.0) 15s', ('% 15s' % FORMATS[0], None))
+        test('%(formats.0)r', (repr(FORMATS[0]), None))
         test('%(height.0)03d', '001')
         test('%(-height.0)04d', '-001')
         test('%(formats.-1.id)s', FORMATS[-1]['id'])
@@ -805,7 +807,7 @@ def expect_same_infodict(out):
         out = json.dumps([{'id': f['id'], 'height.:2': str(f['height'])[:2]}
                           if 'height' in f else {'id': f['id']}
                           for f in FORMATS])
-        test('%(formats.:.{id,height.:2})j', (out, sanitize(out)))
+        test('%(formats.:.{id,height.:2})j', (out, None))
         test('%(formats.:.{id,height}.id)l', ', '.join(f['id'] for f in FORMATS))
         test('%(.{id,title})j', ('{"id": "1234"}', '{＂id＂： ＂1234＂}'))
 
diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index b01477e6f..e9682ddab 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -12,28 +12,38 @@
 from yt_dlp.jsinterp import JS_Undefined, JSInterpreter
 
 
+class NaN:
+    pass
+
+
 class TestJSInterpreter(unittest.TestCase):
-    def _test(self, code, ret, func='f', args=()):
-        self.assertEqual(JSInterpreter(code).call_function(func, *args), ret)
+    def _test(self, jsi_or_code, expected, func='f', args=()):
+        if isinstance(jsi_or_code, str):
+            jsi_or_code = JSInterpreter(jsi_or_code)
+        got = jsi_or_code.call_function(func, *args)
+        if expected is NaN:
+            self.assertTrue(math.isnan(got), f'{got} is not NaN')
+        else:
+            self.assertEqual(got, expected)
 
     def test_basic(self):
         jsi = JSInterpreter('function f(){;}')
         self.assertEqual(repr(jsi.extract_function('f')), 'F<f>')
-        self.assertEqual(jsi.call_function('f'), None)
+        self._test(jsi, None)
 
         self._test('function f(){return 42;}', 42)
         self._test('function f(){42}', None)
         self._test('var f = function(){return 42;}', 42)
 
-    def test_calc(self):
-        self._test('function f(a){return 2*a+1;}', 7, args=[3])
-
     def test_div(self):
         jsi = JSInterpreter('function f(a, b){return a / b;}')
-        self.assertTrue(math.isnan(jsi.call_function('f', 0, 0)))
-        self.assertTrue(math.isnan(jsi.call_function('f', JS_Undefined, 1)))
-        self.assertTrue(math.isinf(jsi.call_function('f', 2, 0)))
-        self.assertEqual(jsi.call_function('f', 0, 3), 0)
+        self._test(jsi, NaN, args=(0, 0))
+        self._test(jsi, NaN, args=(JS_Undefined, 1))
+        self._test(jsi, float('inf'), args=(2, 0))
+        self._test(jsi, 0, args=(0, 3))
+
+    def test_calc(self):
+        self._test('function f(a){return 2*a+1;}', 7, args=[3])
 
     def test_empty_return(self):
         self._test('function f(){return; y()}', None)
@@ -102,16 +112,15 @@ def test_precedence(self):
         ''', [20, 20, 30, 40, 50])
 
     def test_builtins(self):
-        jsi = JSInterpreter('function f() { return NaN }')
-        self.assertTrue(math.isnan(jsi.call_function('f')))
+        self._test('function f() { return NaN }', NaN)
 
     def test_date(self):
         self._test('function f() { return new Date("Wednesday 31 December 1969 18:01:26 MDT") - 0; }', 86000)
 
         jsi = JSInterpreter('function f(dt) { return new Date(dt) - 0; }')
-        self.assertEqual(jsi.call_function('f', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
-        self.assertEqual(jsi.call_function('f', '12/31/1969 18:01:26 MDT'), 86000)  # m/d/y
-        self.assertEqual(jsi.call_function('f', '1 January 1970 00:00:00 UTC'), 0)
+        self._test(jsi, 86000, args=['Wednesday 31 December 1969 18:01:26 MDT'])
+        self._test(jsi, 86000, args=['12/31/1969 18:01:26 MDT'])  # m/d/y
+        self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC'])
 
     def test_call(self):
         jsi = JSInterpreter('''
@@ -119,8 +128,8 @@ def test_call(self):
             function y(a) { return x() + (a?a:0); }
             function z() { return y(3); }
         ''')
-        self.assertEqual(jsi.call_function('z'), 5)
-        self.assertEqual(jsi.call_function('y'), 2)
+        self._test(jsi, 5, func='z')
+        self._test(jsi, 2, func='y')
 
     def test_if(self):
         self._test('''
@@ -167,9 +176,9 @@ def test_switch(self):
                 default:x=0;
             } return x }
         ''')
-        self.assertEqual(jsi.call_function('f', 1), 7)
-        self.assertEqual(jsi.call_function('f', 3), 6)
-        self.assertEqual(jsi.call_function('f', 5), 0)
+        self._test(jsi, 7, args=[1])
+        self._test(jsi, 6, args=[3])
+        self._test(jsi, 0, args=[5])
 
     def test_switch_default(self):
         jsi = JSInterpreter('''
@@ -182,9 +191,9 @@ def test_switch_default(self):
                 case 1: x+=1;
             } return x }
         ''')
-        self.assertEqual(jsi.call_function('f', 1), 2)
-        self.assertEqual(jsi.call_function('f', 5), 11)
-        self.assertEqual(jsi.call_function('f', 9), 14)
+        self._test(jsi, 2, args=[1])
+        self._test(jsi, 11, args=[5])
+        self._test(jsi, 14, args=[9])
 
     def test_try(self):
         self._test('function f() { try{return 10} catch(e){return 5} }', 10)
@@ -312,12 +321,12 @@ def test_replace(self):
 
     def test_char_code_at(self):
         jsi = JSInterpreter('function f(i){return "test".charCodeAt(i)}')
-        self.assertEqual(jsi.call_function('f', 0), 116)
-        self.assertEqual(jsi.call_function('f', 1), 101)
-        self.assertEqual(jsi.call_function('f', 2), 115)
-        self.assertEqual(jsi.call_function('f', 3), 116)
-        self.assertEqual(jsi.call_function('f', 4), None)
-        self.assertEqual(jsi.call_function('f', 'not_a_number'), 116)
+        self._test(jsi, 116, args=[0])
+        self._test(jsi, 101, args=[1])
+        self._test(jsi, 115, args=[2])
+        self._test(jsi, 116, args=[3])
+        self._test(jsi, None, args=[4])
+        self._test(jsi, 116, args=['not_a_number'])
 
     def test_bitwise_operators_overflow(self):
         self._test('function f(){return -524999584 << 5}', 379882496)
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 6759d2c46..811f70e68 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -67,7 +67,7 @@
         'https://www.youtube.com/s/player/6ed0d907/player_ias.vflset/en_US/base.js',
         '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
         'AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
-    )
+    ),
 ]
 
 _NSIG_TESTS = [
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index e51bceef3..7a5e59323 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -259,7 +259,7 @@ class YoutubeDL:
     consoletitle:      Display progress in console window's titlebar.
     writedescription:  Write the video description to a .description file
     writeinfojson:     Write the video description to a .info.json file
-    clean_infojson:    Remove private fields from the infojson
+    clean_infojson:    Remove internal metadata from the infojson
     getcomments:       Extract video comments. This will not be written to disk
                        unless writeinfojson is also given
     writeannotations:  Write the video annotations to a .annotations.xml file
@@ -1902,7 +1902,7 @@ def __process_playlist(self, ie_result, download):
                 continue
 
             entry['__x_forwarded_for_ip'] = ie_result.get('__x_forwarded_for_ip')
-            if not lazy and 'playlist-index' in self.params.get('compat_opts', []):
+            if not lazy and 'playlist-index' in self.params['compat_opts']:
                 playlist_index = ie_result['requested_entries'][i]
 
             entry_copy = collections.ChainMap(entry, {
@@ -2959,8 +2959,7 @@ def print_field(field, actual_field=None, optional=False):
         print_field('url', 'urls')
         print_field('thumbnail', optional=True)
         print_field('description', optional=True)
-        if filename:
-            print_field('filename')
+        print_field('filename')
         if self.params.get('forceduration') and info_copy.get('duration') is not None:
             self.to_stdout(formatSeconds(info_copy['duration']))
         print_field('format')
@@ -3185,7 +3184,6 @@ def existing_video_file(*filepaths):
                         return
 
                 if info_dict.get('requested_formats') is not None:
-                    requested_formats = info_dict['requested_formats']
                     old_ext = info_dict['ext']
                     if self.params.get('merge_output_format') is None:
                         if (info_dict['ext'] == 'webm'
@@ -3212,6 +3210,7 @@ def correct_ext(filename, ext=new_ext):
                     full_filename = correct_ext(full_filename)
                     temp_filename = correct_ext(temp_filename)
                     dl_filename = existing_video_file(full_filename, temp_filename)
+
                     info_dict['__real_download'] = False
 
                     merger = FFmpegMergerPP(self)
@@ -3219,12 +3218,12 @@ def correct_ext(filename, ext=new_ext):
                     if dl_filename is not None:
                         self.report_file_already_downloaded(dl_filename)
                     elif fd:
-                        for f in requested_formats if fd != FFmpegFD else []:
+                        for f in info_dict['requested_formats'] if fd != FFmpegFD else []:
                             f['filepath'] = fname = prepend_extension(
                                 correct_ext(temp_filename, info_dict['ext']),
                                 'f%s' % f['format_id'], info_dict['ext'])
                             downloaded.append(fname)
-                        info_dict['url'] = '\n'.join(f['url'] for f in requested_formats)
+                        info_dict['url'] = '\n'.join(f['url'] for f in info_dict['requested_formats'])
                         success, real_download = self.dl(temp_filename, info_dict)
                         info_dict['__real_download'] = real_download
                     else:
@@ -3248,7 +3247,7 @@ def correct_ext(filename, ext=new_ext):
                                 f'You have requested downloading multiple formats to stdout {reason}. '
                                 'The formats will be streamed one after the other')
                             fname = temp_filename
-                        for f in requested_formats:
+                        for f in info_dict['requested_formats']:
                             new_info = dict(info_dict)
                             del new_info['requested_formats']
                             new_info.update(f)
@@ -4109,8 +4108,11 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
                     ret.append((thumb_filename, thumb_filename_final))
                     t['filepath'] = thumb_filename
                 except network_exceptions as err:
+                    if isinstance(err, urllib.error.HTTPError) and err.code == 404:
+                        self.to_screen(f'[info] {thumb_display_id.title()} does not exist')
+                    else:
+                        self.report_warning(f'Unable to download {thumb_display_id}: {err}')
                     thumbnails.pop(idx)
-                    self.report_warning(f'Unable to download {thumb_display_id}: {err}')
             if ret and not write_all:
                 break
         return ret
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index 8693e0b4a..f21e4f7e7 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -1326,3 +1326,7 @@ def get_cookie_header(self, url):
         cookie_req = urllib.request.Request(escape_url(sanitize_url(url)))
         self.add_cookie_header(cookie_req)
         return cookie_req.get_header('Cookie')
+
+    def clear(self, *args, **kwargs):
+        with contextlib.suppress(KeyError):
+            return super().clear(*args, **kwargs)
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index 477ec3c8a..a0219a350 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -49,7 +49,6 @@ class FileDownloader:
     verbose:            Print additional info to stdout.
     quiet:              Do not print messages to stdout.
     ratelimit:          Download speed limit, in bytes/sec.
-    continuedl:         Attempt to continue downloads if possible
     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
     retries:            Number of times to retry for expected network errors.
                         Default is 0 for API, but 10 for CLI
diff --git a/yt_dlp/downloader/niconico.py b/yt_dlp/downloader/niconico.py
index cfe739784..7d8575c2a 100644
--- a/yt_dlp/downloader/niconico.py
+++ b/yt_dlp/downloader/niconico.py
@@ -7,9 +7,9 @@
 from .external import FFmpegFD
 from ..utils import (
     DownloadError,
-    str_or_none,
-    sanitized_Request,
     WebSocketsWrapper,
+    sanitized_Request,
+    str_or_none,
     try_get,
 )
 
diff --git a/yt_dlp/extractor/ciscowebex.py b/yt_dlp/extractor/ciscowebex.py
index 0fcf02282..40430505d 100644
--- a/yt_dlp/extractor/ciscowebex.py
+++ b/yt_dlp/extractor/ciscowebex.py
@@ -49,7 +49,7 @@ def _real_extract(self, url):
             'https://%s.webex.com/webappng/api/v1/recordings/%s/stream' % (subdomain, video_id),
             video_id, headers=headers, query={'siteurl': siteurl}, expected_status=(403, 429))
 
-        if urlh.status == 403:
+        if urlh.getcode() == 403:
             if stream['code'] == 53004:
                 self.raise_login_required()
             if stream['code'] == 53005:
@@ -59,7 +59,7 @@ def _real_extract(self, url):
                     'This video is protected by a password, use the --video-password option', expected=True)
             raise ExtractorError(f'{self.IE_NAME} said: {stream["code"]} - {stream["message"]}', expected=True)
 
-        if urlh.status == 429:
+        if urlh.getcode() == 429:
             self.raise_login_required(
                 f'{self.IE_NAME} asks you to solve a CAPTCHA. Solve CAPTCHA in browser and',
                 method='cookies')
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index f11a67358..9662a7ee1 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -17,6 +17,7 @@
 import sys
 import time
 import types
+import urllib.error
 import urllib.parse
 import urllib.request
 import xml.etree.ElementTree
@@ -58,6 +59,7 @@
     join_nonempty,
     js_to_json,
     mimetype2ext,
+    netrc_from_content,
     network_exceptions,
     orderedSet,
     parse_bitrate,
@@ -72,7 +74,6 @@
     smuggle_url,
     str_or_none,
     str_to_int,
-    netrc_from_content,
     strip_or_none,
     traverse_obj,
     truncate_string,
diff --git a/yt_dlp/extractor/dumpert.py b/yt_dlp/extractor/dumpert.py
old mode 100755
new mode 100644
diff --git a/yt_dlp/extractor/globalplayer.py b/yt_dlp/extractor/globalplayer.py
old mode 100755
new mode 100644
diff --git a/yt_dlp/extractor/odnoklassniki.py b/yt_dlp/extractor/odnoklassniki.py
index 0d0ad0bb8..e63714e84 100644
--- a/yt_dlp/extractor/odnoklassniki.py
+++ b/yt_dlp/extractor/odnoklassniki.py
@@ -238,10 +238,8 @@ class OdnoklassnikiIE(InfoExtractor):
     def _clear_cookies(self, cdn_url):
         # Direct http downloads will fail if CDN cookies are set
         # so we need to reset them after each format extraction
-        if self._get_cookies('https://notarealsubdomain.mycdn.me/'):
-            self.cookiejar.clear(domain='.mycdn.me')
-        if self._get_cookies(cdn_url):
-            self.cookiejar.clear(domain=urllib.parse.urlparse(cdn_url).hostname)
+        self.cookiejar.clear(domain='.mycdn.me')
+        self.cookiejar.clear(domain=urllib.parse.urlparse(cdn_url).hostname)
 
     @classmethod
     def _extract_embed_urls(cls, url, webpage):
diff --git a/yt_dlp/extractor/tvp.py b/yt_dlp/extractor/tvp.py
index 2aa0dd870..c686044fa 100644
--- a/yt_dlp/extractor/tvp.py
+++ b/yt_dlp/extractor/tvp.py
@@ -488,9 +488,9 @@ def _call_api(self, resource, video_id, query={}, **kwargs):
             f'{self._API_BASE_URL}/{resource}', video_id,
             query={'lang': 'pl', 'platform': 'BROWSER', **query},
             expected_status=lambda x: is_valid(x) or 400 <= x < 500, **kwargs)
-        if is_valid(urlh.status):
+        if is_valid(urlh.getcode()):
             return document
-        raise ExtractorError(f'Woronicza said: {document.get("code")} (HTTP {urlh.status})')
+        raise ExtractorError(f'Woronicza said: {document.get("code")} (HTTP {urlh.getcode()})')
 
     def _parse_video(self, video, with_url=True):
         info_dict = traverse_obj(video, {
diff --git a/yt_dlp/extractor/vidio.py b/yt_dlp/extractor/vidio.py
index 770aa284d..23e1aaf20 100644
--- a/yt_dlp/extractor/vidio.py
+++ b/yt_dlp/extractor/vidio.py
@@ -39,7 +39,7 @@ def is_logged_in():
         login_post, login_post_urlh = self._download_webpage_handle(
             self._LOGIN_URL, None, 'Logging in', data=urlencode_postdata(login_form), expected_status=[302, 401])
 
-        if login_post_urlh.status == 401:
+        if login_post_urlh.getcode() == 401:
             if get_element_by_class('onboarding-content-register-popup__title', login_post):
                 raise ExtractorError(
                     'Unable to log in: The provided email has not registered yet.', expected=True)
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 4daa4f50e..11e47904a 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -811,7 +811,7 @@ def _extract_badges(self, badge_list: list):
             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW,
             'BADGE_STYLE_TYPE_VERIFIED': BadgeType.VERIFIED,
-            'BADGE_STYLE_TYPE_VERIFIED_ARTIST': BadgeType.VERIFIED
+            'BADGE_STYLE_TYPE_VERIFIED_ARTIST': BadgeType.VERIFIED,
         }
 
         label_map = {
@@ -821,7 +821,7 @@ def _extract_badges(self, badge_list: list):
             'live': BadgeType.LIVE_NOW,
             'premium': BadgeType.AVAILABILITY_PREMIUM,
             'verified': BadgeType.VERIFIED,
-            'official artist channel': BadgeType.VERIFIED
+            'official artist channel': BadgeType.VERIFIED,
         }
 
         badges = []
@@ -3935,7 +3935,7 @@ def process_manifest_format(f, proto, client_name, itag):
             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
             if f['quality'] == -1 and f.get('height'):
                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
-            if self.get_param('verbose'):
+            if self.get_param('verbose') or all_formats:
                 f['format_note'] = join_nonempty(f.get('format_note'), client_name, delim=', ')
             if f.get('fps') and f['fps'] <= 1:
                 del f['fps']
@@ -4531,7 +4531,7 @@ def process_language(container, base_url, lang_code, sub_name, query):
             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
         ):
             upload_date = strftime_or_none(
-                self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
+                self._parse_time_text(self._get_text(vpir, 'dateText'))) or upload_date
         info['upload_date'] = upload_date
 
         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
@@ -5071,7 +5071,7 @@ def _get_uncropped(url):
         last_updated_unix = self._parse_time_text(
             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
-        info['modified_date'] = strftime_or_none(last_updated_unix, '%Y%m%d')
+        info['modified_date'] = strftime_or_none(last_updated_unix)
 
         info['view_count'] = self._get_count(playlist_stats, 1)
         if info['view_count'] is None:  # 0 is allowed
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index b174a24af..9d6dbec9f 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -1414,8 +1414,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         '--clean-info-json', '--clean-infojson',
         action='store_true', dest='clean_infojson', default=None,
         help=(
-            'Remove some private fields such as filenames from the infojson. '
-            'Note that it could still contain some personal information (default)'))
+            'Remove some internal metadata such as filenames from the infojson (default)'))
     filesystem.add_option(
         '--no-clean-info-json', '--no-clean-infojson',
         action='store_false', dest='clean_infojson',
@@ -1678,8 +1677,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
             'Execute a command, optionally prefixed with when to execute it, separated by a ":". '
             'Supported values of "WHEN" are the same as that of --use-postprocessor (default: after_move). '
             'Same syntax as the output template can be used to pass any field as arguments to the command. '
-            'After download, an additional field "filepath" that contains the final path of the downloaded file '
-            'is also available, and if no fields are passed, %(filepath,_filename|)q is appended to the end of the command. '
+            'If no fields are passed, %(filepath,_filename|)q is appended to the end of the command. '
             'This option can be used multiple times'))
     postproc.add_option(
         '--no-exec',
diff --git a/yt_dlp/utils/_legacy.py b/yt_dlp/utils/_legacy.py
index 1097778f0..96ac468b1 100644
--- a/yt_dlp/utils/_legacy.py
+++ b/yt_dlp/utils/_legacy.py
@@ -6,7 +6,7 @@
 import urllib.parse
 import zlib
 
-from ._utils import decode_base_n, preferredencoding
+from ._utils import Popen, decode_base_n, preferredencoding
 from .traversal import traverse_obj
 from ..dependencies import certifi, websockets
 
@@ -174,3 +174,7 @@ def handle_youtubedl_headers(headers):
         del filtered_headers['Youtubedl-no-compression']
 
     return filtered_headers
+
+
+def process_communicate_or_kill(p, *args, **kwargs):
+    return Popen.communicate_or_kill(p, *args, **kwargs)
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 28c2785cb..bc1bc9116 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -872,12 +872,6 @@ def __init__(self, content):
             self._parse('-', stream, False)
 
 
-def process_communicate_or_kill(p, *args, **kwargs):
-    deprecation_warning(f'"{__name__}.process_communicate_or_kill" is deprecated and may be removed '
-                        f'in a future version. Use "{__name__}.Popen.communicate_or_kill" instead')
-    return Popen.communicate_or_kill(p, *args, **kwargs)
-
-
 class Popen(subprocess.Popen):
     if sys.platform == 'win32':
         _startupinfo = subprocess.STARTUPINFO()
@@ -1662,7 +1656,7 @@ def unified_strdate(date_str, day_first=True):
 
 
 def unified_timestamp(date_str, day_first=True):
-    if date_str is None:
+    if not isinstance(date_str, str):
         return None
 
     date_str = re.sub(r'\s+', ' ', re.sub(
@@ -2454,7 +2448,7 @@ def request_to_url(req):
         return req
 
 
-def strftime_or_none(timestamp, date_format, default=None):
+def strftime_or_none(timestamp, date_format='%Y%m%d', default=None):
     datetime_object = None
     try:
         if isinstance(timestamp, (int, float)):  # unix timestamp

From 84078a8b38f403495d00b46654c8750774d821de Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 05:45:09 +0530
Subject: [PATCH 380/871] [core] Fix `filepath` being copied to underlying
 format dict

Closes #6536
---
 yt_dlp/YoutubeDL.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 7a5e59323..503aafbc7 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3212,6 +3212,8 @@ def correct_ext(filename, ext=new_ext):
                     dl_filename = existing_video_file(full_filename, temp_filename)
 
                     info_dict['__real_download'] = False
+                    # NOTE: Copy so that original format dicts are not modified
+                    info_dict['requested_formats'] = list(map(dict, info_dict['requested_formats']))
 
                     merger = FFmpegMergerPP(self)
                     downloaded = []

From 1619ab3e67d8dc4f86fc7ed292c79345bc0d91a0 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 06:49:56 +0530
Subject: [PATCH 381/871] Bugfix for ebe1b4e34f43c3acad30e4bcb8484681a030c114

---
 test/test_YoutubeDL.py |  2 ++
 yt_dlp/YoutubeDL.py    | 10 ++++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 05dd3ed41..f495fa6d9 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -630,6 +630,7 @@ def test_add_extra_info(self):
         self.assertEqual(test_dict['playlist'], 'funny videos')
 
     outtmpl_info = {
+        'id': '1234',
         'id': '1234',
         'ext': 'mp4',
         'width': None,
@@ -754,6 +755,7 @@ def expect_same_infodict(out):
         test('%(ext)c', 'm')
         test('%(id)d %(id)r', "1234 '1234'")
         test('%(id)r %(height)r', "'1234' 1080")
+        test('%(title5)a %(height)a', (R"'\xe1\xe9\xed \U0001d400' 1080", None))
         test('%(ext)s-%(ext|def)d', 'mp4-def')
         test('%(width|0)04d', '0')
         test('a%(width|b)d', 'ab', outtmpl_na_placeholder='none')
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 503aafbc7..bc5c1b95e 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1328,17 +1328,19 @@ def create_key(outer_mobj):
                     value = str(value)[0]
                 else:
                     fmt = str_fmt
-            elif fmt[-1] not in 'rs':  # numeric
+            elif fmt[-1] not in 'rsa':  # numeric
                 value = float_or_none(value)
                 if value is None:
                     value, fmt = default, 's'
 
             if sanitize:
+                # If value is an object, sanitize might convert it to a string
+                # So we convert it to repr first
                 if fmt[-1] == 'r':
-                    # If value is an object, sanitize might convert it to a string
-                    # So we convert it to repr first
                     value, fmt = repr(value), str_fmt
-                if fmt[-1] in 'csr':
+                elif fmt[-1] == 'a':
+                    value, fmt = ascii(value), str_fmt
+                if fmt[-1] in 'csra':
                     value = sanitizer(initial_field, value)
 
             key = '%s\0%s' % (key.replace('%', '%\0'), outer_mobj.group('format'))

From 42f2d40b475db66486a4b4fe5b56751a640db5db Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 08:51:14 +0530
Subject: [PATCH 382/871] Update to ytdl-commit-07af47

[YouTube] Improve fix for ae8ba2c
https://github.com/ytdl-org/youtube-dl/commit/07af47960f3bb262ead02490ce65c8c45c01741e
---
 test/test_jsinterp.py | 26 ++++++++++++++++++++++++++
 yt_dlp/casefold.py    |  5 +++++
 yt_dlp/jsinterp.py    |  2 +-
 3 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 yt_dlp/casefold.py

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index e9682ddab..86928a6a0 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -35,6 +35,21 @@ def test_basic(self):
         self._test('function f(){42}', None)
         self._test('var f = function(){return 42;}', 42)
 
+    def test_add(self):
+        self._test('function f(){return 42 + 7;}', 49)
+        self._test('function f(){return 42 + undefined;}', NaN)
+        self._test('function f(){return 42 + null;}', 42)
+
+    def test_sub(self):
+        self._test('function f(){return 42 - 7;}', 35)
+        self._test('function f(){return 42 - undefined;}', NaN)
+        self._test('function f(){return 42 - null;}', 42)
+
+    def test_mul(self):
+        self._test('function f(){return 42 * 7;}', 294)
+        self._test('function f(){return 42 * undefined;}', NaN)
+        self._test('function f(){return 42 * null;}', 0)
+
     def test_div(self):
         jsi = JSInterpreter('function f(a, b){return a / b;}')
         self._test(jsi, NaN, args=(0, 0))
@@ -42,6 +57,17 @@ def test_div(self):
         self._test(jsi, float('inf'), args=(2, 0))
         self._test(jsi, 0, args=(0, 3))
 
+    def test_mod(self):
+        self._test('function f(){return 42 % 7;}', 0)
+        self._test('function f(){return 42 % 0;}', NaN)
+        self._test('function f(){return 42 % undefined;}', NaN)
+
+    def test_exp(self):
+        self._test('function f(){return 42 ** 2;}', 1764)
+        self._test('function f(){return 42 ** undefined;}', NaN)
+        self._test('function f(){return 42 ** null;}', 1)
+        self._test('function f(){return undefined ** 42;}', NaN)
+
     def test_calc(self):
         self._test('function f(a){return 2*a+1;}', 7, args=[3])
 
diff --git a/yt_dlp/casefold.py b/yt_dlp/casefold.py
new file mode 100644
index 000000000..41a53e5b6
--- /dev/null
+++ b/yt_dlp/casefold.py
@@ -0,0 +1,5 @@
+import warnings
+
+warnings.warn(DeprecationWarning(f'{__name__} is deprecated'))
+
+casefold = str.casefold
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index 9c280fb86..bda3fb459 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -812,9 +812,9 @@ def extract_function_code(self, funcname):
                 \((?P<args>[^)]*)\)\s*
                 (?P<code>{.+})''' % {'name': re.escape(funcname)},
             self.code)
-        code, _ = self._separate_at_paren(func_m.group('code'))
         if func_m is None:
             raise self.Exception(f'Could not find JS function "{funcname}"')
+        code, _ = self._separate_at_paren(func_m.group('code'))
         return [x.strip() for x in func_m.group('args').split(',')], code
 
     def extract_function(self, funcname):

From d1b21561497b6bbb8ff1202e63f48eb41bd315af Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@example.com>
Date: Wed, 21 Jun 2023 04:02:40 +0000
Subject: [PATCH 383/871] Release 2023.06.21

Created by: pukkandan

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |   8 +-
 .../ISSUE_TEMPLATE/2_site_support_request.yml |   8 +-
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |   8 +-
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |   8 +-
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |   8 +-
 .github/ISSUE_TEMPLATE/6_question.yml         |   8 +-
 CONTRIBUTORS                                  |  46 +++
 Changelog.md                                  | 280 ++++++++++++++++++
 README.md                                     |   2 +-
 supportedsites.md                             | 111 +++++--
 yt_dlp/version.py                             |   4 +-
 11 files changed, 439 insertions(+), 52 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index 77b777d5a..351454b12 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -64,7 +64,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -72,8 +72,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.04, Current version: 2023.03.04
-        yt-dlp is up to date (2023.03.04)
+        Latest version: 2023.06.21, Current version: 2023.06.21
+        yt-dlp is up to date (2023.06.21)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index 890df48fa..b2a613e2f 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -76,7 +76,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -84,8 +84,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.04, Current version: 2023.03.04
-        yt-dlp is up to date (2023.03.04)
+        Latest version: 2023.06.21, Current version: 2023.06.21
+        yt-dlp is up to date (2023.06.21)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index ef9bda36a..c100561eb 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -72,7 +72,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -80,8 +80,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.04, Current version: 2023.03.04
-        yt-dlp is up to date (2023.03.04)
+        Latest version: 2023.06.21, Current version: 2023.06.21
+        yt-dlp is up to date (2023.06.21)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index 122dda4f2..e97d7b507 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -57,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -65,8 +65,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.04, Current version: 2023.03.04
-        yt-dlp is up to date (2023.03.04)
+        Latest version: 2023.06.21, Current version: 2023.06.21
+        yt-dlp is up to date (2023.06.21)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index b17c65658..a44612d79 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -53,7 +53,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -61,7 +61,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.04, Current version: 2023.03.04
-        yt-dlp is up to date (2023.03.04)
+        Latest version: 2023.06.21, Current version: 2023.06.21
+        yt-dlp is up to date (2023.06.21)
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index c694e5a5a..a15a46968 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.03.04** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -59,7 +59,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.03.04 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -67,7 +67,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.03.04, Current version: 2023.03.04
-        yt-dlp is up to date (2023.03.04)
+        Latest version: 2023.06.21, Current version: 2023.06.21
+        yt-dlp is up to date (2023.06.21)
         <more lines>
       render: shell
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index d6ba617b7..3b35895d9 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -409,3 +409,49 @@ Hill-98
 LXYan2333
 mushbite
 venkata-krishnas
+7vlad7
+alexklapheke
+arobase-che
+bepvte
+bergoid
+blmarket
+brandon-dacrib
+c-basalt
+CoryTibbettsDev
+Cyberes
+D0LLYNH0
+danog
+DataGhost
+falbrechtskirchinger
+foreignBlade
+garret1317
+hasezoey
+hoaluvn
+ItzMaxTV
+ivanskodje
+jo-nike
+kangalio
+linsui
+makew0rld
+menschel
+mikf
+mrscrapy
+NDagestad
+Neurognostic
+NextFire
+nick-cd
+permunkle
+pzhlkj6612
+ringus1
+rjy
+Schmoaaaaah
+sjthespian
+theperfectpunk
+toomyzoom
+truedread
+TxI5
+unbeatable-101
+vampirefrog
+vidiot720
+viktor-enzell
+zhgwn
diff --git a/Changelog.md b/Changelog.md
index 186998ede..d7a1cb495 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,286 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.06.21
+
+#### Important changes
+- YouTube: Improved throttling and signature fixes
+
+#### Core changes
+- [Add `--compat-option playlist-match-filter`](https://github.com/yt-dlp/yt-dlp/commit/93b39cdbd9dcf351bfa0c4ee252805b4617fdca9) by [pukkandan](https://github.com/pukkandan)
+- [Add `--no-quiet`](https://github.com/yt-dlp/yt-dlp/commit/d669772c65e8630162fd6555d0a578b246591921) by [pukkandan](https://github.com/pukkandan)
+- [Add option `--color`](https://github.com/yt-dlp/yt-dlp/commit/8417f26b8a819cd7ffcd4e000ca3e45033e670fb) ([#6904](https://github.com/yt-dlp/yt-dlp/issues/6904)) by [Grub4K](https://github.com/Grub4K)
+- [Add option `--netrc-cmd`](https://github.com/yt-dlp/yt-dlp/commit/db3ad8a67661d7b234a6954d9c6a4a9b1749f5eb) ([#6682](https://github.com/yt-dlp/yt-dlp/issues/6682)) by [NDagestad](https://github.com/NDagestad), [pukkandan](https://github.com/pukkandan)
+- [Add option `--xff`](https://github.com/yt-dlp/yt-dlp/commit/c16644642b08e2bf4130a6c5fa01395d8718c990) by [pukkandan](https://github.com/pukkandan)
+- [Auto-select default format in `-f-`](https://github.com/yt-dlp/yt-dlp/commit/372a0f3b9dadd1e52234b498aa4c7040ef868c7d) ([#7101](https://github.com/yt-dlp/yt-dlp/issues/7101)) by [ivanskodje](https://github.com/ivanskodje), [pukkandan](https://github.com/pukkandan)
+- [Deprecate internal `Youtubedl-no-compression` header](https://github.com/yt-dlp/yt-dlp/commit/955c89584b66fcd0fcfab3e611f1edeb1ca63886) ([#6876](https://github.com/yt-dlp/yt-dlp/issues/6876)) by [coletdjnz](https://github.com/coletdjnz)
+- [Do not translate newlines in `--print-to-file`](https://github.com/yt-dlp/yt-dlp/commit/9874e82b5a61582169300bea561b3e8899ad1ef7) by [pukkandan](https://github.com/pukkandan)
+- [Ensure pre-processor errors do not block `--print`](https://github.com/yt-dlp/yt-dlp/commit/f005a35aa7e4f67a0c603a946c0dd714c151b2d6) by [pukkandan](https://github.com/pukkandan) (With fixes in [17ba434](https://github.com/yt-dlp/yt-dlp/commit/17ba4343cf99701692a7f4798fd42b50f644faba))
+- [Fix `filepath` being copied to underlying format dict](https://github.com/yt-dlp/yt-dlp/commit/84078a8b38f403495d00b46654c8750774d821de) by [pukkandan](https://github.com/pukkandan)
+- [Improve HTTP redirect handling](https://github.com/yt-dlp/yt-dlp/commit/08916a49c777cb6e000eec092881eb93ec22076c) ([#7094](https://github.com/yt-dlp/yt-dlp/issues/7094)) by [coletdjnz](https://github.com/coletdjnz)
+- [Populate `filename` and `urls` fields at all stages of `--print`](https://github.com/yt-dlp/yt-dlp/commit/170605840ea9d5ad75da6576485ea7d125b428ee) by [pukkandan](https://github.com/pukkandan) (With fixes in [b5f61b6](https://github.com/yt-dlp/yt-dlp/commit/b5f61b69d4561b81fc98c226b176f0c15493e688))
+- [Relaxed validation for numeric format filters](https://github.com/yt-dlp/yt-dlp/commit/c3f624ef0a5d7a6ae1c5ffeb243087e9fc7d79dc) by [pukkandan](https://github.com/pukkandan)
+- [Support decoding multiple content encodings](https://github.com/yt-dlp/yt-dlp/commit/daafbf49b3482edae4d70dd37070be99742a926e) ([#7142](https://github.com/yt-dlp/yt-dlp/issues/7142)) by [coletdjnz](https://github.com/coletdjnz)
+- [Support loading info.json with a list at it's root](https://github.com/yt-dlp/yt-dlp/commit/ab1de9cb1e39cf421c2b7dc6756c6ff1955bb313) by [pukkandan](https://github.com/pukkandan)
+- [Workaround erroneous urllib Windows proxy parsing](https://github.com/yt-dlp/yt-dlp/commit/3f66b6fe50f8d5b545712f8b19d5ae62f5373980) ([#7092](https://github.com/yt-dlp/yt-dlp/issues/7092)) by [coletdjnz](https://github.com/coletdjnz)
+- **cookies**
+    - [Defer extraction of v11 key from keyring](https://github.com/yt-dlp/yt-dlp/commit/9b7a48abd1b187eae1e3f6c9839c47d43ccec00b) by [Grub4K](https://github.com/Grub4K)
+    - [Move `YoutubeDLCookieJar` to cookies module](https://github.com/yt-dlp/yt-dlp/commit/b87e01c123fd560b6a674ce00f45a9459d82d98a) ([#7091](https://github.com/yt-dlp/yt-dlp/issues/7091)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Support custom Safari cookies path](https://github.com/yt-dlp/yt-dlp/commit/a58182b75a05fe0a10c5e94a536711d3ade19c20) ([#6783](https://github.com/yt-dlp/yt-dlp/issues/6783)) by [NextFire](https://github.com/NextFire)
+    - [Update for chromium changes](https://github.com/yt-dlp/yt-dlp/commit/b38d4c941d1993ab27e4c0f8e024e23c2ec0f8f8) ([#6897](https://github.com/yt-dlp/yt-dlp/issues/6897)) by [mbway](https://github.com/mbway)
+- **Cryptodome**: [Fix `__bool__`](https://github.com/yt-dlp/yt-dlp/commit/98ac902c4979e4529b166e873473bef42baa2e3e) by [pukkandan](https://github.com/pukkandan)
+- **jsinterp**
+    - [Do not compile regex](https://github.com/yt-dlp/yt-dlp/commit/7aeda6cc9e73ada0b0a0b6a6748c66bef63a20a8) by [pukkandan](https://github.com/pukkandan)
+    - [Fix division](https://github.com/yt-dlp/yt-dlp/commit/b4a252fba81f53631c07ca40ce7583f5d19a8a36) ([#7279](https://github.com/yt-dlp/yt-dlp/issues/7279)) by [bashonly](https://github.com/bashonly)
+    - [Fix global object extraction](https://github.com/yt-dlp/yt-dlp/commit/01aba2519a0884ef17d5f85608dbd2a455577147) by [pukkandan](https://github.com/pukkandan)
+    - [Handle `NaN` in bitwise operators](https://github.com/yt-dlp/yt-dlp/commit/1d7656184c6b8aa46b29149893894b3c24f1df00) by [pukkandan](https://github.com/pukkandan)
+    - [Handle negative numbers better](https://github.com/yt-dlp/yt-dlp/commit/7cf51f21916292cd80bdeceb37489f5322f166dd) by [pukkandan](https://github.com/pukkandan)
+- **outtmpl**
+    - [Allow `\n` in replacements and default.](https://github.com/yt-dlp/yt-dlp/commit/78fde6e3398ff11e5d383a66b28664badeab5180) by [pukkandan](https://github.com/pukkandan)
+    - [Fix some minor bugs](https://github.com/yt-dlp/yt-dlp/commit/ebe1b4e34f43c3acad30e4bcb8484681a030c114) by [pukkandan](https://github.com/pukkandan) (With fixes in [1619ab3](https://github.com/yt-dlp/yt-dlp/commit/1619ab3e67d8dc4f86fc7ed292c79345bc0d91a0))
+    - [Support `str.format` syntax inside replacements](https://github.com/yt-dlp/yt-dlp/commit/ec9311c41b111110bc52cfbd6ea682c6fb23f77a) by [pukkandan](https://github.com/pukkandan)
+- **update**
+    - [Better error handling](https://github.com/yt-dlp/yt-dlp/commit/d2e84d5eb01c66fc5304e8566348d65a7be24ed7) by [pukkandan](https://github.com/pukkandan)
+    - [Do not restart into versions without `--update-to`](https://github.com/yt-dlp/yt-dlp/commit/02948a17d903f544363bb20b51a6d8baed7bba08) by [pukkandan](https://github.com/pukkandan)
+    - [Implement `--update-to` repo](https://github.com/yt-dlp/yt-dlp/commit/665472a7de3880578c0b7b3f95c71570c056368e) by [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan)
+- **upstream**
+    - [Merged with youtube-dl 07af47](https://github.com/yt-dlp/yt-dlp/commit/42f2d40b475db66486a4b4fe5b56751a640db5db) by [pukkandan](https://github.com/pukkandan)
+    - [Merged with youtube-dl d1c6c5](https://github.com/yt-dlp/yt-dlp/commit/4823ec9f461512daa1b8ab362893bb86a6320b26) by [pukkandan](https://github.com/pukkandan) (With fixes in [edbe5b5](https://github.com/yt-dlp/yt-dlp/commit/edbe5b589dd0860a67b4e03f58db3cd2539d91c2) by [bashonly](https://github.com/bashonly))
+- **utils**
+    - `FormatSorter`: [Improve `size` and `br`](https://github.com/yt-dlp/yt-dlp/commit/eedda5252c05327748dede204a8fccafa0288118) by [pukkandan](https://github.com/pukkandan), [u-spec-png](https://github.com/u-spec-png)
+    - `js_to_json`: [Implement template strings](https://github.com/yt-dlp/yt-dlp/commit/0898c5c8ccadfc404472456a7a7751b72afebadd) ([#6623](https://github.com/yt-dlp/yt-dlp/issues/6623)) by [Grub4K](https://github.com/Grub4K)
+    - `locked_file`: [Fix for virtiofs](https://github.com/yt-dlp/yt-dlp/commit/45998b3e371b819ce0dbe50da703809a048cc2fe) ([#6840](https://github.com/yt-dlp/yt-dlp/issues/6840)) by [brandon-dacrib](https://github.com/brandon-dacrib)
+    - `strftime_or_none`: [Handle negative timestamps](https://github.com/yt-dlp/yt-dlp/commit/a35af4306d24c56c6358f89cdf204860d1cd62b4) by [dirkf](https://github.com/dirkf), [pukkandan](https://github.com/pukkandan)
+    - `traverse_obj`
+        - [Allow iterables in traversal](https://github.com/yt-dlp/yt-dlp/commit/21b5ec86c2c37d10c5bb97edd7051d3aac16bb3e) ([#6902](https://github.com/yt-dlp/yt-dlp/issues/6902)) by [Grub4K](https://github.com/Grub4K)
+        - [More fixes](https://github.com/yt-dlp/yt-dlp/commit/b079c26f0af8085bccdadc72c61c8164ca5ab0f8) ([#6959](https://github.com/yt-dlp/yt-dlp/issues/6959)) by [Grub4K](https://github.com/Grub4K)
+    - `write_string`: [Fix noconsole behavior](https://github.com/yt-dlp/yt-dlp/commit/3b479100df02e20dd949e046003ae96ddbfced57) by [Grub4K](https://github.com/Grub4K)
+
+#### Extractor changes
+- [Do not exit early for unsuitable `url_result`](https://github.com/yt-dlp/yt-dlp/commit/baa922b5c74b10e3b86ff5e6cf6529b3aae8efab) by [pukkandan](https://github.com/pukkandan)
+- [Do not warn for invalid chapter data in description](https://github.com/yt-dlp/yt-dlp/commit/84ffeb7d5e72e3829319ba7720a8480fc4c7503b) by [pukkandan](https://github.com/pukkandan)
+- [Extract more metadata from ISM](https://github.com/yt-dlp/yt-dlp/commit/f68434cc74cfd3db01b266476a2eac8329fbb267) by [pukkandan](https://github.com/pukkandan)
+- **abematv**: [Add fallback for title and description extraction and extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/c449c0655d7c8549e6e1389c26b628053b253d39) ([#6994](https://github.com/yt-dlp/yt-dlp/issues/6994)) by [Lesmiscore](https://github.com/Lesmiscore)
+- **acast**: [Support embeds](https://github.com/yt-dlp/yt-dlp/commit/c91ac833ea99b00506e470a44cf930e4e23378c9) ([#7212](https://github.com/yt-dlp/yt-dlp/issues/7212)) by [pabs3](https://github.com/pabs3)
+- **adobepass**: [Handle `Charter_Direct` MSO as `Spectrum`](https://github.com/yt-dlp/yt-dlp/commit/ea0570820336a0fe9c3b530d1b0d1e59313274f4) ([#6824](https://github.com/yt-dlp/yt-dlp/issues/6824)) by [bashonly](https://github.com/bashonly)
+- **aeonco**: [Support Youtube embeds](https://github.com/yt-dlp/yt-dlp/commit/ed81b74802b4247ee8d9dc0ef87eb52baefede1c) ([#6591](https://github.com/yt-dlp/yt-dlp/issues/6591)) by [alexklapheke](https://github.com/alexklapheke)
+- **afreecatv**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/fdd69db38924c38194ef236b26325d66ac815c88) ([#6283](https://github.com/yt-dlp/yt-dlp/issues/6283)) by [blmarket](https://github.com/blmarket)
+- **ARDBetaMediathek**: [Add thumbnail](https://github.com/yt-dlp/yt-dlp/commit/f78eb41e1c0f1dcdb10317358a26bf541dc7ee15) ([#6890](https://github.com/yt-dlp/yt-dlp/issues/6890)) by [StefanLobbenmeier](https://github.com/StefanLobbenmeier)
+- **bibeltv**: [Fix extraction, support live streams and series](https://github.com/yt-dlp/yt-dlp/commit/4ad58667c102bd82a7c4cca8aa395ec1682e3b4c) ([#6505](https://github.com/yt-dlp/yt-dlp/issues/6505)) by [flashdagger](https://github.com/flashdagger)
+- **bilibili**
+    - [Support festival videos](https://github.com/yt-dlp/yt-dlp/commit/ab29e47029e2f5b48abbbab78e82faf7cf6e9506) ([#6547](https://github.com/yt-dlp/yt-dlp/issues/6547)) by [qbnu](https://github.com/qbnu)
+    - SpaceVideo: [Extract signature](https://github.com/yt-dlp/yt-dlp/commit/6f10cdcf7eeaeae5b75e0a4428cd649c156a2d83) ([#7149](https://github.com/yt-dlp/yt-dlp/issues/7149)) by [elyse0](https://github.com/elyse0)
+- **biliIntl**: [Add comment extraction](https://github.com/yt-dlp/yt-dlp/commit/b093c38cc9f26b59a8504211d792f053142c847d) ([#6079](https://github.com/yt-dlp/yt-dlp/issues/6079)) by [HobbyistDev](https://github.com/HobbyistDev)
+- **bitchute**: [Add more fallback subdomains](https://github.com/yt-dlp/yt-dlp/commit/0c4e0fbcade0fc92d14c2a6d63e360fe067f6192) ([#6907](https://github.com/yt-dlp/yt-dlp/issues/6907)) by [Neurognostic](https://github.com/Neurognostic)
+- **booyah**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/f7f7a877bf8e87fd4eb0ad2494ad948ca7691114) by [pukkandan](https://github.com/pukkandan)
+- **BrainPOP**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/979568f26ece80bca72b48f0dd57d676e431059a) ([#6106](https://github.com/yt-dlp/yt-dlp/issues/6106)) by [MinePlayersPE](https://github.com/MinePlayersPE)
+- **bravotv**
+    - [Detect DRM](https://github.com/yt-dlp/yt-dlp/commit/1fe5bf240e6ade487d18079a62aa36bcc440a27a) ([#7171](https://github.com/yt-dlp/yt-dlp/issues/7171)) by [bashonly](https://github.com/bashonly)
+    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/06966cb8966b9aa4f60ab9c44c182a057d4ca3a3) ([#6568](https://github.com/yt-dlp/yt-dlp/issues/6568)) by [bashonly](https://github.com/bashonly)
+- **camfm**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/4cbfa570a1b9bd65b0f48770693377e8d842dcb0) ([#7083](https://github.com/yt-dlp/yt-dlp/issues/7083)) by [garret1317](https://github.com/garret1317)
+- **cbc**
+    - [Fix live extractor, playlist `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/7a7b1376fbce0067cf37566bb47131bc0022638d) ([#6625](https://github.com/yt-dlp/yt-dlp/issues/6625)) by [makew0rld](https://github.com/makew0rld)
+    - [Ignore 426 from API](https://github.com/yt-dlp/yt-dlp/commit/4afb208cf07b59291ae3b0c4efc83945ee5b8812) ([#6781](https://github.com/yt-dlp/yt-dlp/issues/6781)) by [jo-nike](https://github.com/jo-nike)
+    - gem: [Update `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/871c907454693940cb56906ed9ea49fcb7154829) ([#6499](https://github.com/yt-dlp/yt-dlp/issues/6499)) by [makeworld-the-better-one](https://github.com/makeworld-the-better-one)
+- **cbs**: [Add `ParamountPressExpress` extractor](https://github.com/yt-dlp/yt-dlp/commit/44369c9afa996e14e9f466754481d878811b5b4a) ([#6604](https://github.com/yt-dlp/yt-dlp/issues/6604)) by [bashonly](https://github.com/bashonly)
+- **cbsnews**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/f6e43d6fa9804c24525e1fed0a87782754dab7ed) ([#6681](https://github.com/yt-dlp/yt-dlp/issues/6681)) by [bashonly](https://github.com/bashonly)
+- **chilloutzone**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/6f4fc5660f40f3458882a8f51601eae4af7be609) ([#6445](https://github.com/yt-dlp/yt-dlp/issues/6445)) by [bashonly](https://github.com/bashonly)
+- **clipchamp**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/2f07c4c1da4361af213e5791279b9d152d2e4ce3) ([#6978](https://github.com/yt-dlp/yt-dlp/issues/6978)) by [bashonly](https://github.com/bashonly)
+- **comedycentral**: [Add support for movies](https://github.com/yt-dlp/yt-dlp/commit/66468bbf49562ff82670cbbd456c5e8448a6df34) ([#7108](https://github.com/yt-dlp/yt-dlp/issues/7108)) by [sqrtNOT](https://github.com/sqrtNOT)
+- **crtvg**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/26c517b29c8727e47948d6fff749d5297f0efb60) ([#7168](https://github.com/yt-dlp/yt-dlp/issues/7168)) by [ItzMaxTV](https://github.com/ItzMaxTV)
+- **crunchyroll**: [Rework with support for movies, music and artists](https://github.com/yt-dlp/yt-dlp/commit/032de83ea9ff2f4977d9c71a93bbc1775597b762) ([#6237](https://github.com/yt-dlp/yt-dlp/issues/6237)) by [Grub4K](https://github.com/Grub4K)
+- **dacast**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/c25cac2f8e5fbac2737a426d7778fd2f0efc5381) ([#6896](https://github.com/yt-dlp/yt-dlp/issues/6896)) by [bashonly](https://github.com/bashonly)
+- **daftsex**: [Update domain and embed player url](https://github.com/yt-dlp/yt-dlp/commit/fc5a7f9b27d2a89b1f3ca7d33a95301c21d832cd) ([#5966](https://github.com/yt-dlp/yt-dlp/issues/5966)) by [JChris246](https://github.com/JChris246)
+- **DigitalConcertHall**: [Support films](https://github.com/yt-dlp/yt-dlp/commit/55ed4ff73487feb3177b037dfc2ea527e777da3e) ([#7202](https://github.com/yt-dlp/yt-dlp/issues/7202)) by [ItzMaxTV](https://github.com/ItzMaxTV)
+- **discogs**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/6daaf21092888beff11b807cd46f832f1f9c46a0) ([#6624](https://github.com/yt-dlp/yt-dlp/issues/6624)) by [rjy](https://github.com/rjy)
+- **dlf**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/b423b6a48e0b19260bc95ab7d72d2138d7f124dc) ([#6697](https://github.com/yt-dlp/yt-dlp/issues/6697)) by [nick-cd](https://github.com/nick-cd)
+- **drtv**: [Fix radio page extraction](https://github.com/yt-dlp/yt-dlp/commit/9a06b7b1891b48cebbe275652ae8025a36d97d97) ([#6552](https://github.com/yt-dlp/yt-dlp/issues/6552)) by [viktor-enzell](https://github.com/viktor-enzell)
+- **Dumpert**: [Fix m3u8 and support new URL pattern](https://github.com/yt-dlp/yt-dlp/commit/f8ae441501596733e2b967430471643a1d7cacb8) ([#6091](https://github.com/yt-dlp/yt-dlp/issues/6091)) by [DataGhost](https://github.com/DataGhost), [pukkandan](https://github.com/pukkandan)
+- **elevensports**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/ecfe47973f6603b5367fe2cc3c65274627d94516) ([#7172](https://github.com/yt-dlp/yt-dlp/issues/7172)) by [ItzMaxTV](https://github.com/ItzMaxTV)
+- **ettutv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/83465fc4100a2fb2c188898fbc2f3021f6a9b4dd) ([#6579](https://github.com/yt-dlp/yt-dlp/issues/6579)) by [elyse0](https://github.com/elyse0)
+- **europarl**: [Rewrite extractor](https://github.com/yt-dlp/yt-dlp/commit/03789976d301eaed3e957dbc041573098f6af059) ([#7114](https://github.com/yt-dlp/yt-dlp/issues/7114)) by [HobbyistDev](https://github.com/HobbyistDev)
+- **eurosport**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/45e87ea106ad37b2a002663fa30ee41ce97b16cd) ([#7076](https://github.com/yt-dlp/yt-dlp/issues/7076)) by [HobbyistDev](https://github.com/HobbyistDev)
+- **facebook**: [Fix metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/3b52a606881e6adadc33444abdeacce562b79330) ([#6856](https://github.com/yt-dlp/yt-dlp/issues/6856)) by [ringus1](https://github.com/ringus1)
+- **foxnews**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/97d60ad8cd6c99f01e463a9acfce8693aff2a609) ([#7222](https://github.com/yt-dlp/yt-dlp/issues/7222)) by [bashonly](https://github.com/bashonly)
+- **funker530**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/cab94a0cd8b6d3fffed5a6faff030274adbed182) ([#7291](https://github.com/yt-dlp/yt-dlp/issues/7291)) by [Cyberes](https://github.com/Cyberes)
+- **generic**
+    - [Accept values for `fragment_query`, `variant_query`](https://github.com/yt-dlp/yt-dlp/commit/5cc0a8fd2e9fec50026fb92170b57993af939e4a) ([#6600](https://github.com/yt-dlp/yt-dlp/issues/6600)) by [bashonly](https://github.com/bashonly) (With fixes in [9bfe0d1](https://github.com/yt-dlp/yt-dlp/commit/9bfe0d15bd7dbdc6b0e6378fa9f5e2e289b2373b))
+    - [Add extractor-args `hls_key`, `variant_query`](https://github.com/yt-dlp/yt-dlp/commit/c2e0fc40a73dd85ab3920f977f579d475e66ef59) ([#6567](https://github.com/yt-dlp/yt-dlp/issues/6567)) by [bashonly](https://github.com/bashonly)
+    - [Attempt to detect live HLS](https://github.com/yt-dlp/yt-dlp/commit/93e7c6995e07dafb9dcc06c0d06acf6c5bdfecc5) ([#6775](https://github.com/yt-dlp/yt-dlp/issues/6775)) by [bashonly](https://github.com/bashonly)
+- **genius**: [Add support for articles](https://github.com/yt-dlp/yt-dlp/commit/460da07439718d9af1e3661da2a23e05a913a2e6) ([#6474](https://github.com/yt-dlp/yt-dlp/issues/6474)) by [bashonly](https://github.com/bashonly)
+- **globalplayer**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/30647668a92a0ca5cd108776804baac0996bd9f7) ([#6903](https://github.com/yt-dlp/yt-dlp/issues/6903)) by [garret1317](https://github.com/garret1317)
+- **gmanetwork**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/2d97d154fe4fb84fe2ed3a4e1ed5819e89b71e88) ([#5945](https://github.com/yt-dlp/yt-dlp/issues/5945)) by [HobbyistDev](https://github.com/HobbyistDev)
+- **gronkh**: [Extract duration and chapters](https://github.com/yt-dlp/yt-dlp/commit/9c92b803fa24e48543ce969468d5404376e315b7) ([#6817](https://github.com/yt-dlp/yt-dlp/issues/6817)) by [satan1st](https://github.com/satan1st)
+- **hentaistigma**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/04f8018a0544736a18494bc3899d06b05b78fae6) by [pukkandan](https://github.com/pukkandan)
+- **hidive**: [Fix login](https://github.com/yt-dlp/yt-dlp/commit/e6ab678e36c40ded0aae305bbb866cdab554d417) by [pukkandan](https://github.com/pukkandan)
+- **hollywoodreporter**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/6bdb64e2a2a6d504d8ce1dc830fbfb8a7f199c63) ([#6614](https://github.com/yt-dlp/yt-dlp/issues/6614)) by [bashonly](https://github.com/bashonly)
+- **hotstar**: [Support `/shows/` URLs](https://github.com/yt-dlp/yt-dlp/commit/7f8ddebbb51c9fd4a347306332a718ba41b371b8) ([#7225](https://github.com/yt-dlp/yt-dlp/issues/7225)) by [bashonly](https://github.com/bashonly)
+- **hrefli**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/7e35526d5b970a034b9d76215ee3e4bd7631edcd) ([#6762](https://github.com/yt-dlp/yt-dlp/issues/6762)) by [selfisekai](https://github.com/selfisekai)
+- **idolplus**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/5c14b213679ed4401288bdc86ae696932e219222) ([#6732](https://github.com/yt-dlp/yt-dlp/issues/6732)) by [ping](https://github.com/ping)
+- **iq**: [Set more language codes](https://github.com/yt-dlp/yt-dlp/commit/2d5cae9636714ff922d28c548c349d5f2b48f317) ([#6476](https://github.com/yt-dlp/yt-dlp/issues/6476)) by [D0LLYNH0](https://github.com/D0LLYNH0)
+- **iwara**
+    - [Accept old URLs](https://github.com/yt-dlp/yt-dlp/commit/ab92d8651c48d247dfb7d3f0a824cc986e47c7ed) by [Lesmiscore](https://github.com/Lesmiscore)
+    - [Fix authentication](https://github.com/yt-dlp/yt-dlp/commit/0a5d7c39e17bb9bd50c9db42bcad40eb82d7f784) ([#7137](https://github.com/yt-dlp/yt-dlp/issues/7137)) by [toomyzoom](https://github.com/toomyzoom)
+    - [Fix format sorting](https://github.com/yt-dlp/yt-dlp/commit/56793f74c36899742d7abd52afb0deca97d469e1) ([#6651](https://github.com/yt-dlp/yt-dlp/issues/6651)) by [hasezoey](https://github.com/hasezoey)
+    - [Fix typo](https://github.com/yt-dlp/yt-dlp/commit/d1483ec693c79f0b4ddf493870bcb840aca4da08) by [Lesmiscore](https://github.com/Lesmiscore)
+    - [Implement login](https://github.com/yt-dlp/yt-dlp/commit/21b9413cf7dd4830b2ece57af21589dd4538fc52) ([#6721](https://github.com/yt-dlp/yt-dlp/issues/6721)) by [toomyzoom](https://github.com/toomyzoom)
+    - [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/c14af7a741931b364bab3d9546c0f4359f318f8c) ([#6557](https://github.com/yt-dlp/yt-dlp/issues/6557)) by [Lesmiscore](https://github.com/Lesmiscore)
+    - [Report private videos](https://github.com/yt-dlp/yt-dlp/commit/95a383be1b6fb00c92ee3fb091732c4f6009acb6) ([#6641](https://github.com/yt-dlp/yt-dlp/issues/6641)) by [Lesmiscore](https://github.com/Lesmiscore)
+- **JStream**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/3459d3c5af3b2572ed51e8ecfda6c11022a838c6) ([#6252](https://github.com/yt-dlp/yt-dlp/issues/6252)) by [Lesmiscore](https://github.com/Lesmiscore)
+- **jwplatform**: [Update `_extract_embed_urls`](https://github.com/yt-dlp/yt-dlp/commit/cf9fd52fabe71d6e7c30d3ea525029ffa561fc9c) ([#6383](https://github.com/yt-dlp/yt-dlp/issues/6383)) by [carusocr](https://github.com/carusocr)
+- **kick**: [Make initial request non-fatal](https://github.com/yt-dlp/yt-dlp/commit/0a6918a4a1431960181d8c50e0bbbcb0afbaff9a) by [bashonly](https://github.com/bashonly)
+- **LastFM**: [Rewrite playlist extraction](https://github.com/yt-dlp/yt-dlp/commit/026435714cb7c39613a0d7d2acd15d3823b78d94) ([#6379](https://github.com/yt-dlp/yt-dlp/issues/6379)) by [hatienl0i261299](https://github.com/hatienl0i261299), [pukkandan](https://github.com/pukkandan)
+- **lbry**: [Extract original quality formats](https://github.com/yt-dlp/yt-dlp/commit/44c0d66442b568d9e1359e669d8b029b08a77fa7) ([#7257](https://github.com/yt-dlp/yt-dlp/issues/7257)) by [bashonly](https://github.com/bashonly)
+- **line**: [Remove extractors](https://github.com/yt-dlp/yt-dlp/commit/faa0332ed69e070cf3bd31390589a596e962f392) ([#6734](https://github.com/yt-dlp/yt-dlp/issues/6734)) by [sian1468](https://github.com/sian1468)
+- **livestream**: [Support videos with account id](https://github.com/yt-dlp/yt-dlp/commit/bfdf144c7e5d7a93fbfa9d8e65598c72bf2b542a) ([#6324](https://github.com/yt-dlp/yt-dlp/issues/6324)) by [theperfectpunk](https://github.com/theperfectpunk)
+- **medaltv**: [Fix clips](https://github.com/yt-dlp/yt-dlp/commit/1e3c2b6ec28d7ab5e31341fa93c47b65be4fbff4) ([#6502](https://github.com/yt-dlp/yt-dlp/issues/6502)) by [xenova](https://github.com/xenova)
+- **mediastream**: [Improve `WinSports` and embed extraction](https://github.com/yt-dlp/yt-dlp/commit/03025b6e105139d01cd415ddc51fd692957fd2ba) ([#6426](https://github.com/yt-dlp/yt-dlp/issues/6426)) by [bashonly](https://github.com/bashonly)
+- **mgtv**: [Fix formats extraction](https://github.com/yt-dlp/yt-dlp/commit/59d9fe08312bbb76ee26238d207a8ca35410a48d) ([#7234](https://github.com/yt-dlp/yt-dlp/issues/7234)) by [bashonly](https://github.com/bashonly)
+- **Mzaalo**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/dc3c44f349ba85af320e706e2a27ad81a78b1c6e) ([#7163](https://github.com/yt-dlp/yt-dlp/issues/7163)) by [ItzMaxTV](https://github.com/ItzMaxTV)
+- **nbc**: [Fix `NBCStations` direct mp4 formats](https://github.com/yt-dlp/yt-dlp/commit/9be0fe1fd967f62cbf3c60bd14e1021a70abc147) ([#6637](https://github.com/yt-dlp/yt-dlp/issues/6637)) by [bashonly](https://github.com/bashonly)
+- **nebula**: [Add `beta.nebula.tv`](https://github.com/yt-dlp/yt-dlp/commit/cbfe2e5cbe0f4649a91e323a82b8f5f774f36662) ([#6516](https://github.com/yt-dlp/yt-dlp/issues/6516)) by [unbeatable-101](https://github.com/unbeatable-101)
+- **nekohacker**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/489f51279d00318018478fd7461eddbe3b45297e) ([#7003](https://github.com/yt-dlp/yt-dlp/issues/7003)) by [hasezoey](https://github.com/hasezoey)
+- **nhk**
+    - [Add `NhkRadiru` extractor](https://github.com/yt-dlp/yt-dlp/commit/8f0be90ecb3b8d862397177bb226f17b245ef933) ([#6819](https://github.com/yt-dlp/yt-dlp/issues/6819)) by [garret1317](https://github.com/garret1317)
+    - [Fix API extraction](https://github.com/yt-dlp/yt-dlp/commit/f41b949a2ef646fbc36375febbe3f0c19d742c0f) ([#7180](https://github.com/yt-dlp/yt-dlp/issues/7180)) by [menschel](https://github.com/menschel), [sjthespian](https://github.com/sjthespian)
+    - `NhkRadiruLive`: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/81c8b9bdd9841b72cbfc1bbff9dab5fb4aa038b0) ([#7332](https://github.com/yt-dlp/yt-dlp/issues/7332)) by [garret1317](https://github.com/garret1317)
+- **niconico**
+    - [Download comments from the new endpoint](https://github.com/yt-dlp/yt-dlp/commit/52ecc33e221f7de7eb6fed6c22489f0c5fdd2c6d) ([#6773](https://github.com/yt-dlp/yt-dlp/issues/6773)) by [Lesmiscore](https://github.com/Lesmiscore)
+    - live: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/f8f9250fe280d37f0988646cd5cc0072f4d33a6d) ([#5764](https://github.com/yt-dlp/yt-dlp/issues/5764)) by [Lesmiscore](https://github.com/Lesmiscore)
+    - series: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/c86e433c35fe5da6cb29f3539eef97497f84ed38) ([#6898](https://github.com/yt-dlp/yt-dlp/issues/6898)) by [sqrtNOT](https://github.com/sqrtNOT)
+- **nubilesporn**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/d4e6ef40772e0560a8ed33b844ef7549e86837be) ([#6231](https://github.com/yt-dlp/yt-dlp/issues/6231)) by [permunkle](https://github.com/permunkle)
+- **odnoklassniki**: [Fix formats extraction](https://github.com/yt-dlp/yt-dlp/commit/1a2eb5bda51d8b7a78a65acebf72a0dcf9da196b) ([#7217](https://github.com/yt-dlp/yt-dlp/issues/7217)) by [bashonly](https://github.com/bashonly)
+- **opencast**
+    - [Add ltitools to `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/3588be59cee429a0ab5c4ceb2f162298bb44147d) ([#6371](https://github.com/yt-dlp/yt-dlp/issues/6371)) by [C0D3D3V](https://github.com/C0D3D3V)
+    - [Fix format bug](https://github.com/yt-dlp/yt-dlp/commit/89dbf0848370deaa55af88c3593a2a264124caf5) ([#6512](https://github.com/yt-dlp/yt-dlp/issues/6512)) by [C0D3D3V](https://github.com/C0D3D3V)
+- **owncloud**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/c6d4b82a8b8bce59b1c9ce5e6d349ea428dac0a7) ([#6533](https://github.com/yt-dlp/yt-dlp/issues/6533)) by [C0D3D3V](https://github.com/C0D3D3V)
+- **Parler**: [Rewrite extractor](https://github.com/yt-dlp/yt-dlp/commit/80ea6d3dea8483cddd39fc89b5ee1fc06670c33c) ([#6446](https://github.com/yt-dlp/yt-dlp/issues/6446)) by [JChris246](https://github.com/JChris246)
+- **pgatour**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/3ae182ad89e1427ff7b1684d6a44ff93fa857a0c) ([#6613](https://github.com/yt-dlp/yt-dlp/issues/6613)) by [bashonly](https://github.com/bashonly)
+- **playsuisse**: [Support new url format](https://github.com/yt-dlp/yt-dlp/commit/94627c5dde12a72766bdba36e056916c29c40ed1) ([#6528](https://github.com/yt-dlp/yt-dlp/issues/6528)) by [sbor23](https://github.com/sbor23)
+- **polskieradio**: [Improve extractors](https://github.com/yt-dlp/yt-dlp/commit/738c90a463257634455ada3e5c18b714c531dede) ([#5948](https://github.com/yt-dlp/yt-dlp/issues/5948)) by [selfisekai](https://github.com/selfisekai)
+- **pornez**: [Support new URL formats](https://github.com/yt-dlp/yt-dlp/commit/cbdf9408e6f1e35e98fd6477b3d6902df5b8a47f) ([#6792](https://github.com/yt-dlp/yt-dlp/issues/6792)) by [zhgwn](https://github.com/zhgwn)
+- **pornhub**: [Set access cookies to fix extraction](https://github.com/yt-dlp/yt-dlp/commit/62beefa818c75c20b6941389bb197051554a5d41) ([#6685](https://github.com/yt-dlp/yt-dlp/issues/6685)) by [arobase-che](https://github.com/arobase-che), [Schmoaaaaah](https://github.com/Schmoaaaaah)
+- **rai**: [Rewrite extractors](https://github.com/yt-dlp/yt-dlp/commit/c6d3f81a4077aaf9cffc6aa2d0dec92f38e74bb0) ([#5940](https://github.com/yt-dlp/yt-dlp/issues/5940)) by [danog](https://github.com/danog), [nixxo](https://github.com/nixxo)
+- **recurbate**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/c2502cfed91415c7ccfff925fd3404d230046484) ([#6297](https://github.com/yt-dlp/yt-dlp/issues/6297)) by [mrscrapy](https://github.com/mrscrapy)
+- **reddit**
+    - [Add login support](https://github.com/yt-dlp/yt-dlp/commit/4d9280c9c853733534dda60486fa949bcca36c9e) ([#6950](https://github.com/yt-dlp/yt-dlp/issues/6950)) by [bashonly](https://github.com/bashonly)
+    - [Support cookies and short URLs](https://github.com/yt-dlp/yt-dlp/commit/7a6f6f24592a8065376f11a58e44878807732cf6) ([#6825](https://github.com/yt-dlp/yt-dlp/issues/6825)) by [bashonly](https://github.com/bashonly)
+- **rokfin**: [Re-construct manifest url](https://github.com/yt-dlp/yt-dlp/commit/7a6c8a0807941dd24fbf0d6172e811884f98e027) ([#6507](https://github.com/yt-dlp/yt-dlp/issues/6507)) by [vampirefrog](https://github.com/vampirefrog)
+- **rottentomatoes**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/2d306c03d6f2697fcbabb7da35aa62cc078359d3) ([#6844](https://github.com/yt-dlp/yt-dlp/issues/6844)) by [JChris246](https://github.com/JChris246)
+- **rozhlas**
+    - [Extract manifest formats](https://github.com/yt-dlp/yt-dlp/commit/e4cf7741f9302b3faa092962f2895b55cb3d89bb) ([#6590](https://github.com/yt-dlp/yt-dlp/issues/6590)) by [bashonly](https://github.com/bashonly)
+    - `MujRozhlas`: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/c2b801fea59628d5c873e06a0727fbf2051bbd1f) ([#7129](https://github.com/yt-dlp/yt-dlp/issues/7129)) by [stanoarn](https://github.com/stanoarn)
+- **rtvc**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/9b30cd3dfce83c2f0201b28a7a3ef44ab9722664) ([#6578](https://github.com/yt-dlp/yt-dlp/issues/6578)) by [elyse0](https://github.com/elyse0)
+- **rumble**
+    - [Detect timeline format](https://github.com/yt-dlp/yt-dlp/commit/78bc1868ff3352108ab2911033d1ac67a55f151e) by [pukkandan](https://github.com/pukkandan)
+    - [Fix videos without quality selection](https://github.com/yt-dlp/yt-dlp/commit/6994afc030d2a786d8032075ed71a14d7eac5a4f) by [pukkandan](https://github.com/pukkandan)
+- **sbs**: [Overhaul extractor for new API](https://github.com/yt-dlp/yt-dlp/commit/6a765f135ccb654861336ea27a2c1c24ea8e286f) ([#6839](https://github.com/yt-dlp/yt-dlp/issues/6839)) by [bashonly](https://github.com/bashonly), [dirkf](https://github.com/dirkf), [vidiot720](https://github.com/vidiot720)
+- **shemaroome**: [Pass `stream_key` header to downloader](https://github.com/yt-dlp/yt-dlp/commit/7bc92517463f5766e9d9b92c3823b5cf403c0e3d) ([#7224](https://github.com/yt-dlp/yt-dlp/issues/7224)) by [bashonly](https://github.com/bashonly)
+- **sonyliv**: [Fix login with token](https://github.com/yt-dlp/yt-dlp/commit/4815d35c191e7d375b94492a6486dd2ba43a8954) ([#7223](https://github.com/yt-dlp/yt-dlp/issues/7223)) by [bashonly](https://github.com/bashonly)
+- **stageplus**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/e5265dc6517478e589ee3c1ff0cb19bdf4e35ce1) ([#6838](https://github.com/yt-dlp/yt-dlp/issues/6838)) by [bashonly](https://github.com/bashonly)
+- **stripchat**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/f9213f8a2d7ba46b912afe1dd3ce6bb700a33d72) ([#7306](https://github.com/yt-dlp/yt-dlp/issues/7306)) by [foreignBlade](https://github.com/foreignBlade)
+- **substack**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/12037d8b0a578fcc78a5c8f98964e48ee6060e25) ([#7218](https://github.com/yt-dlp/yt-dlp/issues/7218)) by [bashonly](https://github.com/bashonly)
+- **sverigesradio**: [Support slug URLs](https://github.com/yt-dlp/yt-dlp/commit/5ee9a7d6e18ceea956e831994cf11c423979354f) ([#7220](https://github.com/yt-dlp/yt-dlp/issues/7220)) by [bashonly](https://github.com/bashonly)
+- **tagesschau**: [Fix single audio urls](https://github.com/yt-dlp/yt-dlp/commit/af7585c824a1e405bd8afa46d87b4be322edc93c) ([#6626](https://github.com/yt-dlp/yt-dlp/issues/6626)) by [flashdagger](https://github.com/flashdagger)
+- **teamcoco**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/c459d45dd4d417fb80a52e1a04e607776a44baa4) ([#6437](https://github.com/yt-dlp/yt-dlp/issues/6437)) by [bashonly](https://github.com/bashonly)
+- **telecaribe**: [Expand livestream support](https://github.com/yt-dlp/yt-dlp/commit/69b2f838d3d3e37dc17367ef64d978db1bea45cf) ([#6601](https://github.com/yt-dlp/yt-dlp/issues/6601)) by [bashonly](https://github.com/bashonly)
+- **tencent**: [Fix fatal metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/971d901d129403e875a04dd92109507a03fbc070) ([#7219](https://github.com/yt-dlp/yt-dlp/issues/7219)) by [bashonly](https://github.com/bashonly)
+- **thesun**: [Update `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/0181b9a1b31db3fde943f7cd3fe9662f23bff292) ([#6522](https://github.com/yt-dlp/yt-dlp/issues/6522)) by [hatienl0i261299](https://github.com/hatienl0i261299)
+- **tiktok**
+    - [Extract 1080p adaptive formats](https://github.com/yt-dlp/yt-dlp/commit/c2a1bdb00931969193f2a31ea27b9c66a07aaec2) ([#7228](https://github.com/yt-dlp/yt-dlp/issues/7228)) by [bashonly](https://github.com/bashonly)
+    - [Fix and improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/925936908a3c3ee0e508621db14696b9f6a8b563) ([#6777](https://github.com/yt-dlp/yt-dlp/issues/6777)) by [bashonly](https://github.com/bashonly)
+    - [Fix mp3 formats](https://github.com/yt-dlp/yt-dlp/commit/8ceb07e870424c219dced8f4348729553f05c5cc) ([#6615](https://github.com/yt-dlp/yt-dlp/issues/6615)) by [bashonly](https://github.com/bashonly)
+    - [Fix resolution extraction](https://github.com/yt-dlp/yt-dlp/commit/ab6057ec80aa75db6303b8206916d00c376c622c) ([#7237](https://github.com/yt-dlp/yt-dlp/issues/7237)) by [puc9](https://github.com/puc9)
+    - [Improve `TikTokLive` extractor](https://github.com/yt-dlp/yt-dlp/commit/216bcb66d7dce0762767d751dad10650cb57da9d) ([#6520](https://github.com/yt-dlp/yt-dlp/issues/6520)) by [bashonly](https://github.com/bashonly)
+- **triller**: [Support short URLs, detect removed videos](https://github.com/yt-dlp/yt-dlp/commit/33b737bedf8383c0d00d4e1d06a5273dcdfdb756) ([#6636](https://github.com/yt-dlp/yt-dlp/issues/6636)) by [bashonly](https://github.com/bashonly)
+- **tv4**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/125ffaa1737dd04716f2f6fbb0595ad3eb7a4b1c) ([#5649](https://github.com/yt-dlp/yt-dlp/issues/5649)) by [dirkf](https://github.com/dirkf), [TxI5](https://github.com/TxI5)
+- **tvp**: [Use new API](https://github.com/yt-dlp/yt-dlp/commit/0c7ce146e4d2a84e656d78f6857952bfd25ab389) ([#6989](https://github.com/yt-dlp/yt-dlp/issues/6989)) by [selfisekai](https://github.com/selfisekai)
+- **tvplay**: [Remove outdated domains](https://github.com/yt-dlp/yt-dlp/commit/937264419f9bf375d5656785ae6e53282587c15d) ([#7106](https://github.com/yt-dlp/yt-dlp/issues/7106)) by [ivanskodje](https://github.com/ivanskodje)
+- **twitch**
+    - [Extract original size thumbnail](https://github.com/yt-dlp/yt-dlp/commit/80b732b7a9585b2a61e456dc0d2d014a439cbaee) ([#6629](https://github.com/yt-dlp/yt-dlp/issues/6629)) by [JC-Chung](https://github.com/JC-Chung)
+    - [Fix `is_live`](https://github.com/yt-dlp/yt-dlp/commit/0551511b45f7847f40e4314aa9e624e80d086539) ([#6500](https://github.com/yt-dlp/yt-dlp/issues/6500)) by [elyse0](https://github.com/elyse0)
+    - [Support mobile clips](https://github.com/yt-dlp/yt-dlp/commit/02312c03cf53eb1da24c9ad022ee79af26060733) ([#6699](https://github.com/yt-dlp/yt-dlp/issues/6699)) by [bepvte](https://github.com/bepvte)
+    - [Update `_CLIENT_ID` and add extractor-arg](https://github.com/yt-dlp/yt-dlp/commit/01231feb142e80828985aabdec04ac608e3d43e2) ([#7200](https://github.com/yt-dlp/yt-dlp/issues/7200)) by [bashonly](https://github.com/bashonly)
+    - vod: [Support links from schedule tab](https://github.com/yt-dlp/yt-dlp/commit/dbce5afa6bb61f6272ade613f2e9a3d66b88c7ea) ([#7071](https://github.com/yt-dlp/yt-dlp/issues/7071)) by [falbrechtskirchinger](https://github.com/falbrechtskirchinger)
+- **twitter**
+    - [Add login support](https://github.com/yt-dlp/yt-dlp/commit/d1795f4a6af99c976c9d3ea2dabe5cf4f8965d3c) ([#7258](https://github.com/yt-dlp/yt-dlp/issues/7258)) by [bashonly](https://github.com/bashonly)
+    - [Default to GraphQL, handle auth errors](https://github.com/yt-dlp/yt-dlp/commit/147e62fc584c3ea6fdb09bb7a47905df68553a22) ([#6957](https://github.com/yt-dlp/yt-dlp/issues/6957)) by [bashonly](https://github.com/bashonly)
+    - spaces: [Add `release_timestamp`](https://github.com/yt-dlp/yt-dlp/commit/1c16d9df5330819cc79ad588b24aa5b72765c168) ([#7186](https://github.com/yt-dlp/yt-dlp/issues/7186)) by [CeruleanSky](https://github.com/CeruleanSky)
+- **urplay**: [Extract all subtitles](https://github.com/yt-dlp/yt-dlp/commit/7bcd4813215ac98daa4949af2ffc677c78307a38) ([#7309](https://github.com/yt-dlp/yt-dlp/issues/7309)) by [hoaluvn](https://github.com/hoaluvn)
+- **voot**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/4f7b11cc1c1cebf598107e00cd7295588ed484da) ([#7227](https://github.com/yt-dlp/yt-dlp/issues/7227)) by [bashonly](https://github.com/bashonly)
+- **vrt**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/1a7dcca378e80a387923ee05c250d8ba122441c6) ([#6244](https://github.com/yt-dlp/yt-dlp/issues/6244)) by [bashonly](https://github.com/bashonly), [bergoid](https://github.com/bergoid), [jeroenj](https://github.com/jeroenj)
+- **weverse**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/b844a3f8b16500663e7ab6c6ec061cc9b30f71ac) ([#6711](https://github.com/yt-dlp/yt-dlp/issues/6711)) by [bashonly](https://github.com/bashonly) (With fixes in [fd5d93f](https://github.com/yt-dlp/yt-dlp/commit/fd5d93f7040f9776fd541f4e4079dad7d3b3fb4f))
+- **wevidi**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/1ea15603d852971ed7d92f4de12808b27b3d9370) ([#6868](https://github.com/yt-dlp/yt-dlp/issues/6868)) by [truedread](https://github.com/truedread)
+- **weyyak**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/6dc00acf0f1f1107a626c21befd1691403e6aeeb) ([#7124](https://github.com/yt-dlp/yt-dlp/issues/7124)) by [ItzMaxTV](https://github.com/ItzMaxTV)
+- **whyp**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/2c566ed14101673c651c08c306c30fa5b4010b85) ([#6803](https://github.com/yt-dlp/yt-dlp/issues/6803)) by [CoryTibbettsDev](https://github.com/CoryTibbettsDev)
+- **wrestleuniverse**
+    - [Fix cookies support](https://github.com/yt-dlp/yt-dlp/commit/c8561c6d03f025268d6d3972abeb47987c8d7cbb) by [bashonly](https://github.com/bashonly)
+    - [Fix extraction, add login](https://github.com/yt-dlp/yt-dlp/commit/ef8fb7f029b816dfc95600727d84400591a3b5c5) ([#6982](https://github.com/yt-dlp/yt-dlp/issues/6982)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+- **wykop**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/aed945e1b9b7d3af2a907e1a12e6508cc81d6a20) ([#6140](https://github.com/yt-dlp/yt-dlp/issues/6140)) by [selfisekai](https://github.com/selfisekai)
+- **ximalaya**: [Sort playlist entries](https://github.com/yt-dlp/yt-dlp/commit/8790ea7b2536332777bce68590386b1aa935fac7) ([#7292](https://github.com/yt-dlp/yt-dlp/issues/7292)) by [linsui](https://github.com/linsui)
+- **YahooGyaOIE, YahooGyaOPlayerIE**: [Delete extractors due to website close](https://github.com/yt-dlp/yt-dlp/commit/68be95bd0ca3f76aa63c9812935bd826b3a42e53) ([#6218](https://github.com/yt-dlp/yt-dlp/issues/6218)) by [Lesmiscore](https://github.com/Lesmiscore)
+- **yappy**: YappyProfile: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/6f69101dc912690338d32e2aab085c32e44eba3f) ([#7346](https://github.com/yt-dlp/yt-dlp/issues/7346)) by [7vlad7](https://github.com/7vlad7)
+- **youku**: [Improve error message](https://github.com/yt-dlp/yt-dlp/commit/ef0848abd425dfda6db62baa8d72897eefb0007f) ([#6690](https://github.com/yt-dlp/yt-dlp/issues/6690)) by [carusocr](https://github.com/carusocr)
+- **youporn**: [Extract m3u8 formats](https://github.com/yt-dlp/yt-dlp/commit/ddae33754ae1f32dd9c64cf895c47d20f6b5f336) by [pukkandan](https://github.com/pukkandan)
+- **youtube**
+    - [Add client name to `format_note` when `-v`](https://github.com/yt-dlp/yt-dlp/commit/c795c39f27244cbce846067891827e4847036441) ([#6254](https://github.com/yt-dlp/yt-dlp/issues/6254)) by [Lesmiscore](https://github.com/Lesmiscore), [pukkandan](https://github.com/pukkandan)
+    - [Add extractor-arg `include_duplicate_formats`](https://github.com/yt-dlp/yt-dlp/commit/86cb922118b236306310a72657f70426c20e28bb) by [pukkandan](https://github.com/pukkandan)
+    - [Bypass throttling for `-f17`](https://github.com/yt-dlp/yt-dlp/commit/c9abebb851e6188cb34b9eb744c1863dd46af919) by [pukkandan](https://github.com/pukkandan)
+    - [Construct fragment list lazily](https://github.com/yt-dlp/yt-dlp/commit/2a23d92d9ec44a0168079e38bcf3d383e5c4c7bb) by [pukkandan](https://github.com/pukkandan) (With fixes in [e389d17](https://github.com/yt-dlp/yt-dlp/commit/e389d172b6f42e4f332ae679dc48543fb7b9b61d))
+    - [Define strict uploader metadata mapping](https://github.com/yt-dlp/yt-dlp/commit/7666b93604b97e9ada981c6b04ccf5605dd1bd44) ([#6384](https://github.com/yt-dlp/yt-dlp/issues/6384)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Determine audio language using automatic captions](https://github.com/yt-dlp/yt-dlp/commit/ff9b0e071ffae5543cc309e6f9e647ac51e5846e) by [pukkandan](https://github.com/pukkandan)
+    - [Extract `channel_is_verified`](https://github.com/yt-dlp/yt-dlp/commit/8213ce28a485e200f6a7e1af1434a987c8e702bd) ([#7213](https://github.com/yt-dlp/yt-dlp/issues/7213)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Extract `heatmap` data](https://github.com/yt-dlp/yt-dlp/commit/5caf30dbc34f10b0be60676fece635b5c59f0d72) ([#7100](https://github.com/yt-dlp/yt-dlp/issues/7100)) by [tntmod54321](https://github.com/tntmod54321)
+    - [Extract more metadata for comments](https://github.com/yt-dlp/yt-dlp/commit/c35448b7b14113b35c4415dbfbf488c4731f006f) ([#7179](https://github.com/yt-dlp/yt-dlp/issues/7179)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Extract uploader metadata for feed/playlist items](https://github.com/yt-dlp/yt-dlp/commit/93e12ed76ef49252dc6869b59d21d0777e5e11af) by [coletdjnz](https://github.com/coletdjnz)
+    - [Fix comment loop detection for pinned comments](https://github.com/yt-dlp/yt-dlp/commit/141a8dff98874a426d7fbe772e0a8421bb42656f) ([#6714](https://github.com/yt-dlp/yt-dlp/issues/6714)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Fix continuation loop with no comments](https://github.com/yt-dlp/yt-dlp/commit/18f8fba7c89a87f99cc3313a1795848867e84fff) ([#7148](https://github.com/yt-dlp/yt-dlp/issues/7148)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Fix parsing `comment_count`](https://github.com/yt-dlp/yt-dlp/commit/071670cbeaa01ddf2cc20a95ae6da25f8f086431) ([#6523](https://github.com/yt-dlp/yt-dlp/issues/6523)) by [nick-cd](https://github.com/nick-cd)
+    - [Handle incomplete initial data from watch page](https://github.com/yt-dlp/yt-dlp/commit/607510b9f2f67bfe7d33d74031a5c1fe22a24862) ([#6510](https://github.com/yt-dlp/yt-dlp/issues/6510)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Ignore wrong fps of some formats](https://github.com/yt-dlp/yt-dlp/commit/97afb093d4cbe5df889145afa5f9ede4535e93e4) by [pukkandan](https://github.com/pukkandan)
+    - [Misc cleanup](https://github.com/yt-dlp/yt-dlp/commit/14a14335b280766fbf5a469ae26836d6c1fe450a) by [coletdjnz](https://github.com/coletdjnz)
+    - [Prioritize premium formats](https://github.com/yt-dlp/yt-dlp/commit/51a07b0dca4c079d58311c19b6d1c097c24bb021) by [pukkandan](https://github.com/pukkandan)
+    - [Revert default formats to `https`](https://github.com/yt-dlp/yt-dlp/commit/c6786ff3baaf72a5baa4d56d34058e54cbcf8ceb) by [pukkandan](https://github.com/pukkandan)
+    - [Support podcasts and releases tabs](https://github.com/yt-dlp/yt-dlp/commit/447afb9eaa65bc677e3245c83e53a8e69c174a3c) by [coletdjnz](https://github.com/coletdjnz)
+    - [Support shorter relative time format](https://github.com/yt-dlp/yt-dlp/commit/2fb35f6004c7625f0dd493da4a5abf0690f7777c) ([#7191](https://github.com/yt-dlp/yt-dlp/issues/7191)) by [coletdjnz](https://github.com/coletdjnz)
+    - music_search_url: [Extract title](https://github.com/yt-dlp/yt-dlp/commit/69a40e4a7f6caa5662527ebd2f3c4e8aa02857a2) ([#7102](https://github.com/yt-dlp/yt-dlp/issues/7102)) by [kangalio](https://github.com/kangalio)
+- **zaiko**
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/345b4c0aedd9d19898ce00d5cef35fe0d277a052) ([#7254](https://github.com/yt-dlp/yt-dlp/issues/7254)) by [c-basalt](https://github.com/c-basalt)
+    - ZaikoETicket: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/5cc09c004bd5edbbada9b041c08a720cadc4f4df) ([#7347](https://github.com/yt-dlp/yt-dlp/issues/7347)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **zdf**: [Fix formats extraction](https://github.com/yt-dlp/yt-dlp/commit/ee0ed0338df328cd986f97315c8162b5a151476d) by [bashonly](https://github.com/bashonly)
+- **zee5**: [Fix extraction of new content](https://github.com/yt-dlp/yt-dlp/commit/9d7fde89a40360396f0baa2ee8bf507f92108b32) ([#7280](https://github.com/yt-dlp/yt-dlp/issues/7280)) by [bashonly](https://github.com/bashonly)
+- **zingmp3**: [Fix and improve extractors](https://github.com/yt-dlp/yt-dlp/commit/17d7ca84ea723c20668bd9bfa938be7ea0e64f6b) ([#6367](https://github.com/yt-dlp/yt-dlp/issues/6367)) by [hatienl0i261299](https://github.com/hatienl0i261299)
+- **zoom**
+    - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/79c77e85b70ae3b9942d5a88c14d021a9bd24222) ([#6741](https://github.com/yt-dlp/yt-dlp/issues/6741)) by [shreyasminocha](https://github.com/shreyasminocha)
+    - [Fix share URL extraction](https://github.com/yt-dlp/yt-dlp/commit/90c1f5120694105496a6ad9e3ecfc6c25de6cae1) ([#6789](https://github.com/yt-dlp/yt-dlp/issues/6789)) by [bashonly](https://github.com/bashonly)
+
+#### Downloader changes
+- **curl**: [Fix progress reporting](https://github.com/yt-dlp/yt-dlp/commit/66aeaac9aa30b5959069ba84e53a5508232deb38) by [pukkandan](https://github.com/pukkandan)
+- **fragment**: [Do not sleep between fragments](https://github.com/yt-dlp/yt-dlp/commit/424f3bf03305088df6e01d62f7311be8601ad3f4) by [pukkandan](https://github.com/pukkandan)
+
+#### Postprocessor changes
+- [Fix chapters if duration is not extracted](https://github.com/yt-dlp/yt-dlp/commit/01ddec7e661bf90dc4c34e6924eb9d7629886cef) ([#6037](https://github.com/yt-dlp/yt-dlp/issues/6037)) by [bashonly](https://github.com/bashonly)
+- [Print newline for `--progress-template`](https://github.com/yt-dlp/yt-dlp/commit/13ff78095372fd98900a32572cf817994c07ccb5) by [pukkandan](https://github.com/pukkandan)
+- **EmbedThumbnail, FFmpegMetadata**: [Fix error on attaching thumbnails and info json for mkv/mka](https://github.com/yt-dlp/yt-dlp/commit/0f0875ed555514f32522a0f30554fb08825d5124) ([#6647](https://github.com/yt-dlp/yt-dlp/issues/6647)) by [Lesmiscore](https://github.com/Lesmiscore)
+- **FFmpegFixupM3u8PP**: [Check audio codec before fixup](https://github.com/yt-dlp/yt-dlp/commit/3f7e2bd80e3c5d8a1682f20a1b245fcd974f295d) ([#6778](https://github.com/yt-dlp/yt-dlp/issues/6778)) by [bashonly](https://github.com/bashonly)
+- **FixupDuplicateMoov**: [Fix bug in triggering](https://github.com/yt-dlp/yt-dlp/commit/26010b5cec50193b98ad7845d1d77450f9f14c2b) by [pukkandan](https://github.com/pukkandan)
+
+#### Misc. changes
+- [Add automatic duplicate issue detection](https://github.com/yt-dlp/yt-dlp/commit/15b2d3db1d40b0437fca79d8874d392aa54b3cdd) by [pukkandan](https://github.com/pukkandan)
+- **build**
+    - [Fix macOS target](https://github.com/yt-dlp/yt-dlp/commit/44a79958f0b596ee71e1eb25f158610aada29d1b) by [Grub4K](https://github.com/Grub4K)
+    - [Implement build verification using `--update-to`](https://github.com/yt-dlp/yt-dlp/commit/b73193c99aa23b135732408a5fcf655c68d731c6) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+    - [Pin `pyinstaller` version for MacOS](https://github.com/yt-dlp/yt-dlp/commit/427a8fafbb0e18c28d0ed7960be838d7b26b88d3) by [pukkandan](https://github.com/pukkandan)
+    - [Various build workflow improvements](https://github.com/yt-dlp/yt-dlp/commit/c4efa0aefec8daef1de62fd1693f13edf3c8b03c) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+- **cleanup**
+    - Miscellaneous
+        - [6f2287c](https://github.com/yt-dlp/yt-dlp/commit/6f2287cb18cbfb27518f068d868fa9390fee78ad) by [pukkandan](https://github.com/pukkandan)
+        - [ad54c91](https://github.com/yt-dlp/yt-dlp/commit/ad54c9130e793ce433bf9da334fa80df9f3aee58) by [freezboltz](https://github.com/freezboltz), [mikf](https://github.com/mikf), [pukkandan](https://github.com/pukkandan)
+- **cleanup, utils**: [Split into submodules](https://github.com/yt-dlp/yt-dlp/commit/69bec6730ec9d724bcedeab199d9d684d61423ba) ([#7090](https://github.com/yt-dlp/yt-dlp/issues/7090)) by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+- **cli_to_api**: [Add script](https://github.com/yt-dlp/yt-dlp/commit/46f1370e9af6f8af8762f67e27e5acb8f0c48a47) by [pukkandan](https://github.com/pukkandan)
+- **devscripts**: `make_changelog`: [Various improvements](https://github.com/yt-dlp/yt-dlp/commit/23c39a4beadee382060bb47fdaa21316ca707d38) by [Grub4K](https://github.com/Grub4K)
+- **docs**: [Misc improvements](https://github.com/yt-dlp/yt-dlp/commit/c8bc203fbf3bb09914e53f0833eed622ab7edbb9) by [pukkandan](https://github.com/pukkandan)
+
 ### 2023.03.04
 
 #### Extractor changes
diff --git a/README.md b/README.md
index d9a5e6cef..2f1fd9a0d 100644
--- a/README.md
+++ b/README.md
@@ -910,7 +910,7 @@ ## Authentication Options:
                                     either the path or its containing directory.
                                     Defaults to ~/.netrc
     --netrc-cmd NETRC_CMD           Command to execute to get the credentials
-                                    credentials for an extractor.
+                                    for an extractor.
     --video-password PASSWORD       Video password (vimeo, youku)
     --ap-mso MSO                    Adobe Pass multiple-system operator (TV
                                     provider) identifier, use --ap-list-mso for
diff --git a/supportedsites.md b/supportedsites.md
index f5c8c3829..882b272aa 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -150,7 +150,9 @@ # Supported sites
  - **bfmtv**
  - **bfmtv:article**
  - **bfmtv:live**
- - **BibelTV**
+ - **bibeltv:live**: BibelTV live program
+ - **bibeltv:series**: BibelTV series playlist
+ - **bibeltv:video**: BibelTV single video
  - **Bigflix**
  - **Bigo**
  - **Bild**: Bild.de
@@ -183,12 +185,17 @@ # Supported sites
  - **Bloomberg**
  - **BokeCC**
  - **BongaCams**
- - **BooyahClips**
  - **BostonGlobe**
  - **Box**
  - **BoxCastVideo**
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk
+ - **BrainPOP**: [*brainpop*](## "netrc machine")
+ - **BrainPOPELL**: [*brainpop*](## "netrc machine")
+ - **BrainPOPEsp**: [*brainpop*](## "netrc machine") BrainPOP Español
+ - **BrainPOPFr**: [*brainpop*](## "netrc machine") BrainPOP Français
+ - **BrainPOPIl**: [*brainpop*](## "netrc machine") BrainPOP Hebrew
+ - **BrainPOPJr**: [*brainpop*](## "netrc machine")
  - **BravoTV**
  - **Break**
  - **BreitBart**
@@ -207,6 +214,8 @@ # Supported sites
  - **CAM4**
  - **Camdemy**
  - **CamdemyFolder**
+ - **CamFMEpisode**
+ - **CamFMShow**
  - **CamModels**
  - **Camsoda**
  - **CamtasiaEmbed**
@@ -214,8 +223,6 @@ # Supported sites
  - **CanalAlpha**
  - **canalc2.tv**
  - **Canalplus**: mycanal.fr and piwiplus.fr
- - **Canvas**
- - **CanvasEen**: canvas.be and een.be
  - **CarambaTV**
  - **CarambaTVPage**
  - **CartoonNetwork**
@@ -225,8 +232,10 @@ # Supported sites
  - **CBSInteractive**
  - **CBSLocal**
  - **CBSLocalArticle**
+ - **CBSLocalLive**
  - **cbsnews**: CBS News
  - **cbsnews:embed**
+ - **cbsnews:live**: CBS News Livestream
  - **cbsnews:livevideo**: CBS News Live Videos
  - **cbssports**
  - **cbssports:embed**
@@ -252,6 +261,7 @@ # Supported sites
  - **CiscoLiveSession**
  - **ciscowebex**: Cisco Webex
  - **CJSW**
+ - **Clipchamp**
  - **cliphunter**
  - **Clippit**
  - **ClipRs**
@@ -271,6 +281,7 @@ # Supported sites
  - **CNNIndonesia**
  - **ComedyCentral**
  - **ComedyCentralTV**
+ - **ConanClassic**
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **CONtv**
  - **CookingChannel**
@@ -286,7 +297,10 @@ # Supported sites
  - **CrooksAndLiars**
  - **CrowdBunker**
  - **CrowdBunkerChannel**
+ - **Crtvg**
  - **crunchyroll**: [*crunchyroll*](## "netrc machine")
+ - **crunchyroll:artist**: [*crunchyroll*](## "netrc machine")
+ - **crunchyroll:music**: [*crunchyroll*](## "netrc machine")
  - **crunchyroll:playlist**: [*crunchyroll*](## "netrc machine")
  - **CSpan**: C-SPAN
  - **CSpanCongress**
@@ -301,6 +315,8 @@ # Supported sites
  - **CWTV**
  - **Cybrary**: [*cybrary*](## "netrc machine")
  - **CybraryCourse**: [*cybrary*](## "netrc machine")
+ - **DacastPlaylist**
+ - **DacastVOD**
  - **Daftsex**
  - **DagelijkseKost**: dagelijksekost.een.be
  - **DailyMail**
@@ -331,6 +347,7 @@ # Supported sites
  - **DigitalConcertHall**: [*digitalconcerthall*](## "netrc machine") DigitalConcertHall extractor
  - **DigitallySpeaking**
  - **Digiteka**
+ - **DiscogsReleasePlaylist**
  - **Discovery**
  - **DiscoveryLife**
  - **DiscoveryNetworksDe**
@@ -341,6 +358,8 @@ # Supported sites
  - **DiscoveryPlusItalyShow**
  - **Disney**
  - **DIYNetwork**
+ - **dlf**
+ - **dlf:corpus**: DLF Multi-feed Archives
  - **dlive:stream**
  - **dlive:vod**
  - **Dotsub**
@@ -378,6 +397,7 @@ # Supported sites
  - **EinsUndEinsTVRecordings**: [*1und1tv*](## "netrc machine")
  - **Einthusan**
  - **eitb.tv**
+ - **ElevenSports**
  - **EllenTube**
  - **EllenTubePlaylist**
  - **EllenTubeVideo**
@@ -400,6 +420,7 @@ # Supported sites
  - **ESPNArticle**
  - **ESPNCricInfo**
  - **EsriVideo**
+ - **EttuTv**
  - **Europa**
  - **EuroParlWebstream**
  - **EuropeanTour**
@@ -460,6 +481,7 @@ # Supported sites
  - **funimation:page**: [*funimation*](## "netrc machine")
  - **funimation:show**: [*funimation*](## "netrc machine")
  - **Funk**
+ - **Funker530**
  - **Fusion**
  - **Fux**
  - **FuyinTV**
@@ -493,10 +515,16 @@ # Supported sites
  - **GlattvisionTVLive**: [*glattvisiontv*](## "netrc machine")
  - **GlattvisionTVRecordings**: [*glattvisiontv*](## "netrc machine")
  - **Glide**: Glide mobile video messages (glide.me)
+ - **GlobalPlayerAudio**
+ - **GlobalPlayerAudioEpisode**
+ - **GlobalPlayerLive**
+ - **GlobalPlayerLivePlaylist**
+ - **GlobalPlayerVideo**
  - **Globo**: [*globo*](## "netrc machine")
  - **GloboArticle**
  - **glomex**: Glomex videos
  - **glomex:embed**: Glomex embedded videos
+ - **GMANetworkVideo**
  - **Go**
  - **GoDiscovery**
  - **GodTube**
@@ -522,7 +550,6 @@ # Supported sites
  - **Heise**
  - **HellPorno**
  - **Helsinki**: helsinki.fi
- - **HentaiStigma**
  - **hetklokhuis**
  - **hgtv.com:show**
  - **HGTVDe**
@@ -535,6 +562,8 @@ # Supported sites
  - **hitbox:live**
  - **HitRecord**
  - **hketv**: 香港教育局教育電視 (HKETV) Educational Television, Hong Kong Educational Bureau
+ - **HollywoodReporter**
+ - **HollywoodReporterPlaylist**
  - **Holodex**
  - **HotNewHipHop**
  - **hotstar**
@@ -558,6 +587,7 @@ # Supported sites
  - **Hypem**
  - **Hytale**
  - **Icareus**
+ - **IdolPlus**
  - **iflix:episode**
  - **IflixSeries**
  - **ign.com**
@@ -600,9 +630,9 @@ # Supported sites
  - **ivi:compilation**: ivi.ru compilations
  - **ivideon**: Ivideon TV
  - **IVXPlayer**
- - **Iwara**
- - **iwara:playlist**
- - **iwara:user**
+ - **iwara**: [*iwara*](## "netrc machine")
+ - **iwara:playlist**: [*iwara*](## "netrc machine")
+ - **iwara:user**: [*iwara*](## "netrc machine")
  - **Ixigua**
  - **Izlesene**
  - **Jable**
@@ -612,6 +642,7 @@ # Supported sites
  - **JeuxVideo**
  - **Joj**
  - **Jove**
+ - **JStream**
  - **JWPlatform**
  - **Kakao**
  - **Kaltura**
@@ -678,8 +709,6 @@ # Supported sites
  - **limelight**
  - **limelight:channel**
  - **limelight:channel_list**
- - **LineLive**
- - **LineLiveChannel**
  - **LinkedIn**: [*linkedin*](## "netrc machine")
  - **linkedin:learning**: [*linkedin*](## "netrc machine")
  - **linkedin:​learning:course**: [*linkedin*](## "netrc machine")
@@ -806,6 +835,7 @@ # Supported sites
  - **mtvservices:embedded**
  - **MTVUutisetArticle**
  - **MuenchenTV**: münchen.tv
+ - **MujRozhlas**
  - **Murrtube**
  - **MurrtubeUser**: Murrtube user profile
  - **MuseScore**
@@ -827,6 +857,7 @@ # Supported sites
  - **MyVideoGe**
  - **MyVidster**
  - **MyviEmbed**
+ - **Mzaalo**
  - **n-tv.de**
  - **N1Info:article**
  - **N1InfoAsset**
@@ -858,6 +889,7 @@ # Supported sites
  - **Nebula**: [*watchnebula*](## "netrc machine")
  - **nebula:channel**: [*watchnebula*](## "netrc machine")
  - **nebula:subscriptions**: [*watchnebula*](## "netrc machine")
+ - **NekoHacker**
  - **NerdCubedFeed**
  - **netease:album**: 网易云音乐 - 专辑
  - **netease:djradio**: 网易云音乐 - 电台
@@ -893,6 +925,9 @@ # Supported sites
  - **NhkForSchoolBangumi**
  - **NhkForSchoolProgramList**
  - **NhkForSchoolSubject**: Portal page for each school subjects, like Japanese (kokugo, 国語) or math (sansuu/suugaku or 算数・数学)
+ - **NhkRadioNewsPage**
+ - **NhkRadiru**: NHK らじる (Radiru/Rajiru)
+ - **NhkRadiruLive**
  - **NhkVod**
  - **NhkVodProgram**
  - **nhl.com**
@@ -903,6 +938,7 @@ # Supported sites
  - **nicknight**
  - **niconico**: [*niconico*](## "netrc machine") ニコニコ動画
  - **niconico:history**: NicoNico user history or likes. Requires cookies.
+ - **niconico:live**: ニコニコ生放送
  - **niconico:playlist**
  - **niconico:series**
  - **niconico:tag**: NicoNico video tag URLs
@@ -947,6 +983,7 @@ # Supported sites
  - **NRKTVSeries**
  - **NRLTV**
  - **ntv.ru**
+ - **NubilesPorn**: [*nubiles-porn*](## "netrc machine")
  - **Nuvid**
  - **NYTimes**
  - **NYTimesArticle**
@@ -987,6 +1024,7 @@ # Supported sites
  - **OsnatelTVLive**: [*osnateltv*](## "netrc machine")
  - **OsnatelTVRecordings**: [*osnateltv*](## "netrc machine")
  - **OutsideTV**
+ - **OwnCloud**
  - **PacktPub**: [*packtpub*](## "netrc machine")
  - **PacktPubCourse**
  - **PalcoMP3:artist**
@@ -999,6 +1037,7 @@ # Supported sites
  - **ParamountNetwork**
  - **ParamountPlus**
  - **ParamountPlusSeries**
+ - **ParamountPressExpress**
  - **Parler**: Posts on parler.com
  - **parliamentlive.tv**: UK parliament videos
  - **Parlview**
@@ -1016,6 +1055,7 @@ # Supported sites
  - **PerformGroup**
  - **periscope**: Periscope
  - **periscope:user**: Periscope user videos
+ - **PGATour**
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **phoenix.de**
  - **Photobucket**
@@ -1057,7 +1097,6 @@ # Supported sites
  - **PolskieRadio**
  - **polskieradio:audition**
  - **polskieradio:category**
- - **polskieradio:kierowcow**
  - **polskieradio:legacy**
  - **polskieradio:player**
  - **polskieradio:podcast**
@@ -1122,6 +1161,7 @@ # Supported sites
  - **radlive:channel**
  - **radlive:season**
  - **Rai**
+ - **RaiCultura**
  - **RaiNews**
  - **RaiPlay**
  - **RaiPlayLive**
@@ -1142,11 +1182,12 @@ # Supported sites
  - **RCTIPlusSeries**
  - **RCTIPlusTV**
  - **RDS**: RDS.ca
+ - **Recurbate**
  - **RedBull**
  - **RedBullEmbed**
  - **RedBullTV**
  - **RedBullTVRrnContent**
- - **Reddit**
+ - **Reddit**: [*reddit*](## "netrc machine")
  - **RedGifs**
  - **RedGifsSearch**: Redgifs search
  - **RedGifsUser**: Redgifs user
@@ -1186,6 +1227,9 @@ # Supported sites
  - **RTP**
  - **RTRFM**
  - **RTS**: RTS.ch
+ - **RTVCKaltura**
+ - **RTVCPlay**
+ - **RTVCPlayEmbed**
  - **rtve.es:alacarta**: RTVE a la carta
  - **rtve.es:audio**: RTVE audio
  - **rtve.es:infantil**: RTVE infantil
@@ -1239,6 +1283,7 @@ # Supported sites
  - **SCTE**: [*scte*](## "netrc machine")
  - **SCTECourse**: [*scte*](## "netrc machine")
  - **Seeker**
+ - **SenalColombiaLive**
  - **SenateGov**
  - **SenateISVP**
  - **SendtoNews**
@@ -1315,6 +1360,7 @@ # Supported sites
  - **sr:mediathek**: Saarländischer Rundfunk
  - **SRGSSR**
  - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
+ - **StagePlusVODConcert**: [*stageplus*](## "netrc machine")
  - **stanfordoc**: Stanford Open ClassRoom
  - **StarTrek**
  - **startv**
@@ -1427,6 +1473,7 @@ # Supported sites
  - **TrailerAddict**: (**Currently broken**)
  - **TravelChannel**
  - **Triller**: [*triller*](## "netrc machine")
+ - **TrillerShort**
  - **TrillerUser**: [*triller*](## "netrc machine")
  - **Trilulilu**
  - **Trovo**
@@ -1499,12 +1546,12 @@ # Supported sites
  - **TwitchVideos**: [*twitch*](## "netrc machine")
  - **TwitchVideosClips**: [*twitch*](## "netrc machine")
  - **TwitchVideosCollections**: [*twitch*](## "netrc machine")
- - **twitter**
- - **twitter:amplify**
- - **twitter:broadcast**
+ - **twitter**: [*twitter*](## "netrc machine")
+ - **twitter:amplify**: [*twitter*](## "netrc machine")
+ - **twitter:broadcast**: [*twitter*](## "netrc machine")
  - **twitter:card**
- - **twitter:shortener**
- - **twitter:spaces**
+ - **twitter:shortener**: [*twitter*](## "netrc machine")
+ - **twitter:spaces**: [*twitter*](## "netrc machine")
  - **Txxx**
  - **udemy**: [*udemy*](## "netrc machine")
  - **udemy:course**: [*udemy*](## "netrc machine")
@@ -1541,7 +1588,6 @@ # Supported sites
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
  - **vhx:embed**: [*vimeo*](## "netrc machine")
- - **Viafree**
  - **vice**
  - **vice:article**
  - **vice:show**
@@ -1607,8 +1653,8 @@ # Supported sites
  - **voicy**
  - **voicy:channel**
  - **VolejTV**
- - **Voot**
- - **VootSeries**
+ - **Voot**: [*voot*](## "netrc machine")
+ - **VootSeries**: [*voot*](## "netrc machine")
  - **VoxMedia**
  - **VoxMediaVolume**
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
@@ -1616,7 +1662,7 @@ # Supported sites
  - **vqq:video**
  - **Vrak**
  - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza
- - **VrtNU**: [*vrtnu*](## "netrc machine") VrtNU.be
+ - **VrtNU**: [*vrtnu*](## "netrc machine") VRT MAX
  - **vrv**: [*vrv*](## "netrc machine")
  - **vrv:series**
  - **VShare**
@@ -1660,7 +1706,16 @@ # Supported sites
  - **WeiqiTV**: WQTV
  - **wetv:episode**
  - **WeTvSeries**
+ - **Weverse**: [*weverse*](## "netrc machine")
+ - **WeverseLive**: [*weverse*](## "netrc machine")
+ - **WeverseLiveTab**: [*weverse*](## "netrc machine")
+ - **WeverseMedia**: [*weverse*](## "netrc machine")
+ - **WeverseMediaTab**: [*weverse*](## "netrc machine")
+ - **WeverseMoment**: [*weverse*](## "netrc machine")
+ - **WeVidi**
+ - **Weyyak**
  - **whowatch**
+ - **Whyp**
  - **wikimedia.org**
  - **Willow**
  - **WimTV**
@@ -1674,13 +1729,17 @@ # Supported sites
  - **WorldStarHipHop**
  - **wppilot**
  - **wppilot:channels**
- - **WrestleUniversePPV**
- - **WrestleUniverseVOD**
+ - **WrestleUniversePPV**: [*wrestleuniverse*](## "netrc machine")
+ - **WrestleUniverseVOD**: [*wrestleuniverse*](## "netrc machine")
  - **WSJ**: Wall Street Journal
  - **WSJArticle**
  - **WWE**
  - **wyborcza:video**
  - **WyborczaPodcast**
+ - **wykop:dig**
+ - **wykop:​dig:comment**
+ - **wykop:post**
+ - **wykop:​post:comment**
  - **Xanimu**
  - **XBef**
  - **XboxClips**
@@ -1701,8 +1760,6 @@ # Supported sites
  - **xvideos:quickies**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
- - **yahoo:gyao**
- - **yahoo:​gyao:player**
  - **yahoo:japannews**: Yahoo! Japan News
  - **YandexDisk**
  - **yandexmusic:album**: Яндекс.Музыка - Альбом
@@ -1714,6 +1771,7 @@ # Supported sites
  - **YandexVideoPreview**
  - **YapFiles**
  - **Yappy**
+ - **YappyProfile**
  - **YesJapan**
  - **yinyuetai:video**: 音悦Tai
  - **YleAreena**
@@ -1746,6 +1804,8 @@ # Supported sites
  - **youtube:watchlater**: Youtube watch later list; ":ytwatchlater" keyword (requires cookies)
  - **YoutubeLivestreamEmbed**: YouTube livestream embeds
  - **YoutubeYtBe**: youtu.be
+ - **Zaiko**
+ - **ZaikoETicket**
  - **Zapiks**
  - **Zattoo**: [*zattoo*](## "netrc machine")
  - **ZattooLive**: [*zattoo*](## "netrc machine")
@@ -1763,6 +1823,7 @@ # Supported sites
  - **zingmp3:album**
  - **zingmp3:chart-home**
  - **zingmp3:chart-music-video**
+ - **zingmp3:hub**
  - **zingmp3:user**
  - **zingmp3:week-chart**
  - **zoom**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index 04bece038..f4474db9a 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.03.04'
+__version__ = '2023.06.21'
 
-RELEASE_GIT_HEAD = '392389b7df7b818f794b231f14dc396d4875fbad'
+RELEASE_GIT_HEAD = '42f2d40b475db66486a4b4fe5b56751a640db5db'
 
 VARIANT = None
 

From d7cd97e8d8d42b500fea9abb2aa4ac9b0f98b2ad Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 12:12:15 +0530
Subject: [PATCH 384/871] Fix bug in db3ad8a67661d7b234a6954d9c6a4a9b1749f5eb

Closes #7367
---
 README.md                  | 2 +-
 yt_dlp/extractor/common.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2f1fd9a0d..a2bc33fbd 100644
--- a/README.md
+++ b/README.md
@@ -1221,7 +1221,7 @@ ### Authentication with netrc
 
 The default location of the .netrc file is `~` (see below).
 
-As an alternative to using the `.netrc` file, which has the disadvantage of keeping your passwords in a plain text file, you can configure a custom shell command to provide the credentials for an extractor. This is done by providing the `--netrc-cmd` parameter, it shall output the credentials in the netrc format and return `0` on success, other values will be treated as an error. `{}` in the command will be replaced by the name of the extractor to make it possible to select the credentials for the right extractor (To use literal braces, double them like `{{}}`).
+As an alternative to using the `.netrc` file, which has the disadvantage of keeping your passwords in a plain text file, you can configure a custom shell command to provide the credentials for an extractor. This is done by providing the `--netrc-cmd` parameter, it shall output the credentials in the netrc format and return `0` on success, other values will be treated as an error. `{}` in the command will be replaced by the name of the extractor to make it possible to select the credentials for the right extractor.
 
 E.g. To use an encrypted `.netrc` file stored as `.authinfo.gpg`
 ```
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 9662a7ee1..2ea36c63d 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -1297,8 +1297,9 @@ def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=Tr
     def _get_netrc_login_info(self, netrc_machine=None):
         netrc_machine = netrc_machine or self._NETRC_MACHINE
 
-        cmd = self.get_param('netrc_cmd', '').format(netrc_machine)
+        cmd = self.get_param('netrc_cmd')
         if cmd:
+            cmd = cmd.replace('{}', netrc_machine)
             self.to_screen(f'Executing command: {cmd}')
             stdout, _, ret = Popen.run(cmd, text=True, shell=True, stdout=subprocess.PIPE)
             if ret != 0:

From db22142f6f817ff673d417b4b78e8db497bf8ab3 Mon Sep 17 00:00:00 2001
From: OverlordQ <overlordq@gmail.com>
Date: Wed, 21 Jun 2023 03:17:07 -0400
Subject: [PATCH 385/871] [extractor/dropout] Fix season extraction (#7304)

Authored by: OverlordQ
---
 yt_dlp/extractor/dropout.py | 54 ++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/yt_dlp/extractor/dropout.py b/yt_dlp/extractor/dropout.py
index e280b1c9f..80ae6c126 100644
--- a/yt_dlp/extractor/dropout.py
+++ b/yt_dlp/extractor/dropout.py
@@ -1,13 +1,17 @@
+import functools
+
 from .common import InfoExtractor
 from .vimeo import VHXEmbedIE
 from ..utils import (
     ExtractorError,
+    OnDemandPagedList,
     clean_html,
+    extract_attributes,
     get_element_by_class,
     get_element_by_id,
-    get_elements_by_class,
+    get_elements_html_by_class,
     int_or_none,
-    join_nonempty,
+    traverse_obj,
     unified_strdate,
     urlencode_postdata,
 )
@@ -162,12 +166,13 @@ def _real_extract(self, url):
 
 
 class DropoutSeasonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?P<id>[^\/$&?#]+)(?:/?$|/season:[0-9]+/?$)'
+    _PAGE_SIZE = 24
+    _VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?P<id>[^\/$&?#]+)(?:/?$|/season:(?P<season>[0-9]+)/?$)'
     _TESTS = [
         {
             'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1',
             'note': 'Multi-season series with the season in the url',
-            'playlist_count': 17,
+            'playlist_count': 24,
             'info_dict': {
                 'id': 'dimension-20-fantasy-high-season-1',
                 'title': 'Dimension 20 Fantasy High - Season 1'
@@ -176,7 +181,7 @@ class DropoutSeasonIE(InfoExtractor):
         {
             'url': 'https://www.dropout.tv/dimension-20-fantasy-high',
             'note': 'Multi-season series with the season not in the url',
-            'playlist_count': 17,
+            'playlist_count': 24,
             'info_dict': {
                 'id': 'dimension-20-fantasy-high-season-1',
                 'title': 'Dimension 20 Fantasy High - Season 1'
@@ -190,29 +195,30 @@ class DropoutSeasonIE(InfoExtractor):
                 'id': 'dimension-20-shriek-week-season-1',
                 'title': 'Dimension 20 Shriek Week - Season 1'
             }
+        },
+        {
+            'url': 'https://www.dropout.tv/breaking-news-no-laugh-newsroom/season:3',
+            'note': 'Multi-season series with season in the url that requires pagination',
+            'playlist_count': 25,
+            'info_dict': {
+                'id': 'breaking-news-no-laugh-newsroom-season-3',
+                'title': 'Breaking News No Laugh Newsroom - Season 3'
+            }
         }
     ]
 
+    def _fetch_page(self, url, season_id, page):
+        page += 1
+        webpage = self._download_webpage(
+            f'{url}?page={page}', season_id, note=f'Downloading page {page}', expected_status={400})
+        yield from [self.url_result(item_url, DropoutIE) for item_url in traverse_obj(
+            get_elements_html_by_class('browse-item-link', webpage), (..., {extract_attributes}, 'href'))]
+
     def _real_extract(self, url):
         season_id = self._match_id(url)
+        season_num = self._match_valid_url(url).group('season') or 1
         season_title = season_id.replace('-', ' ').title()
-        webpage = self._download_webpage(url, season_id)
 
-        entries = [
-            self.url_result(
-                url=self._search_regex(r'<a href=["\'](.+?)["\'] class=["\']browse-item-link["\']',
-                                       item, 'item_url'),
-                ie=DropoutIE.ie_key()
-            ) for item in get_elements_by_class('js-collection-item', webpage)
-        ]
-
-        seasons = (get_element_by_class('select-dropdown-wrapper', webpage) or '').strip().replace('\n', '')
-        current_season = self._search_regex(r'<option[^>]+selected>([^<]+)</option>',
-                                            seasons, 'current_season', default='').strip()
-
-        return {
-            '_type': 'playlist',
-            'id': join_nonempty(season_id, current_season.lower().replace(' ', '-')),
-            'title': join_nonempty(season_title, current_season, delim=' - '),
-            'entries': entries
-        }
+        return self.playlist_result(
+            OnDemandPagedList(functools.partial(self._fetch_page, url, season_id), self._PAGE_SIZE),
+            f'{season_id}-season-{season_num}', f'{season_title} - Season {season_num}')

From 7f9c6a63b16e145495479e9f666f5b9e2ee69e2f Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Wed, 21 Jun 2023 03:24:24 -0500
Subject: [PATCH 386/871] [cleanup] Misc

Authored by: bashonly
---
 README.md                  | 8 ++++----
 yt_dlp/extractor/twitch.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index a2bc33fbd..d9b7a9e5d 100644
--- a/README.md
+++ b/README.md
@@ -1780,7 +1780,7 @@ # Do not set any "synopsis" in the video metadata
 $ yt-dlp --parse-metadata ":(?P<meta_synopsis>)"
 
 # Remove "formats" field from the infojson by setting it to an empty string
-$ yt-dlp --parse-metadata ":(?P<formats>)" -j
+$ yt-dlp --parse-metadata "video::(?P<formats>)" --write-info-json
 
 # Replace all spaces and "_" in title and uploader with a `-`
 $ yt-dlp --replace-in-metadata "title,uploader" "[ _]" "-"
@@ -1854,11 +1854,11 @@ #### rokfinchannel
 #### twitter
 * `legacy_api`: Force usage of the legacy Twitter API instead of the GraphQL API for tweet extraction. Has no effect if login cookies are passed
 
-### wrestleuniverse
+#### wrestleuniverse
 * `device_id`: UUID value assigned by the website and used to enforce device limits for paid livestream content. Can be found in browser local storage
 
-#### twitchstream (Twitch)
-* `client_id`: Client ID value to be sent with GraphQL requests, e.g. `twitchstream:client_id=kimne78kx3ncx6brgo4mv6wki5h1ko`
+#### twitch
+* `client_id`: Client ID value to be sent with GraphQL requests, e.g. `twitch:client_id=kimne78kx3ncx6brgo4mv6wki5h1ko`
 
 #### nhkradirulive (NHK らじる★らじる LIVE)
 * `area`: Which regional variation to extract. Valid areas are: `sapporo`, `sendai`, `tokyo`, `nagoya`, `osaka`, `hiroshima`, `matsuyama`, `fukuoka`. Defaults to `tokyo`
diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index 31b349bc6..c8ee52014 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -60,7 +60,7 @@ class TwitchBaseIE(InfoExtractor):
     @property
     def _CLIENT_ID(self):
         return self._configuration_arg(
-            'client_id', ['ue6666qo983tsx6so1t0vnawi233wa'], ie_key=TwitchStreamIE, casesense=True)[0]
+            'client_id', ['ue6666qo983tsx6so1t0vnawi233wa'], ie_key='Twitch', casesense=True)[0]
 
     def _perform_login(self, username, password):
         def fail(message):

From 3f756c8c4095b942cf49788eb0862ceaf57847f2 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@users.noreply.github.com>
Date: Wed, 21 Jun 2023 10:29:34 +0200
Subject: [PATCH 387/871] [extractor/nebula] Fix extractor (#7156)

Closes #7017
Authored by: Lamieur, rohieb

Co-authored-by: Lam <github@Lam.pl>
---
 yt_dlp/extractor/nebula.py | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py
index 5c1b7c712..7a5a02dfa 100644
--- a/yt_dlp/extractor/nebula.py
+++ b/yt_dlp/extractor/nebula.py
@@ -3,7 +3,7 @@
 import urllib.error
 
 from .common import InfoExtractor
-from ..utils import ExtractorError, parse_iso8601
+from ..utils import ExtractorError, make_archive_id, parse_iso8601, remove_start
 
 _BASE_URL_RE = r'https?://(?:www\.|beta\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)'
 
@@ -65,19 +65,20 @@ def _fetch_nebula_bearer_token(self):
         return response['token']
 
     def _fetch_video_formats(self, slug):
-        stream_info = self._call_nebula_api(f'https://content.watchnebula.com/video/{slug}/stream/',
+        stream_info = self._call_nebula_api(f'https://content.api.nebula.app/video/{slug}/stream/',
                                             video_id=slug,
                                             auth_type='bearer',
                                             note='Fetching video stream info')
         manifest_url = stream_info['manifest']
-        return self._extract_m3u8_formats_and_subtitles(manifest_url, slug)
+        return self._extract_m3u8_formats_and_subtitles(manifest_url, slug, 'mp4')
 
     def _build_video_info(self, episode):
         fmts, subs = self._fetch_video_formats(episode['slug'])
         channel_slug = episode['channel_slug']
         channel_title = episode['channel_title']
+        zype_id = episode.get('zype_id')
         return {
-            'id': episode['zype_id'],
+            'id': remove_start(episode['id'], 'video_episode:'),
             'display_id': episode['slug'],
             'formats': fmts,
             'subtitles': subs,
@@ -99,6 +100,9 @@ def _build_video_info(self, episode):
             'uploader_url': f'https://nebula.tv/{channel_slug}',
             'series': channel_title,
             'creator': channel_title,
+            'extractor_key': NebulaIE.ie_key(),
+            'extractor': NebulaIE.IE_NAME,
+            '_old_archive_ids': [make_archive_id(NebulaIE, zype_id)] if zype_id else None,
         }
 
     def _perform_login(self, username=None, password=None):
@@ -113,7 +117,7 @@ class NebulaIE(NebulaBaseIE):
             'url': 'https://nebula.tv/videos/that-time-disney-remade-beauty-and-the-beast',
             'md5': '14944cfee8c7beeea106320c47560efc',
             'info_dict': {
-                'id': '5c271b40b13fd613090034fd',
+                'id': '84ed544d-4afd-4723-8cd5-2b95261f0abf',
                 'ext': 'mp4',
                 'title': 'That Time Disney Remade Beauty and the Beast',
                 'description': 'Note: this video was originally posted on YouTube with the sponsor read included. We weren’t able to remove it without reducing video quality, so it’s presented here in its original context.',
@@ -137,22 +141,22 @@ class NebulaIE(NebulaBaseIE):
             'url': 'https://nebula.tv/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
             'md5': 'd05739cf6c38c09322422f696b569c23',
             'info_dict': {
-                'id': '5e7e78171aaf320001fbd6be',
+                'id': '7e623145-1b44-4ca3-aa0b-ed25a247ea34',
                 'ext': 'mp4',
                 'title': 'Landing Craft - How The Allies Got Ashore',
                 'description': r're:^In this episode we explore the unsung heroes of D-Day, the landing craft.',
                 'upload_date': '20200327',
                 'timestamp': 1585348140,
-                'channel': 'Real Engineering',
-                'channel_id': 'realengineering',
-                'uploader': 'Real Engineering',
-                'uploader_id': 'realengineering',
-                'series': 'Real Engineering',
+                'channel': 'Real Engineering — The Logistics of D-Day',
+                'channel_id': 'd-day',
+                'uploader': 'Real Engineering — The Logistics of D-Day',
+                'uploader_id': 'd-day',
+                'series': 'Real Engineering — The Logistics of D-Day',
                 'display_id': 'the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
-                'creator': 'Real Engineering',
+                'creator': 'Real Engineering — The Logistics of D-Day',
                 'duration': 841,
-                'channel_url': 'https://nebula.tv/realengineering',
-                'uploader_url': 'https://nebula.tv/realengineering',
+                'channel_url': 'https://nebula.tv/d-day',
+                'uploader_url': 'https://nebula.tv/d-day',
                 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*',
             },
         },
@@ -160,7 +164,7 @@ class NebulaIE(NebulaBaseIE):
             'url': 'https://nebula.tv/videos/money-episode-1-the-draw',
             'md5': 'ebe28a7ad822b9ee172387d860487868',
             'info_dict': {
-                'id': '5e779ebdd157bc0001d1c75a',
+                'id': 'b96c5714-9e2b-4ec3-b3f1-20f6e89cc553',
                 'ext': 'mp4',
                 'title': 'Episode 1: The Draw',
                 'description': r'contains:There’s free money on offer… if the players can all work together.',
@@ -190,7 +194,7 @@ class NebulaIE(NebulaBaseIE):
     ]
 
     def _fetch_video_metadata(self, slug):
-        return self._call_nebula_api(f'https://content.watchnebula.com/video/{slug}/',
+        return self._call_nebula_api(f'https://content.api.nebula.app/video/{slug}/',
                                      video_id=slug,
                                      auth_type='bearer',
                                      note='Fetching video meta data')

From a4486bfc1dc7057efca9dd3fe70d7fa25c56f700 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 21 Jun 2023 12:35:14 +0530
Subject: [PATCH 388/871] Revert "[misc] Add automatic duplicate issue
 detection"

This reverts commit 15b2d3db1d40b0437fca79d8874d392aa54b3cdd.
---
 .github/workflows/potential-duplicates.yml | 20 --------------------
 1 file changed, 20 deletions(-)
 delete mode 100644 .github/workflows/potential-duplicates.yml

diff --git a/.github/workflows/potential-duplicates.yml b/.github/workflows/potential-duplicates.yml
deleted file mode 100644
index cfc583186..000000000
--- a/.github/workflows/potential-duplicates.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-name: Potential Duplicates
-on:
-  issues:
-    types: [opened, edited]
-
-jobs:
-  run:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: wow-actions/potential-duplicates@v1
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          label: potential-duplicate
-          state: all
-          threshold: 0.3
-          comment: |
-            This issue is potentially a duplicate of one of the following issues:
-            {{#issues}}
-              - #{{ number }} ({{ accuracy }}%)
-            {{/issues}}

From 81ca451480051d7ce1a31c017e005358345a9149 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 22 Jun 2023 00:15:22 +0530
Subject: [PATCH 389/871] [extractor/youtube] Workaround 403 for android
 formats

Ref: https://github.com/TeamNewPipe/NewPipe/issues/9038#issuecomment-1289756816
---
 yt_dlp/extractor/youtube.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 11e47904a..a77a626ac 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3599,7 +3599,7 @@ def _is_agegated(player_response):
     def _is_unplayable(player_response):
         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
 
-    _STORY_PLAYER_PARAMS = '8AEB'
+    _PLAYER_PARAMS = 'CgIQBg=='
 
     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
 
@@ -3613,7 +3613,7 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
             'videoId': video_id,
         }
         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
-            yt_query['params'] = self._STORY_PLAYER_PARAMS
+            yt_query['params'] = self._PLAYER_PARAMS
 
         yt_query.update(self._generate_player_context(sts))
         return self._extract_response(
@@ -4011,8 +4011,8 @@ def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
         webpage = None
         if 'webpage' not in self._configuration_arg('player_skip'):
             query = {'bpctr': '9999999999', 'has_verified': '1'}
-            if smuggled_data.get('is_story'):
-                query['pp'] = self._STORY_PLAYER_PARAMS
+            if smuggled_data.get('is_story'):  # XXX: Deprecated
+                query['pp'] = self._PLAYER_PARAMS
             webpage = self._download_webpage(
                 webpage_url, video_id, fatal=False, query=query)
 

From 1e75d97db21152acc764b30a688e516f04b8a142 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 22 Jun 2023 00:20:04 +0530
Subject: [PATCH 390/871] [extractor/youtube] Add `ios` to default clients used

* IOS is affected neither by 403 or by nsig so helps mitigate them preemptively
* IOS also has higher bit-rate "premium" formats though they are not labeled as such
---
 README.md                   | 2 +-
 yt_dlp/extractor/youtube.py | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index d9b7a9e5d..d10832103 100644
--- a/README.md
+++ b/README.md
@@ -1798,7 +1798,7 @@ # EXTRACTOR ARGUMENTS
 #### youtube
 * `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube.py](https://github.com/yt-dlp/yt-dlp/blob/c26f9b991a0681fd3ea548d535919cec1fbbd430/yt_dlp/extractor/youtube.py#L381-L390) for list of supported content language codes
 * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
-* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb` and `tv_embedded` (agegate bypass) with no variants. By default, `android,web` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly, the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
+* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb` and `tv_embedded` (agegate bypass) with no variants. By default, `ios,android,web` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly, the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
 * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
 * `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all`
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index a77a626ac..a90118680 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -258,7 +258,7 @@ def build_innertube_clients():
     THIRD_PARTY = {
         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
     }
-    BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
+    BASE_CLIENTS = ('ios', 'android', 'web', 'tv', 'mweb')
     priority = qualities(BASE_CLIENTS[::-1])
 
     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
@@ -3625,7 +3625,7 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
 
     def _get_requested_clients(self, url, smuggled_data):
         requested_clients = []
-        default = ['android', 'web']
+        default = ['ios', 'android', 'web']
         allowed_clients = sorted(
             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
@@ -3932,6 +3932,10 @@ def process_manifest_format(f, proto, client_name, itag):
             elif itag:
                 f['format_id'] = itag
 
+            if itag in ('616', '235'):
+                f['format_note'] = join_nonempty(f.get('format_note'), 'Premium', delim=' ')
+                f['source_preference'] = (f.get('source_preference') or -1) + 100
+
             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
             if f['quality'] == -1 and f.get('height'):
                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])

From 0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 22 Jun 2023 01:37:55 +0530
Subject: [PATCH 391/871] Indicate `filesize` approximated from `tbr` better

---
 yt_dlp/YoutubeDL.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index bc5c1b95e..79b7d47b0 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2666,7 +2666,8 @@ def is_wellformed(f):
                 format['dynamic_range'] = 'SDR'
             if format.get('aspect_ratio') is None:
                 format['aspect_ratio'] = try_call(lambda: round(format['width'] / format['height'], 2))
-            if (info_dict.get('duration') and format.get('tbr')
+            if (not format.get('manifest_url')  # For fragmented formats, "tbr" is often max bitrate and not average
+                    and info_dict.get('duration') and format.get('tbr')
                     and not format.get('filesize') and not format.get('filesize_approx')):
                 format['filesize_approx'] = int(info_dict['duration'] * format['tbr'] * (1024 / 8))
             format['http_headers'] = self._calc_headers(collections.ChainMap(format, info_dict))
@@ -3707,8 +3708,11 @@ def simplified_codec(f, field):
                 format_field(f, 'fps', '\t%d', func=round),
                 format_field(f, 'dynamic_range', '%s', ignore=(None, 'SDR')).replace('HDR', ''),
                 format_field(f, 'audio_channels', '\t%s'),
-                delim,
-                format_field(f, 'filesize', ' \t%s', func=format_bytes) + format_field(f, 'filesize_approx', '~\t%s', func=format_bytes),
+                delim, (
+                    format_field(f, 'filesize', ' \t%s', func=format_bytes)
+                    or format_field(f, 'filesize_approx', '≈\t%s', func=format_bytes)
+                    or format_field(try_call(lambda: format_bytes(int(info_dict['duration'] * f['tbr'] * (1024 / 8)))),
+                                    None, self._format_out('~\t%s', self.Styles.SUPPRESS))),
                 format_field(f, 'tbr', '\t%dk', func=round),
                 shorten_protocol_name(f.get('protocol', '')),
                 delim,

From 5fd8367496b42c7b900b896a0d5460561a2859de Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 22 Jun 2023 02:57:00 +0530
Subject: [PATCH 392/871] [extractor] Support multiple `_VALID_URL`s (#5812)

Authored by: nixxo
---
 devscripts/lazy_load_template.py | 1 +
 yt_dlp/extractor/common.py       | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py
index c8815e01b..6f52165c5 100644
--- a/devscripts/lazy_load_template.py
+++ b/devscripts/lazy_load_template.py
@@ -6,6 +6,7 @@
     age_restricted,
     bug_reports_message,
     classproperty,
+    variadic,
     write_string,
 )
 
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 2ea36c63d..3f7dcb82b 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -475,8 +475,8 @@ class InfoExtractor:
 
 
     Subclasses of this should also be added to the list of extractors and
-    should define a _VALID_URL regexp and, re-define the _real_extract() and
-    (optionally) _real_initialize() methods.
+    should define _VALID_URL as a regexp or a Sequence of regexps, and
+    re-define the _real_extract() and (optionally) _real_initialize() methods.
 
     Subclasses may also override suitable() if necessary, but ensure the function
     signature is preserved and that this function imports everything it needs
@@ -566,8 +566,8 @@ def _match_valid_url(cls, url):
         # we have cached the regexp for *this* class, whereas getattr would also
         # match the superclass
         if '_VALID_URL_RE' not in cls.__dict__:
-            cls._VALID_URL_RE = re.compile(cls._VALID_URL)
-        return cls._VALID_URL_RE.match(url)
+            cls._VALID_URL_RE = tuple(map(re.compile, variadic(cls._VALID_URL)))
+        return next(filter(None, (regex.match(url) for regex in cls._VALID_URL_RE)), None)
 
     @classmethod
     def suitable(cls, url):

From f2ff0f6f1914b82d4a51681a72cc0828115dcb4a Mon Sep 17 00:00:00 2001
From: rexlambert22 <135362223+rexlambert22@users.noreply.github.com>
Date: Wed, 21 Jun 2023 20:00:54 -0400
Subject: [PATCH 393/871] [extractor/motherless] Add gallery support, fix
 groups (#7211)

Authored by: rexlambert22
---
 yt_dlp/extractor/_extractors.py |   3 +-
 yt_dlp/extractor/motherless.py  | 223 ++++++++++++++++++--------------
 2 files changed, 125 insertions(+), 101 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ff659a7a2..15344a6e5 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1119,7 +1119,8 @@
 from .morningstar import MorningstarIE
 from .motherless import (
     MotherlessIE,
-    MotherlessGroupIE
+    MotherlessGroupIE,
+    MotherlessGalleryIE,
 )
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
diff --git a/yt_dlp/extractor/motherless.py b/yt_dlp/extractor/motherless.py
index c24ef9b0d..769b52ce6 100644
--- a/yt_dlp/extractor/motherless.py
+++ b/yt_dlp/extractor/motherless.py
@@ -1,32 +1,39 @@
 import datetime
 import re
+import urllib.parse
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
-    InAdvancePagedList,
-    orderedSet,
+    OnDemandPagedList,
+    remove_end,
     str_to_int,
     unified_strdate,
 )
 
 
 class MotherlessIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/|G[VIG]?[A-F0-9]+/)?(?P<id>[A-F0-9]+)'
     _TESTS = [{
-        'url': 'http://motherless.com/AC3FFE1',
-        'md5': '310f62e325a9fafe64f68c0bccb6e75f',
+        'url': 'http://motherless.com/EE97006',
+        'md5': 'cb5e7438f7a3c4e886b7bccc1292a3bc',
         'info_dict': {
-            'id': 'AC3FFE1',
+            'id': 'EE97006',
             'ext': 'mp4',
-            'title': 'Fucked in the ass while playing PS3',
-            'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
-            'upload_date': '20100913',
-            'uploader_id': 'famouslyfuckedup',
+            'title': 'Dogging blond Brit getting glazed (comp)',
+            'categories': ['UK', 'slag', 'whore', 'dogging', 'cunt', 'cumhound', 'big tits', 'Pearl Necklace'],
+            'upload_date': '20230519',
+            'uploader_id': 'deathbird',
             'thumbnail': r're:https?://.*\.jpg',
             'age_limit': 18,
-        }
+            'comment_count': int,
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {
+            # Incomplete cert chains
+            'nocheckcertificate': True,
+        },
     }, {
         'url': 'http://motherless.com/532291B',
         'md5': 'bc59a6b47d1f958e61fbd38a4d31b131',
@@ -49,16 +56,36 @@ class MotherlessIE(InfoExtractor):
             'id': '633979F',
             'ext': 'mp4',
             'title': 'Turtlette',
-            'categories': ['superheroine heroine  superher'],
+            'categories': ['superheroine heroine superher'],
             'upload_date': '20140827',
             'uploader_id': 'shade0230',
             'thumbnail': r're:https?://.*\.jpg',
             'age_limit': 18,
-        }
+            'like_count': int,
+            'comment_count': int,
+            'view_count': int,
+        },
+        'params': {
+            'nocheckcertificate': True,
+        },
     }, {
-        # no keywords
         'url': 'http://motherless.com/8B4BBC1',
-        'only_matching': True,
+        'info_dict': {
+            'id': '8B4BBC1',
+            'ext': 'mp4',
+            'title': 'VIDEO00441.mp4',
+            'categories': [],
+            'upload_date': '20160214',
+            'uploader_id': 'NMWildGirl',
+            'thumbnail': r're:https?://.*\.jpg',
+            'age_limit': 18,
+            'like_count': int,
+            'comment_count': int,
+            'view_count': int,
+        },
+        'params': {
+            'nocheckcertificate': True,
+        },
     }, {
         # see https://motherless.com/videos/recent for recent videos with
         # uploaded date in "ago" format
@@ -72,9 +99,12 @@ class MotherlessIE(InfoExtractor):
             'uploader_id': 'anonymous',
             'thumbnail': r're:https?://.*\.jpg',
             'age_limit': 18,
+            'like_count': int,
+            'comment_count': int,
+            'view_count': int,
         },
         'params': {
-            'skip_download': True,
+            'nocheckcertificate': True,
         },
     }]
 
@@ -128,10 +158,8 @@ def _real_extract(self, url):
             (r'''<span\b[^>]+\bclass\s*=\s*["']username\b[^>]*>([^<]+)</span>''',
              r'''(?s)['"](?:media-meta-member|thumb-member-username)\b[^>]+>\s*<a\b[^>]+\bhref\s*=\s*['"]/m/([^"']+)'''),
             webpage, 'uploader_id', fatal=False)
-
-        categories = self._html_search_meta('keywords', webpage, default=None)
-        if categories:
-            categories = [cat.strip() for cat in categories.split(',')]
+        categories = self._html_search_meta('keywords', webpage, default='')
+        categories = [cat.strip() for cat in categories.split(',') if cat.strip()]
 
         return {
             'id': video_id,
@@ -148,102 +176,97 @@ def _real_extract(self, url):
         }
 
 
-class MotherlessGroupIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?motherless\.com/gv?/(?P<id>[a-z0-9_]+)'
+class MotherlessPaginatedIE(InfoExtractor):
+    _PAGE_SIZE = 60
+
+    def _correct_path(self, url, item_id):
+        raise NotImplementedError('This method must be implemented by subclasses')
+
+    def _extract_entries(self, webpage, base):
+        for mobj in re.finditer(r'href="[^"]*(?P<href>/[A-F0-9]+)"\s+title="(?P<title>[^"]+)',
+                                webpage):
+            video_url = urllib.parse.urljoin(base, mobj.group('href'))
+            video_id = MotherlessIE.get_temp_id(video_url)
+
+            if video_id:
+                yield self.url_result(video_url, MotherlessIE, video_id, mobj.group('title'))
+
+    def _real_extract(self, url):
+        item_id = self._match_id(url)
+        real_url = self._correct_path(url, item_id)
+        webpage = self._download_webpage(real_url, item_id, 'Downloading page 1')
+
+        def get_page(idx):
+            page = idx + 1
+            current_page = webpage if not idx else self._download_webpage(
+                real_url, item_id, note=f'Downloading page {page}', query={'page': page})
+            yield from self._extract_entries(current_page, real_url)
+
+        return self.playlist_result(
+            OnDemandPagedList(get_page, self._PAGE_SIZE), item_id,
+            remove_end(self._html_extract_title(webpage), ' | MOTHERLESS.COM ™'))
+
+
+class MotherlessGroupIE(MotherlessPaginatedIE):
+    _VALID_URL = r'https?://(?:www\.)?motherless\.com/g[vifm]?/(?P<id>[a-z0-9_]+)/?(?:$|[#?])'
     _TESTS = [{
-        'url': 'http://motherless.com/g/movie_scenes',
+        'url': 'http://motherless.com/gv/movie_scenes',
         'info_dict': {
             'id': 'movie_scenes',
             'title': 'Movie Scenes',
-            'description': 'Hot and sexy scenes from "regular" movies... '
-                           'Beautiful actresses fully nude... A looot of '
-                           'skin! :)Enjoy!',
         },
-        'playlist_mincount': 662,
+        'playlist_mincount': 540,
     }, {
-        'url': 'http://motherless.com/gv/sex_must_be_funny',
+        'url': 'http://motherless.com/g/sex_must_be_funny',
         'info_dict': {
             'id': 'sex_must_be_funny',
             'title': 'Sex must be funny',
-            'description': 'Sex can be funny. Wide smiles,laugh, games, fun of '
-                           'any kind!'
         },
-        'playlist_mincount': 0,
-        'expected_warnings': [
-            'This group has no videos.',
-        ]
+        'playlist_count': 0,
     }, {
-        'url': 'https://motherless.com/g/beautiful_cock',
+        'url': 'https://motherless.com/gv/beautiful_cock',
         'info_dict': {
             'id': 'beautiful_cock',
             'title': 'Beautiful Cock',
-            'description': 'Group for lovely cocks yours, mine, a friends anything human',
         },
-        'playlist_mincount': 2500,
+        'playlist_mincount': 2040,
     }]
 
-    @classmethod
-    def suitable(cls, url):
-        return (False if MotherlessIE.suitable(url)
-                else super(MotherlessGroupIE, cls).suitable(url))
+    def _correct_path(self, url, item_id):
+        return urllib.parse.urljoin(url, f'/gv/{item_id}')
 
-    def _extract_entries(self, webpage, base):
-        entries = []
-        for mobj in re.finditer(
-                r'href="(?P<href>/[^"]+)"[^>]*>(?:\s*<img[^>]+alt="[^-]+-\s(?P<title>[^"]+)")?',
-                webpage):
-            video_url = compat_urlparse.urljoin(base, mobj.group('href'))
-            if not MotherlessIE.suitable(video_url):
-                continue
-            video_id = MotherlessIE._match_id(video_url)
-            title = mobj.group('title')
-            entries.append(self.url_result(
-                video_url, ie=MotherlessIE.ie_key(), video_id=video_id,
-                video_title=title))
-        # Alternative fallback
-        if not entries:
-            entries = [
-                self.url_result(
-                    compat_urlparse.urljoin(base, '/' + entry_id),
-                    ie=MotherlessIE.ie_key(), video_id=entry_id)
-                for entry_id in orderedSet(re.findall(
-                    r'data-codename=["\']([A-Z0-9]+)', webpage))]
-        return entries
 
-    def _real_extract(self, url):
-        group_id = self._match_id(url)
-        page_url = compat_urlparse.urljoin(url, '/gv/%s' % group_id)
-        webpage = self._download_webpage(page_url, group_id)
-        title = self._search_regex(
-            r'<title>([\w\s]+\w)\s+-', webpage, 'title', fatal=False)
-        description = self._html_search_meta(
-            'description', webpage, fatal=False)
-        page_count = str_to_int(self._search_regex(
-            r'(\d+)\s*</(?:a|span)>\s*<(?:a|span)[^>]+(?:>\s*NEXT|\brel\s*=\s*["\']?next)\b',
-            webpage, 'page_count', default=0))
-        if not page_count:
-            message = self._search_regex(
-                r'''class\s*=\s*['"]error-page\b[^>]*>\s*<p[^>]*>\s*(?P<error_msg>[^<]+)(?<=\S)\s*''',
-                webpage, 'error_msg', default=None) or 'This group has no videos.'
-            self.report_warning(message, group_id)
-            page_count = 1
-        PAGE_SIZE = 80
+class MotherlessGalleryIE(MotherlessPaginatedIE):
+    _VALID_URL = r'https?://(?:www\.)?motherless\.com/G[VIG]?(?P<id>[A-F0-9]+)/?(?:$|[#?])'
+    _TESTS = [{
+        'url': 'https://motherless.com/GV338999F',
+        'info_dict': {
+            'id': '338999F',
+            'title': 'Random',
+        },
+        'playlist_mincount': 190,
+    }, {
+        'url': 'https://motherless.com/GVABD6213',
+        'info_dict': {
+            'id': 'ABD6213',
+            'title': 'Cuties',
+        },
+        'playlist_mincount': 2,
+    }, {
+        'url': 'https://motherless.com/GVBCF7622',
+        'info_dict': {
+            'id': 'BCF7622',
+            'title': 'Vintage',
+        },
+        'playlist_count': 0,
+    }, {
+        'url': 'https://motherless.com/G035DE2F',
+        'info_dict': {
+            'id': '035DE2F',
+            'title': 'General',
+        },
+        'playlist_mincount': 420,
+    }]
 
-        def _get_page(idx):
-            if idx > 0:
-                webpage = self._download_webpage(
-                    page_url, group_id, query={'page': idx + 1},
-                    note='Downloading page %d/%d' % (idx + 1, page_count)
-                )
-            for entry in self._extract_entries(webpage, url):
-                yield entry
-
-        playlist = InAdvancePagedList(_get_page, page_count, PAGE_SIZE)
-
-        return {
-            '_type': 'playlist',
-            'id': group_id,
-            'title': title,
-            'description': description,
-            'entries': playlist
-        }
+    def _correct_path(self, url, item_id):
+        return urllib.parse.urljoin(url, f'/GV{item_id}')

From 774aa09dd6aa61ced9ec818d1f67e53414d22762 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 22 Jun 2023 00:16:39 -0500
Subject: [PATCH 394/871] [extractor/dplay] GlobalCyclingNetworkPlus: Add
 extractor (#7360)

* Allows `country` API param to be configured with `--xff`/`geo_bypass_country`

Closes #7324
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/dplay.py       | 37 +++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 15344a6e5..a49a57a68 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -497,6 +497,7 @@
     DiscoveryPlusItalyIE,
     DiscoveryPlusItalyShowIE,
     DiscoveryPlusIndiaShowIE,
+    GlobalCyclingNetworkPlusIE,
 )
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
diff --git a/yt_dlp/extractor/dplay.py b/yt_dlp/extractor/dplay.py
index 8eb4d8ffa..cf6d14934 100644
--- a/yt_dlp/extractor/dplay.py
+++ b/yt_dlp/extractor/dplay.py
@@ -65,6 +65,7 @@ def _download_video_playback_info(self, disco_base, video_id, headers):
         return streaming_list
 
     def _get_disco_api_info(self, url, display_id, disco_host, realm, country, domain=''):
+        country = self.get_param('geo_bypass_country') or country
         geo_countries = [country.upper()]
         self._initialize_geo_bypass({
             'countries': geo_countries,
@@ -1001,3 +1002,39 @@ class DiscoveryPlusIndiaShowIE(DiscoveryPlusShowBaseIE):
     _SHOW_STR = 'show'
     _INDEX = 4
     _VIDEO_IE = DiscoveryPlusIndiaIE
+
+
+class GlobalCyclingNetworkPlusIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://plus\.globalcyclingnetwork\.com/watch/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://plus.globalcyclingnetwork.com/watch/1397691',
+        'info_dict': {
+            'id': '1397691',
+            'ext': 'mp4',
+            'title': 'The Athertons: Mountain Biking\'s Fastest Family',
+            'description': 'md5:75a81937fcd8b989eec6083a709cd837',
+            'thumbnail': 'https://us1-prod-images.disco-api.com/2021/03/04/eb9e3026-4849-3001-8281-9356466f0557.png',
+            'series': 'gcn',
+            'creator': 'Gcn',
+            'upload_date': '20210309',
+            'timestamp': 1615248000,
+            'duration': 2531.0,
+            'tags': [],
+        },
+        'skip': 'Subscription required',
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    _PRODUCT = 'web'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'disco-api-prod.globalcyclingnetwork.com',
+        'realm': 'gcn',
+        'country': 'us',
+    }
+
+    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
+        headers.update({
+            'x-disco-params': f'realm={realm}',
+            'x-disco-client': f'WEB:UNKNOWN:{self._PRODUCT}:27.3.2',
+            'Authorization': self._get_auth(disco_base, display_id, realm),
+        })

From 98cb1eda7a4cf67c96078980dbd63e6c06ad7f7c Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 22 Jun 2023 00:24:52 -0500
Subject: [PATCH 395/871] [extractor/rheinmaintv] Add extractor (#7311)

Authored by: barthelmannk

Co-authored-by: barthelmannk <81305638+barthelmannk@users.noreply.github.com>
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/rheinmaintv.py | 94 +++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)
 create mode 100644 yt_dlp/extractor/rheinmaintv.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index a49a57a68..49a3f39d3 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1617,6 +1617,7 @@
 from .restudy import RestudyIE
 from .reuters import ReutersIE
 from .reverbnation import ReverbNationIE
+from .rheinmaintv import RheinMainTVIE
 from .rice import RICEIE
 from .rmcdecouverte import RMCDecouverteIE
 from .rockstargames import RockstarGamesIE
diff --git a/yt_dlp/extractor/rheinmaintv.py b/yt_dlp/extractor/rheinmaintv.py
new file mode 100644
index 000000000..c3b352ded
--- /dev/null
+++ b/yt_dlp/extractor/rheinmaintv.py
@@ -0,0 +1,94 @@
+from .common import InfoExtractor
+from ..utils import extract_attributes, merge_dicts, remove_end
+
+
+class RheinMainTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?rheinmaintv\.de/sendungen/(?:[\w-]+/)*(?P<video_id>(?P<display_id>[\w-]+)/vom-\d{2}\.\d{2}\.\d{4}(?:/\d+)?)'
+    _TESTS = [{
+        'url': 'https://www.rheinmaintv.de/sendungen/beitrag-video/auf-dem-weg-zur-deutschen-meisterschaft/vom-07.11.2022/',
+        'info_dict': {
+            'id': 'auf-dem-weg-zur-deutschen-meisterschaft-vom-07.11.2022',
+            'ext': 'ismv',  # ismv+isma will be merged into mp4
+            'alt_title': 'Auf dem Weg zur Deutschen Meisterschaft',
+            'title': 'Auf dem Weg zur Deutschen Meisterschaft',
+            'upload_date': '20221108',
+            'view_count': int,
+            'display_id': 'auf-dem-weg-zur-deutschen-meisterschaft',
+            'thumbnail': r're:^https://.+\.jpg',
+            'description': 'md5:48c59b74192bc819a9b34af1d5ed1eb9',
+            'timestamp': 1667933057,
+            'duration': 243.0,
+        },
+        'params': {'skip_download': 'ism'},
+    }, {
+        'url': 'https://www.rheinmaintv.de/sendungen/beitrag-video/formationsgemeinschaft-rhein-main-bei-den-deutschen-meisterschaften/vom-14.11.2022/',
+        'info_dict': {
+            'id': 'formationsgemeinschaft-rhein-main-bei-den-deutschen-meisterschaften-vom-14.11.2022',
+            'ext': 'ismv',
+            'title': 'Formationsgemeinschaft Rhein-Main bei den Deutschen Meisterschaften',
+            'timestamp': 1668526214,
+            'display_id': 'formationsgemeinschaft-rhein-main-bei-den-deutschen-meisterschaften',
+            'alt_title': 'Formationsgemeinschaft Rhein-Main bei den Deutschen Meisterschaften',
+            'view_count': int,
+            'thumbnail': r're:^https://.+\.jpg',
+            'duration': 345.0,
+            'description': 'md5:9370ba29526984006c2cba1372e5c5a0',
+            'upload_date': '20221115',
+        },
+        'params': {'skip_download': 'ism'},
+    }, {
+        'url': 'https://www.rheinmaintv.de/sendungen/beitrag-video/casino-mainz-bei-den-deutschen-meisterschaften/vom-14.11.2022/',
+        'info_dict': {
+            'id': 'casino-mainz-bei-den-deutschen-meisterschaften-vom-14.11.2022',
+            'ext': 'ismv',
+            'title': 'Casino Mainz bei den Deutschen Meisterschaften',
+            'view_count': int,
+            'timestamp': 1668527402,
+            'alt_title': 'Casino Mainz bei den Deutschen Meisterschaften',
+            'upload_date': '20221115',
+            'display_id': 'casino-mainz-bei-den-deutschen-meisterschaften',
+            'duration': 348.0,
+            'thumbnail': r're:^https://.+\.jpg',
+            'description': 'md5:70fc1660eeba96da17199e5bdff4c0aa',
+        },
+        'params': {'skip_download': 'ism'},
+    }, {
+        'url': 'https://www.rheinmaintv.de/sendungen/beitrag-video/bricks4kids/vom-22.06.2022/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = self._match_valid_url(url)
+        display_id = mobj.group('display_id')
+        video_id = mobj.group('video_id').replace('/', '-')
+        webpage = self._download_webpage(url, video_id)
+
+        source, img = self._search_regex(r'(?s)(?P<source><source[^>]*>)(?P<img><img[^>]*>)',
+                                         webpage, 'video', group=('source', 'img'))
+        source = extract_attributes(source)
+        img = extract_attributes(img)
+
+        raw_json_ld = list(self._yield_json_ld(webpage, video_id))
+        json_ld = self._json_ld(raw_json_ld, video_id)
+        json_ld.pop('url', None)
+
+        ism_manifest_url = (
+            source.get('src')
+            or next(json_ld.get('embedUrl') for json_ld in raw_json_ld if json_ld.get('@type') == 'VideoObject')
+        )
+        formats, subtitles = self._extract_ism_formats_and_subtitles(ism_manifest_url, video_id)
+
+        return merge_dicts({
+            'id': video_id,
+            'display_id': display_id,
+            'title':
+                self._html_search_regex(r'<h1><span class="title">([^<]*)</span>',
+                                        webpage, 'headline', default=None)
+                or img.get('title') or json_ld.get('title') or self._og_search_title(webpage)
+                or remove_end(self._html_extract_title(webpage), ' -'),
+            'alt_title': img.get('alt'),
+            'description': json_ld.get('description') or self._og_search_description(webpage),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnails': [{'url': img['src']}] if 'src' in img else json_ld.get('thumbnails'),
+        }, json_ld)

From 71dc18fa29263a1ff0472c23d81bfc8dd4422d48 Mon Sep 17 00:00:00 2001
From: Berkan Teber <github@accounts.berkanteber.com>
Date: Thu, 22 Jun 2023 10:27:54 +0300
Subject: [PATCH 396/871] [extractor/youtube] Improve description parsing
 performance (#7315)

* The parsing is skipped when not needed
* The regex is improved by simulating atomic groups with lookaheads

Authored by: pukkandan, berkanteber
---
 yt_dlp/extractor/youtube.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index a90118680..ef9f1f11c 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4346,15 +4346,21 @@ def process_language(container, base_url, lang_code, sub_name, query):
                         info[d_k] = parse_duration(query[k][0])
 
         # Youtube Music Auto-generated description
-        if video_description:
+        if (video_description or '').strip().endswith('\nAuto-generated by YouTube.'):
+            # XXX: Causes catastrophic backtracking if description has "·"
+            # E.g. https://www.youtube.com/watch?v=DoPaAxMQoiI
+            # Simulating atomic groups:  (?P<a>[^xy]+)x  =>  (?=(?P<a>[^xy]+))(?P=a)x
+            # reduces it, but does not fully fix it. https://regex101.com/r/8Ssf2h/2
             mobj = re.search(
                 r'''(?xs)
-                    (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
-                    (?P<album>[^\n]+)
+                    (?=(?P<track>[^\n·]+))(?P=track)·
+                    (?=(?P<artist>[^\n]+))(?P=artist)\n+
+                    (?=(?P<album>[^\n]+))(?P=album)\n
                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
-                    (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
-                    .+\nAuto-generated\ by\ YouTube\.\s*$
+                    (.+?\nArtist\s*:\s*
+                        (?=(?P<clean_artist>[^\n]+))(?P=clean_artist)\n
+                    )?.+\nAuto-generated\ by\ YouTube\.\s*$
                 ''', video_description)
             if mobj:
                 release_year = mobj.group('release_year')

From b4e0d75848e9447cee2cd3646ce54d4744a7ff56 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 22 Jun 2023 04:54:39 +0530
Subject: [PATCH 397/871] Improve `--download-sections`

* Support negative time-ranges
* Add `*from-url` to obey time-ranges in URL

Closes #7248
---
 README.md              | 14 +++++-----
 yt_dlp/YoutubeDL.py    |  6 +++--
 yt_dlp/__init__.py     | 59 +++++++++++++++++++++++++++++-------------
 yt_dlp/options.py      |  5 ++--
 yt_dlp/utils/_utils.py | 22 +++++++++++++---
 5 files changed, 74 insertions(+), 32 deletions(-)

diff --git a/README.md b/README.md
index d10832103..8db2d4f06 100644
--- a/README.md
+++ b/README.md
@@ -610,12 +610,14 @@ ## Download Options:
     --no-hls-use-mpegts             Do not use the mpegts container for HLS
                                     videos. This is default when not downloading
                                     live streams
-    --download-sections REGEX       Download only chapters whose title matches
-                                    the given regular expression. Time ranges
-                                    prefixed by a "*" can also be used in place
-                                    of chapters to download the specified range.
-                                    Needs ffmpeg. This option can be used
-                                    multiple times to download multiple
+    --download-sections REGEX       Download only chapters that match the
+                                    regular expression. A "*" prefix denotes
+                                    time-range instead of chapter. Negative
+                                    timestamps are calculated from the end.
+                                    "*from-url" can be used to download between
+                                    the "start_time" and "end_time" extracted
+                                    from the URL. Needs ffmpeg. This option can
+                                    be used multiple times to download multiple
                                     sections, e.g. --download-sections
                                     "*10:15-inf" --download-sections "intro"
     --downloader [PROTO:]NAME       Name or path of the external downloader to
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 79b7d47b0..6dade0b2a 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2806,11 +2806,13 @@ def to_screen(*msg):
                 new_info.update(fmt)
                 offset, duration = info_dict.get('section_start') or 0, info_dict.get('duration') or float('inf')
                 end_time = offset + min(chapter.get('end_time', duration), duration)
+                # duration may not be accurate. So allow deviations <1sec
+                if end_time == float('inf') or end_time > offset + duration + 1:
+                    end_time = None
                 if chapter or offset:
                     new_info.update({
                         'section_start': offset + chapter.get('start_time', 0),
-                        # duration may not be accurate. So allow deviations <1sec
-                        'section_end': end_time if end_time <= offset + duration + 1 else None,
+                        'section_end': end_time,
                         'section_title': chapter.get('title'),
                         'section_number': chapter.get('index'),
                     })
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 46edd88d3..b81277a57 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -320,26 +320,49 @@ def validate_outtmpl(tmpl, msg):
         opts.skip_download = None
         del opts.outtmpl['default']
 
-    def parse_chapters(name, value):
-        chapters, ranges = [], []
+    def parse_chapters(name, value, advanced=False):
         parse_timestamp = lambda x: float('inf') if x in ('inf', 'infinite') else parse_duration(x)
-        for regex in value or []:
-            if regex.startswith('*'):
-                for range_ in map(str.strip, regex[1:].split(',')):
-                    mobj = range_ != '-' and re.fullmatch(r'([^-]+)?\s*-\s*([^-]+)?', range_)
-                    dur = mobj and (parse_timestamp(mobj.group(1) or '0'), parse_timestamp(mobj.group(2) or 'inf'))
-                    if None in (dur or [None]):
-                        raise ValueError(f'invalid {name} time range "{regex}". Must be of the form "*start-end"')
-                    ranges.append(dur)
-                continue
-            try:
-                chapters.append(re.compile(regex))
-            except re.error as err:
-                raise ValueError(f'invalid {name} regex "{regex}" - {err}')
-        return chapters, ranges
+        TIMESTAMP_RE = r'''(?x)(?:
+            (?P<start_sign>-?)(?P<start>[^-]+)
+        )?\s*-\s*(?:
+            (?P<end_sign>-?)(?P<end>[^-]+)
+        )?'''
 
-    opts.remove_chapters, opts.remove_ranges = parse_chapters('--remove-chapters', opts.remove_chapters)
-    opts.download_ranges = download_range_func(*parse_chapters('--download-sections', opts.download_ranges))
+        chapters, ranges, from_url = [], [], False
+        for regex in value or []:
+            if advanced and regex == '*from-url':
+                from_url = True
+                continue
+            elif not regex.startswith('*'):
+                try:
+                    chapters.append(re.compile(regex))
+                except re.error as err:
+                    raise ValueError(f'invalid {name} regex "{regex}" - {err}')
+                continue
+
+            for range_ in map(str.strip, regex[1:].split(',')):
+                mobj = range_ != '-' and re.fullmatch(TIMESTAMP_RE, range_)
+                dur = mobj and [parse_timestamp(mobj.group('start') or '0'), parse_timestamp(mobj.group('end') or 'inf')]
+                signs = mobj and (mobj.group('start_sign'), mobj.group('end_sign'))
+
+                err = None
+                if None in (dur or [None]):
+                    err = 'Must be of the form "*start-end"'
+                elif not advanced and any(signs):
+                    err = 'Negative timestamps are not allowed'
+                else:
+                    dur[0] *= -1 if signs[0] else 1
+                    dur[1] *= -1 if signs[1] else 1
+                    if dur[1] == float('-inf'):
+                        err = '"-inf" is not a valid end'
+                if err:
+                    raise ValueError(f'invalid {name} time range "{regex}". {err}')
+                ranges.append(dur)
+
+        return chapters, ranges, from_url
+
+    opts.remove_chapters, opts.remove_ranges, _ = parse_chapters('--remove-chapters', opts.remove_chapters)
+    opts.download_ranges = download_range_func(*parse_chapters('--download-sections', opts.download_ranges, True))
 
     # Cookies from browser
     if opts.cookiesfrombrowser:
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 9d6dbec9f..163809706 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -1012,8 +1012,9 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         '--download-sections',
         metavar='REGEX', dest='download_ranges', action='append',
         help=(
-            'Download only chapters whose title matches the given regular expression. '
-            'Time ranges prefixed by a "*" can also be used in place of chapters to download the specified range. '
+            'Download only chapters that match the regular expression. '
+            'A "*" prefix denotes time-range instead of chapter. Negative timestamps are calculated from the end. '
+            '"*from-url" can be used to download between the "start_time" and "end_time" extracted from the URL. '
             'Needs ffmpeg. This option can be used multiple times to download multiple sections, '
             'e.g. --download-sections "*10:15-inf" --download-sections "intro"'))
     downloader.add_option(
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index bc1bc9116..56acadd73 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -3753,11 +3753,11 @@ def _match_func(info_dict, incomplete=False):
 
 
 class download_range_func:
-    def __init__(self, chapters, ranges):
-        self.chapters, self.ranges = chapters, ranges
+    def __init__(self, chapters, ranges, from_info=False):
+        self.chapters, self.ranges, self.from_info = chapters, ranges, from_info
 
     def __call__(self, info_dict, ydl):
-        if not self.ranges and not self.chapters:
+        if not any((self.ranges, self.chapters, self.from_info)):
             yield {}
 
         warning = ('There are no chapters matching the regex' if info_dict.get('chapters')
@@ -3770,7 +3770,21 @@ def __call__(self, info_dict, ydl):
         if self.chapters and warning:
             ydl.to_screen(f'[info] {info_dict["id"]}: {warning}')
 
-        yield from ({'start_time': start, 'end_time': end} for start, end in self.ranges or [])
+        for start, end in self.ranges or []:
+            yield {
+                'start_time': self._handle_negative_timestamp(start, info_dict),
+                'end_time': self._handle_negative_timestamp(end, info_dict),
+            }
+
+        if self.from_info and (info_dict.get('start_time') or info_dict.get('end_time')):
+            yield {
+                'start_time': info_dict.get('start_time'),
+                'end_time': info_dict.get('end_time'),
+            }
+
+    @staticmethod
+    def _handle_negative_timestamp(time, info):
+        return max(info['duration'] + time, 0) if info.get('duration') and time < 0 else time
 
     def __eq__(self, other):
         return (isinstance(other, download_range_func)

From cd810afe2ac5567c822b7424800fc470ef2d0045 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 22 Jun 2023 13:23:31 +0530
Subject: [PATCH 398/871] [extractor/youtube] Improve nsig function name
 extraction

---
 test/test_youtube_signature.py | 4 ++++
 yt_dlp/extractor/youtube.py    | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 811f70e68..c5592845b 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -159,6 +159,10 @@
         'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js',
         '1wWCVpRR96eAmMI87L', 'KSkWAVv1ZQxC3A',
     ),
+    (
+        'https://www.youtube.com/s/player/b7910ca8/player_ias.vflset/en_US/base.js',
+        '_hXMCwMt9qE310D', 'LoZMgkkofRMCZQ',
+    ),
 ]
 
 
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index ef9f1f11c..f1c8df141 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3140,7 +3140,7 @@ def _extract_n_function_name(self, jscode):
             return funcname
 
         return json.loads(js_to_json(self._search_regex(
-            rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
+            rf'var {re.escape(funcname)}\s*=\s*(\[.+?\])[,;]', jscode,
             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
 
     def _extract_n_function_code(self, video_id, player_url):

From 812cdfa06c33a40e73a8e04b3e6f42c084666a43 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 22 Jun 2023 10:02:38 +0530
Subject: [PATCH 399/871] [cleanup] Misc

---
 README.md                          |  2 +-
 devscripts/changelog_override.json | 21 +++++++++++++++++++++
 devscripts/make_changelog.py       | 14 ++++++++------
 yt_dlp/extractor/testurl.py        |  8 ++++++--
 yt_dlp/utils/_utils.py             |  7 ++++---
 5 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 8db2d4f06..4de4ece96 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,7 @@
 
 # NEW FEATURES
 
-* Merged with **youtube-dl v2021.12.17+ [commit/2dd6c6e](https://github.com/ytdl-org/youtube-dl/commit/2dd6c6e)** ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21)) and **youtube-dlc v2020.11.11-3+ [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
+* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@42f2d4**](https://github.com/yt-dlp/yt-dlp/commit/42f2d4) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index 73225bdb9..df80f45e0 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -35,5 +35,26 @@
         "when": "8417f26b8a819cd7ffcd4e000ca3e45033e670fb",
         "short": "Add option `--color` (#6904)",
         "authors": ["Grub4K"]
+    },
+    {
+        "action": "change",
+        "when": "7b37e8b23691613f331bd4ebc9d639dd6f93c972",
+        "short": "Improve `--download-sections`\n    - Support negative time-ranges\n    - Add `*from-url` to obey time-ranges in URL"
+    },
+    {
+        "action": "change",
+        "when": "1e75d97db21152acc764b30a688e516f04b8a142",
+        "short": "[extractor/youtube] Add `ios` to default clients used\n        - IOS is affected neither by 403 nor by nsig so helps mitigate them preemptively\n        - IOS also has higher bit-rate 'premium' formats though they are not labeled as such"
+    },
+    {
+        "action": "change",
+        "when": "f2ff0f6f1914b82d4a51681a72cc0828115dcb4a",
+        "short": "[extractor/motherless] Add gallery support, fix groups (#7211)",
+        "authors": ["rexlambert22", "Ti4eeT4e"]
+    },
+    {
+        "action": "change",
+        "when": "a4486bfc1dc7057efca9dd3fe70d7fa25c56f700",
+        "short": "[misc] Revert \"Add automatic duplicate issue detection\""
     }
 ]
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index 2fcdc06d7..0bcfa6ae7 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -196,7 +196,7 @@ def _prepare_cleanup_misc_items(self, items):
         for commit_infos in cleanup_misc_items.values():
             sorted_items.append(CommitInfo(
                 'cleanup', ('Miscellaneous',), ', '.join(
-                    self._format_message_link(None, info.commit.hash)
+                    self._format_message_link(None, info.commit.hash).strip()
                     for info in sorted(commit_infos, key=lambda item: item.commit.hash or '')),
                 [], Commit(None, '', commit_infos[0].commit.authors), []))
 
@@ -205,10 +205,10 @@ def _prepare_cleanup_misc_items(self, items):
     def format_single_change(self, info):
         message = self._format_message_link(info.message, info.commit.hash)
         if info.issues:
-            message = f'{message} ({self._format_issues(info.issues)})'
+            message = message.replace('\n', f' ({self._format_issues(info.issues)})\n', 1)
 
         if info.commit.authors:
-            message = f'{message} by {self._format_authors(info.commit.authors)}'
+            message = message.replace('\n', f' by {self._format_authors(info.commit.authors)}\n', 1)
 
         if info.fixes:
             fix_message = ', '.join(f'{self._format_message_link(None, fix.hash)}' for fix in info.fixes)
@@ -217,14 +217,16 @@ def format_single_change(self, info):
             if authors != info.commit.authors:
                 fix_message = f'{fix_message} by {self._format_authors(authors)}'
 
-            message = f'{message} (With fixes in {fix_message})'
+            message = message.replace('\n', f' (With fixes in {fix_message})\n', 1)
 
-        return message
+        return message[:-1]
 
     def _format_message_link(self, message, hash):
         assert message or hash, 'Improperly defined commit message or override'
         message = message if message else hash[:HASH_LENGTH]
-        return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message
+        if not hash:
+            return f'{message}\n'
+        return f'[{message}\n'.replace('\n', f']({self.repo_url}/commit/{hash})\n', 1)
 
     def _format_issues(self, issues):
         return ', '.join(f'[#{issue}]({self.repo_url}/issues/{issue})' for issue in issues)
diff --git a/yt_dlp/extractor/testurl.py b/yt_dlp/extractor/testurl.py
index 0da01aa53..3cf001776 100644
--- a/yt_dlp/extractor/testurl.py
+++ b/yt_dlp/extractor/testurl.py
@@ -8,7 +8,7 @@ class TestURLIE(InfoExtractor):
     """ Allows addressing of the test cases as test:yout.*be_1 """
 
     IE_DESC = False  # Do not list
-    _VALID_URL = r'test(?:url)?:(?P<extractor>.*?)(?:_(?P<num>[0-9]+))?$'
+    _VALID_URL = r'test(?:url)?:(?P<extractor>.*?)(?:_(?P<num>\d+|all))?$'
 
     def _real_extract(self, url):
         from . import gen_extractor_classes
@@ -36,6 +36,10 @@ def _real_extract(self, url):
             extractor = matching_extractors[0]
 
         testcases = tuple(extractor.get_testcases(True))
+        if num == 'all':
+            return self.playlist_result(
+                [self.url_result(tc['url'], extractor) for tc in testcases],
+                url, f'{extractor.IE_NAME} tests')
         try:
             tc = testcases[int(num or 0)]
         except IndexError:
@@ -43,4 +47,4 @@ def _real_extract(self, url):
                 f'Test case {num or 0} not found, got only {len(testcases)} tests', expected=True)
 
         self.to_screen(f'Test URL: {tc["url"]}')
-        return self.url_result(tc['url'])
+        return self.url_result(tc['url'], extractor)
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 56acadd73..10052009f 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -3507,7 +3507,8 @@ def get_compatible_ext(*, vcodecs, acodecs, vexts, aexts, preferences=None):
         },
     }
 
-    sanitize_codec = functools.partial(try_get, getter=lambda x: x[0].split('.')[0].replace('0', ''))
+    sanitize_codec = functools.partial(
+        try_get, getter=lambda x: x[0].split('.')[0].replace('0', '').lower())
     vcodec, acodec = sanitize_codec(vcodecs), sanitize_codec(acodecs)
 
     for ext in preferences or COMPATIBLE_CODECS.keys():
@@ -5737,9 +5738,9 @@ class FormatSorter:
         'source': {'convert': 'float', 'field': 'source_preference', 'default': -1},
 
         'codec': {'type': 'combined', 'field': ('vcodec', 'acodec')},
-        'br': {'type': 'multiple', 'field': ('tbr', 'vbr', 'abr'),
+        'br': {'type': 'multiple', 'field': ('tbr', 'vbr', 'abr'), 'convert': 'float_none',
                'function': lambda it: next(filter(None, it), None)},
-        'size': {'type': 'multiple', 'field': ('filesize', 'fs_approx'),
+        'size': {'type': 'multiple', 'field': ('filesize', 'fs_approx'), 'convert': 'bytes',
                  'function': lambda it: next(filter(None, it), None)},
         'ext': {'type': 'combined', 'field': ('vext', 'aext')},
         'res': {'type': 'multiple', 'field': ('height', 'width'),

From de4cf77ec1a13f020e6afe4ed04248c6b19fccb6 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@example.com>
Date: Thu, 22 Jun 2023 08:09:31 +0000
Subject: [PATCH 400/871] Release 2023.06.22

Created by: pukkandan

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |  8 ++---
 .../ISSUE_TEMPLATE/2_site_support_request.yml |  8 ++---
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |  8 ++---
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |  8 ++---
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |  8 ++---
 .github/ISSUE_TEMPLATE/6_question.yml         |  8 ++---
 CONTRIBUTORS                                  |  5 ++++
 Changelog.md                                  | 29 +++++++++++++++++++
 supportedsites.md                             |  3 ++
 yt_dlp/version.py                             |  4 +--
 10 files changed, 63 insertions(+), 26 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index 351454b12..a00a11f27 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -64,7 +64,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -72,8 +72,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.21, Current version: 2023.06.21
-        yt-dlp is up to date (2023.06.21)
+        Latest version: 2023.06.22, Current version: 2023.06.22
+        yt-dlp is up to date (2023.06.22)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index b2a613e2f..fc1f41ead 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -76,7 +76,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -84,8 +84,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.21, Current version: 2023.06.21
-        yt-dlp is up to date (2023.06.21)
+        Latest version: 2023.06.22, Current version: 2023.06.22
+        yt-dlp is up to date (2023.06.22)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index c100561eb..ed51dfa97 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -72,7 +72,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -80,8 +80,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.21, Current version: 2023.06.21
-        yt-dlp is up to date (2023.06.21)
+        Latest version: 2023.06.22, Current version: 2023.06.22
+        yt-dlp is up to date (2023.06.22)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index e97d7b507..1c997f3e2 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -57,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -65,8 +65,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.21, Current version: 2023.06.21
-        yt-dlp is up to date (2023.06.21)
+        Latest version: 2023.06.22, Current version: 2023.06.22
+        yt-dlp is up to date (2023.06.22)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index a44612d79..1638945bf 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -53,7 +53,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -61,7 +61,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.21, Current version: 2023.06.21
-        yt-dlp is up to date (2023.06.21)
+        Latest version: 2023.06.22, Current version: 2023.06.22
+        yt-dlp is up to date (2023.06.22)
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index a15a46968..d27bd5742 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.21** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -59,7 +59,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.21 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -67,7 +67,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.21, Current version: 2023.06.21
-        yt-dlp is up to date (2023.06.21)
+        Latest version: 2023.06.22, Current version: 2023.06.22
+        yt-dlp is up to date (2023.06.22)
         <more lines>
       render: shell
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 3b35895d9..0864f16c4 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -455,3 +455,8 @@ vampirefrog
 vidiot720
 viktor-enzell
 zhgwn
+barthelmannk
+berkanteber
+OverlordQ
+rexlambert22
+Ti4eeT4e
diff --git a/Changelog.md b/Changelog.md
index d7a1cb495..c340b74c9 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,35 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.06.22
+
+#### Core changes
+- [Fix bug in db3ad8a67661d7b234a6954d9c6a4a9b1749f5eb](https://github.com/yt-dlp/yt-dlp/commit/d7cd97e8d8d42b500fea9abb2aa4ac9b0f98b2ad) by [pukkandan](https://github.com/pukkandan)
+- [Improve `--download-sections`](https://github.com/yt-dlp/yt-dlp/commit/b4e0d75848e9447cee2cd3646ce54d4744a7ff56) by [pukkandan](https://github.com/pukkandan)
+- [Indicate `filesize` approximated from `tbr` better](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) by [pukkandan](https://github.com/pukkandan)
+
+#### Extractor changes
+- [Support multiple `_VALID_URL`s](https://github.com/yt-dlp/yt-dlp/commit/5fd8367496b42c7b900b896a0d5460561a2859de) ([#5812](https://github.com/yt-dlp/yt-dlp/issues/5812)) by [nixxo](https://github.com/nixxo)
+- **dplay**: GlobalCyclingNetworkPlus: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/774aa09dd6aa61ced9ec818d1f67e53414d22762) ([#7360](https://github.com/yt-dlp/yt-dlp/issues/7360)) by [bashonly](https://github.com/bashonly)
+- **dropout**: [Fix season extraction](https://github.com/yt-dlp/yt-dlp/commit/db22142f6f817ff673d417b4b78e8db497bf8ab3) ([#7304](https://github.com/yt-dlp/yt-dlp/issues/7304)) by [OverlordQ](https://github.com/OverlordQ)
+- **motherless**: [Add gallery support, fix groups](https://github.com/yt-dlp/yt-dlp/commit/f2ff0f6f1914b82d4a51681a72cc0828115dcb4a) ([#7211](https://github.com/yt-dlp/yt-dlp/issues/7211)) by [rexlambert22](https://github.com/rexlambert22), [Ti4eeT4e](https://github.com/Ti4eeT4e)
+- **nebula**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/3f756c8c4095b942cf49788eb0862ceaf57847f2) ([#7156](https://github.com/yt-dlp/yt-dlp/issues/7156)) by [Lamieur](https://github.com/Lamieur), [rohieb](https://github.com/rohieb)
+- **rheinmaintv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/98cb1eda7a4cf67c96078980dbd63e6c06ad7f7c) ([#7311](https://github.com/yt-dlp/yt-dlp/issues/7311)) by [barthelmannk](https://github.com/barthelmannk)
+- **youtube**
+    - [Add `ios` to default clients used](https://github.com/yt-dlp/yt-dlp/commit/1e75d97db21152acc764b30a688e516f04b8a142)
+        - IOS is affected neither by 403 nor by nsig so helps mitigate them preemptively
+        - IOS also has higher bit-rate 'premium' formats though they are not labeled as such
+    - [Improve description parsing performance](https://github.com/yt-dlp/yt-dlp/commit/71dc18fa29263a1ff0472c23d81bfc8dd4422d48) ([#7315](https://github.com/yt-dlp/yt-dlp/issues/7315)) by [berkanteber](https://github.com/berkanteber), [pukkandan](https://github.com/pukkandan)
+    - [Improve nsig function name extraction](https://github.com/yt-dlp/yt-dlp/commit/cd810afe2ac5567c822b7424800fc470ef2d0045) by [pukkandan](https://github.com/pukkandan)
+    - [Workaround 403 for android formats](https://github.com/yt-dlp/yt-dlp/commit/81ca451480051d7ce1a31c017e005358345a9149) by [pukkandan](https://github.com/pukkandan)
+
+#### Misc. changes
+- [Revert "Add automatic duplicate issue detection"](https://github.com/yt-dlp/yt-dlp/commit/a4486bfc1dc7057efca9dd3fe70d7fa25c56f700)
+- **cleanup**
+    - Miscellaneous
+        - [7f9c6a6](https://github.com/yt-dlp/yt-dlp/commit/7f9c6a63b16e145495479e9f666f5b9e2ee69e2f) by [bashonly](https://github.com/bashonly)
+        - [812cdfa](https://github.com/yt-dlp/yt-dlp/commit/812cdfa06c33a40e73a8e04b3e6f42c084666a43) by [pukkandan](https://github.com/pukkandan)
+
 ### 2023.06.21
 
 #### Important changes
diff --git a/supportedsites.md b/supportedsites.md
index 882b272aa..7d99d9e22 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -515,6 +515,7 @@ # Supported sites
  - **GlattvisionTVLive**: [*glattvisiontv*](## "netrc machine")
  - **GlattvisionTVRecordings**: [*glattvisiontv*](## "netrc machine")
  - **Glide**: Glide mobile video messages (glide.me)
+ - **GlobalCyclingNetworkPlus**
  - **GlobalPlayerAudio**
  - **GlobalPlayerAudioEpisode**
  - **GlobalPlayerLive**
@@ -814,6 +815,7 @@ # Supported sites
  - **MonsterSirenHypergryphMusic**
  - **Morningstar**: morningstar.com
  - **Motherless**
+ - **MotherlessGallery**
  - **MotherlessGroup**
  - **Motorsport**: motorsport.com
  - **MotorTrend**
@@ -1198,6 +1200,7 @@ # Supported sites
  - **Restudy**
  - **Reuters**
  - **ReverbNation**
+ - **RheinMainTV**
  - **RICE**
  - **RMCDecouverte**
  - **RockstarGames**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index f4474db9a..434f36f48 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.06.21'
+__version__ = '2023.06.22'
 
-RELEASE_GIT_HEAD = '42f2d40b475db66486a4b4fe5b56751a640db5db'
+RELEASE_GIT_HEAD = '812cdfa06c33a40e73a8e04b3e6f42c084666a43'
 
 VARIANT = None
 

From 89bed013741a776506f60380b7fd89d27d0710b4 Mon Sep 17 00:00:00 2001
From: Simon <35427372+bbilly1@users.noreply.github.com>
Date: Fri, 23 Jun 2023 01:08:42 +0700
Subject: [PATCH 401/871] [extractor/youtube] Fix comments' `is_favorited`
 (#7390)

Authored by: bbilly1
Closes #7389
---
 yt_dlp/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index f1c8df141..a0d0a601a 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3356,7 +3356,7 @@ def _extract_comment(self, comment_renderer, parent=None):
             info['author_is_uploader'] = author_is_uploader
 
         comment_abr = traverse_obj(
-            comment_renderer, ('actionsButtons', 'commentActionButtonsRenderer'), expected_type=dict)
+            comment_renderer, ('actionButtons', 'commentActionButtonsRenderer'), expected_type=dict)
         if comment_abr is not None:
             info['is_favorited'] = 'creatorHeart' in comment_abr
 

From e59e20744eb32ce4b6ea0dece7c673be8376a710 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 22 Jun 2023 23:22:14 +0530
Subject: [PATCH 402/871] Bugfix for b4e0d75848e9447cee2cd3646ce54d4744a7ff56

---
 yt_dlp/utils/_utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 10052009f..de51f6208 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -3758,8 +3758,6 @@ def __init__(self, chapters, ranges, from_info=False):
         self.chapters, self.ranges, self.from_info = chapters, ranges, from_info
 
     def __call__(self, info_dict, ydl):
-        if not any((self.ranges, self.chapters, self.from_info)):
-            yield {}
 
         warning = ('There are no chapters matching the regex' if info_dict.get('chapters')
                    else 'Cannot match chapters since chapter information is unavailable')
@@ -3779,9 +3777,11 @@ def __call__(self, info_dict, ydl):
 
         if self.from_info and (info_dict.get('start_time') or info_dict.get('end_time')):
             yield {
-                'start_time': info_dict.get('start_time'),
-                'end_time': info_dict.get('end_time'),
+                'start_time': info_dict.get('start_time') or 0,
+                'end_time': info_dict.get('end_time') or float('inf'),
             }
+        elif not self.ranges and not self.chapters:
+            yield {}
 
     @staticmethod
     def _handle_negative_timestamp(time, info):

From 58786a10f212bd63f9ad1d0b4d9e4d31c3b385e2 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 25 Jun 2023 20:10:00 +0530
Subject: [PATCH 403/871] [extractor/youtube] Add extractor-arg `formats`

Closes #7417
---
 README.md                   |  3 +--
 yt_dlp/extractor/youtube.py | 22 ++++++++++++++++------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 4de4ece96..d89bb204e 100644
--- a/README.md
+++ b/README.md
@@ -1805,8 +1805,7 @@ #### youtube
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
 * `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all`
     * E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total
-* `include_duplicate_formats`: Extract formats with identical content but different URLs or protocol. This is useful if some of the formats are unavailable or throttled.
-* `include_incomplete_formats`: Extract formats that cannot be downloaded completely (live dash and post-live m3u8)
+* `formats`: Change the types of formats to return. `dashy` (convert http to DASH), `duplicate` (identical content but different URLs or protocol; includes `dashy`), `incomplete` (cannot be downloaded completely - live dash and post-live m3u8)
 * `innertube_host`: Innertube API host to use for all API requests; e.g. `studio.youtube.com`, `youtubei.googleapis.com`. Note that cookies exported from one subdomain will not work on others
 * `innertube_key`: Innertube API key to use for all API requests
 
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index a0d0a601a..bdc631ccb 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3752,7 +3752,12 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l
             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
         ])
         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
-        all_formats = self._configuration_arg('include_duplicate_formats')
+        format_types = self._configuration_arg('formats')
+        all_formats = 'duplicate' in format_types
+        if self._configuration_arg('include_duplicate_formats'):
+            all_formats = True
+            self._downloader.deprecated_feature('[youtube] include_duplicate_formats extractor argument is deprecated. '
+                                                'Use formats=duplicate extractor argument instead')
 
         def build_fragments(f):
             return LazyList({
@@ -3892,18 +3897,23 @@ def build_fragments(f):
             if single_stream and dct.get('ext'):
                 dct['container'] = dct['ext'] + '_dash'
 
-            if all_formats and dct['filesize']:
+            if (all_formats or 'dashy' in format_types) and dct['filesize']:
                 yield {
                     **dct,
                     'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
                     'protocol': 'http_dash_segments',
                     'fragments': build_fragments(dct),
                 }
-            dct['downloader_options'] = {'http_chunk_size': CHUNK_SIZE}
-            yield dct
+            if all_formats or 'dashy' not in format_types:
+                dct['downloader_options'] = {'http_chunk_size': CHUNK_SIZE}
+                yield dct
 
         needs_live_processing = self._needs_live_processing(live_status, duration)
-        skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
+        skip_bad_formats = 'incomplete' not in format_types
+        if self._configuration_arg('include_incomplete_formats'):
+            skip_bad_formats = False
+            self._downloader.deprecated_feature('[youtube] include_incomplete_formats extractor argument is deprecated. '
+                                                'Use formats=incomplete extractor argument instead')
 
         skip_manifests = set(self._configuration_arg('skip'))
         if (not self.get_param('youtube_include_hls_manifest', True)
@@ -3915,7 +3925,7 @@ def build_fragments(f):
             skip_manifests.add('dash')
         if self._configuration_arg('include_live_dash'):
             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
-                                                'Use include_incomplete_formats extractor argument instead')
+                                                'Use formats=incomplete extractor argument instead')
         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
             skip_manifests.add('dash')
 

From f0a1ff118145b6449982ba401f9a9f656ecd8062 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 25 Jun 2023 13:13:28 -0500
Subject: [PATCH 404/871] [extractor/qdance] Add extractor (#7420)

Closes #7385
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/qdance.py      | 150 ++++++++++++++++++++++++++++++++
 2 files changed, 151 insertions(+)
 create mode 100644 yt_dlp/extractor/qdance.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 49a3f39d3..06340fcd8 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1531,6 +1531,7 @@
 )
 from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
+from .qdance import QDanceIE
 from .qingting import QingTingIE
 from .qqmusic import (
     QQMusicIE,
diff --git a/yt_dlp/extractor/qdance.py b/yt_dlp/extractor/qdance.py
new file mode 100644
index 000000000..d817677f0
--- /dev/null
+++ b/yt_dlp/extractor/qdance.py
@@ -0,0 +1,150 @@
+import json
+import time
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    jwt_decode_hs256,
+    str_or_none,
+    traverse_obj,
+    try_call,
+    url_or_none,
+)
+
+
+class QDanceIE(InfoExtractor):
+    _NETRC_MACHINE = 'qdance'
+    _VALID_URL = r'https?://(?:www\.)?q-dance\.com/network/(?:library|live)/(?P<id>\d+)'
+    _TESTS = [{
+        'note': 'vod',
+        'url': 'https://www.q-dance.com/network/library/146542138',
+        'info_dict': {
+            'id': '146542138',
+            'ext': 'mp4',
+            'title': 'Sound Rush [LIVE] | Defqon.1 Weekend Festival 2022 | Friday | RED',
+            'display_id': 'sound-rush-live-v3-defqon-1-weekend-festival-2022-friday-red',
+            'description': 'Relive Defqon.1 - Primal Energy 2022 with the sounds of Sound Rush LIVE at the RED on Friday! 🔥',
+            'season': 'Defqon.1 Weekend Festival 2022',
+            'season_id': '31840632',
+            'series': 'Defqon.1',
+            'series_id': '31840378',
+            'thumbnail': 'https://images.q-dance.network/1674829540-20220624171509-220624171509_delio_dn201093-2.jpg',
+            'availability': 'premium_only',
+            'duration': 1829,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'livestream',
+        'url': 'https://www.q-dance.com/network/live/149170353',
+        'info_dict': {
+            'id': '149170353',
+            'ext': 'mp4',
+            'title': r're:^Defqon\.1 2023 - Friday - RED',
+            'display_id': 'defqon-1-2023-friday-red',
+            'description': 'md5:3c73fbbd4044e578e696adfc64019163',
+            'season': 'Defqon.1 Weekend Festival 2023',
+            'season_id': '141735599',
+            'series': 'Defqon.1',
+            'series_id': '31840378',
+            'thumbnail': 'https://images.q-dance.network/1686849069-area-thumbs_red.png',
+            'availability': 'subscriber_only',
+            'live_status': 'is_live',
+            'channel_id': 'qdancenetwork.video_149170353',
+        },
+        'skip': 'Completed livestream',
+    }]
+
+    _access_token = None
+    _refresh_token = None
+
+    def _call_login_api(self, data, note='Logging in'):
+        login = self._download_json(
+            'https://members.id-t.com/api/auth/login', None, note, headers={
+                'content-type': 'application/json',
+                'brand': 'qdance',
+                'origin': 'https://www.q-dance.com',
+                'referer': 'https://www.q-dance.com/',
+            }, data=json.dumps(data, separators=(',', ':')).encode(),
+            expected_status=lambda x: True)
+
+        tokens = traverse_obj(login, ('data', {
+            '_id-t-accounts-token': ('accessToken', {str}),
+            '_id-t-accounts-refresh': ('refreshToken', {str}),
+            '_id-t-accounts-id-token': ('idToken', {str}),
+        }))
+
+        if not tokens.get('_id-t-accounts-token'):
+            error = ': '.join(traverse_obj(login, ('error', ('code', 'message'), {str})))
+            if 'validation_error' not in error:
+                raise ExtractorError(f'Q-Dance API said "{error}"')
+            msg = 'Invalid username or password' if 'email' in data else 'Refresh token has expired'
+            raise ExtractorError(msg, expected=True)
+
+        for name, value in tokens.items():
+            self._set_cookie('.q-dance.com', name, value)
+
+    def _perform_login(self, username, password):
+        self._call_login_api({'email': username, 'password': password})
+
+    def _real_initialize(self):
+        cookies = self._get_cookies('https://www.q-dance.com/')
+        self._refresh_token = try_call(lambda: cookies['_id-t-accounts-refresh'].value)
+        self._access_token = try_call(lambda: cookies['_id-t-accounts-token'].value)
+        if not self._access_token:
+            self.raise_login_required()
+
+    def _get_auth(self):
+        if (try_call(lambda: jwt_decode_hs256(self._access_token)['exp']) or 0) <= int(time.time() - 120):
+            if not self._refresh_token:
+                raise ExtractorError(
+                    'Cannot refresh access token, login with yt-dlp or refresh cookies in browser')
+            self._call_login_api({'refreshToken': self._refresh_token}, note='Refreshing access token')
+            self._real_initialize()
+
+        return {'Authorization': self._access_token}
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        data = self._search_nuxt_data(webpage, video_id, traverse=('data', 0, 'data'))
+
+        def extract_availability(level):
+            level = int_or_none(level) or 0
+            return self._availability(
+                needs_premium=(level >= 20), needs_subscription=(level >= 15), needs_auth=True)
+
+        info = traverse_obj(data, {
+            'title': ('title', {str.strip}),
+            'description': ('description', {str.strip}),
+            'display_id': ('slug', {str}),
+            'thumbnail': ('thumbnail', {url_or_none}),
+            'duration': ('durationInSeconds', {int_or_none}, {lambda x: x or None}),
+            'availability': ('subscription', 'level', {extract_availability}),
+            'is_live': ('type', {lambda x: x.lower() == 'live'}),
+            'artist': ('acts', ..., {str}),
+            'series': ('event', 'title', {str.strip}),
+            'series_id': ('event', 'id', {str_or_none}),
+            'season': ('eventEdition', 'title', {str.strip}),
+            'season_id': ('eventEdition', 'id', {str_or_none}),
+            'channel_id': ('pubnub', 'channelName', {str}),
+        })
+
+        stream = self._download_json(
+            f'https://dc9h6qmsoymbq.cloudfront.net/api/content/videos/{video_id}/url',
+            video_id, headers=self._get_auth(), expected_status=401)
+
+        m3u8_url = traverse_obj(stream, ('data', 'url', {url_or_none}))
+        if not m3u8_url and traverse_obj(stream, ('error', 'code')) == 'unauthorized':
+            raise ExtractorError('Your account does not have access to this content', expected=True)
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, fatal=False, live=True) if m3u8_url else []
+        if not formats:
+            self.raise_no_formats('No active streams found', expected=bool(info.get('is_live')))
+
+        return {
+            **info,
+            'id': video_id,
+            'formats': formats,
+        }

From 5e16cf92eb496b7c1541a6b1d727cb87542984db Mon Sep 17 00:00:00 2001
From: nnoboa <90611593+nnoboa@users.noreply.github.com>
Date: Sun, 25 Jun 2023 16:22:38 -0400
Subject: [PATCH 405/871] [extractor/AdultSwim] Extract subtitles from m3u8
 (#7421)

Authored by: nnoboa
Closes #6191
---
 yt_dlp/extractor/adultswim.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/adultswim.py b/yt_dlp/extractor/adultswim.py
index bd29eb43e..daaeddeb6 100644
--- a/yt_dlp/extractor/adultswim.py
+++ b/yt_dlp/extractor/adultswim.py
@@ -170,8 +170,10 @@ def _real_extract(self, url):
                         continue
                     ext = determine_ext(asset_url, mimetype2ext(asset.get('mime_type')))
                     if ext == 'm3u8':
-                        info['formats'].extend(self._extract_m3u8_formats(
-                            asset_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                        fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                            asset_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                        info['formats'].extend(fmts)
+                        self._merge_subtitles(subs, target=info['subtitles'])
                     elif ext == 'f4m':
                         continue
                         # info['formats'].extend(self._extract_f4m_formats(

From ef8509c300ea50da86aea447eb214d3d6f6db6bb Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sun, 25 Jun 2023 17:04:42 -0500
Subject: [PATCH 406/871] [extractor/kick] Fix `_VALID_URL`

Closes #7384
Authored by: bashonly
---
 yt_dlp/extractor/kick.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/kick.py b/yt_dlp/extractor/kick.py
index 765ffa0c8..be1dfd4b1 100644
--- a/yt_dlp/extractor/kick.py
+++ b/yt_dlp/extractor/kick.py
@@ -30,7 +30,7 @@ def _call_api(self, path, display_id, note='Downloading API JSON', headers={}, *
 
 
 class KickIE(KickBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?kick\.com/(?!(?:video|categories|search|auth)(?:[/?#]|$))(?P<id>[\w_]+)'
+    _VALID_URL = r'https?://(?:www\.)?kick\.com/(?!(?:video|categories|search|auth)(?:[/?#]|$))(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://kick.com/yuppy',
         'info_dict': {

From d949c10c45bfc359bdacd52e6a180169b8128958 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 26 Jun 2023 07:25:47 +0530
Subject: [PATCH 407/871] [extractor/youtube] Process `post_live` over 2 hours

---
 yt_dlp/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index bdc631ccb..d5607975e 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3737,7 +3737,7 @@ def append_client(*client_names):
 
     def _needs_live_processing(self, live_status, duration):
         if (live_status == 'is_live' and self.get_param('live_from_start')
-                or live_status == 'post_live' and (duration or 0) > 4 * 3600):
+                or live_status == 'post_live' and (duration or 0) > 2 * 3600):
             return live_status
 
     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
@@ -4238,7 +4238,7 @@ def is_bad_format(fmt):
 
         for fmt in filter(is_bad_format, formats):
             fmt['preference'] = (fmt.get('preference') or -1) - 10
-            fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
+            fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 2 hours)', delim=' ')
 
         if needs_live_processing:
             self._prepare_live_from_start_formats(

From 8a8af356e3bba98a7f7d333aff0777d5d92130c8 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 26 Jun 2023 16:13:31 +0530
Subject: [PATCH 408/871] [downloader/aria2c] Add `--no-conf`

Closes #7404
---
 yt_dlp/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index 007689a8c..f637a100b 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -271,7 +271,7 @@ def _call_downloader(self, tmpfilename, info_dict):
         return super()._call_downloader(tmpfilename, info_dict)
 
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [self.exe, '-c',
+        cmd = [self.exe, '-c', '--no-conf',
                '--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
                '--http-accept-gzip=true', '--file-allocation=none', '-x16', '-j16', '-s16']
         if 'fragments' in info_dict:

From f393bbe724b1fc6c7f754a5da507e807b2b40ad2 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 26 Jun 2023 16:14:20 +0530
Subject: [PATCH 409/871] [extractor/sbs] Python 3.7 compat

Closes #7410
---
 yt_dlp/extractor/sbs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/sbs.py b/yt_dlp/extractor/sbs.py
index ac0b6de20..119106e8e 100644
--- a/yt_dlp/extractor/sbs.py
+++ b/yt_dlp/extractor/sbs.py
@@ -139,8 +139,8 @@ def _real_extract(self, url):
                 'release_year': ('releaseYear', {int_or_none}),
                 'duration': ('duration', ({float_or_none}, {parse_duration})),
                 'is_live': ('liveStream', {bool}),
-                'age_limit': (
-                    ('classificationID', 'contentRating'), {str.upper}, {self._AUS_TV_PARENTAL_GUIDELINES.get}),
+                'age_limit': (('classificationID', 'contentRating'), {str.upper}, {
+                    lambda x: self._AUS_TV_PARENTAL_GUIDELINES.get(x)}),  # dict.get is unhashable in py3.7
             }, get_all=False),
             **traverse_obj(media, {
                 'categories': (('genres', ...), ('taxonomy', ('genre', 'subgenre'), 'name'), {str}),

From 91302ed349f34dc26cc1d661bb45a4b71f4417f7 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 26 Jun 2023 16:19:49 +0530
Subject: [PATCH 410/871] [utils] clean_podcast_url: Handle protocol in
 redirect URL

Closes #7430
---
 yt_dlp/utils/_utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index de51f6208..f68cdb968 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -5113,7 +5113,7 @@ def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='',
 
 
 def clean_podcast_url(url):
-    return re.sub(r'''(?x)
+    url = re.sub(r'''(?x)
         (?:
             (?:
                 chtbl\.com/track|
@@ -5127,6 +5127,7 @@ def clean_podcast_url(url):
                 st\.fm # https://podsights.com/docs/
             )/e
         )/''', '', url)
+    return re.sub(r'^\w+://(\w+://)', r'\1', url)
 
 
 _HEX_TABLE = '0123456789abcdef'

From 5b4b92769afcc398475e481bfa839f1158902fe9 Mon Sep 17 00:00:00 2001
From: Aman Salwan <121633121+AmanSal1@users.noreply.github.com>
Date: Wed, 28 Jun 2023 01:58:23 +0530
Subject: [PATCH 411/871] [extractor/crunchyroll:music] Fix `_VALID_URL`
 (#7439)

Closes #7419
Authored by: AmanSal1, rdamas

Co-authored-by: Robert Damas <robert.damas@byom.de>
---
 yt_dlp/extractor/crunchyroll.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py
index d4a21616b..910504ed2 100644
--- a/yt_dlp/extractor/crunchyroll.py
+++ b/yt_dlp/extractor/crunchyroll.py
@@ -490,8 +490,21 @@ class CrunchyrollMusicIE(CrunchyrollBaseIE):
     _VALID_URL = r'''(?x)
         https?://(?:www\.)?crunchyroll\.com/
         (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
-        watch/(?P<type>concert|musicvideo)/(?P<id>\w{10})'''
+        watch/(?P<type>concert|musicvideo)/(?P<id>\w+)'''
     _TESTS = [{
+        'url': 'https://www.crunchyroll.com/de/watch/musicvideo/MV5B02C79',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': 'MV5B02C79',
+            'display_id': 'egaono-hana',
+            'title': 'Egaono Hana',
+            'track': 'Egaono Hana',
+            'artist': 'Goose house',
+            'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'genre': ['J-Pop'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
         'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C',
         'info_dict': {
             'ext': 'mp4',
@@ -519,11 +532,14 @@ class CrunchyrollMusicIE(CrunchyrollBaseIE):
         },
         'params': {'skip_download': 'm3u8'},
     }, {
-        'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C/crossing-field',
+        'url': 'https://www.crunchyroll.com/de/watch/musicvideo/MV5B02C79/egaono-hana',
         'only_matching': True,
     }, {
         'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135/live-is-smile-always-364joker-at-yokohama-arena',
         'only_matching': True,
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C/crossing-field',
+        'only_matching': True,
     }]
     _API_ENDPOINT = 'music'
 

From 8f05fbae2a79ce0713077ccc68b354e63216bf20 Mon Sep 17 00:00:00 2001
From: Xiao Han <38774211+meliber@users.noreply.github.com>
Date: Tue, 27 Jun 2023 16:16:57 -0500
Subject: [PATCH 412/871] [extractor/abc] Fix extraction (#7434)

Closes #6433
Authored by: meliber
---
 yt_dlp/extractor/abc.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/abc.py b/yt_dlp/extractor/abc.py
index 0ca76b85a..f56133eb3 100644
--- a/yt_dlp/extractor/abc.py
+++ b/yt_dlp/extractor/abc.py
@@ -12,6 +12,7 @@
     int_or_none,
     parse_iso8601,
     str_or_none,
+    traverse_obj,
     try_get,
     unescapeHTML,
     update_url_query,
@@ -85,6 +86,15 @@ class ABCIE(InfoExtractor):
             'uploader': 'Behind the News',
             'uploader_id': 'behindthenews',
         }
+    }, {
+        'url': 'https://www.abc.net.au/news/2023-06-25/wagner-boss-orders-troops-back-to-bases-to-avoid-bloodshed/102520540',
+        'info_dict': {
+            'id': '102520540',
+            'title': 'Wagner Group retreating from Russia, leader Prigozhin to move to Belarus',
+            'ext': 'mp4',
+            'description': 'Wagner troops leave Rostov-on-Don and\xa0Yevgeny Prigozhin will move to Belarus under a deal brokered by Belarusian President Alexander Lukashenko to end the mutiny.',
+            'thumbnail': 'https://live-production.wcms.abc-cdn.net.au/0c170f5b57f0105c432f366c0e8e267b?impolicy=wcms_crop_resize&cropH=2813&cropW=5000&xPos=0&yPos=249&width=862&height=485',
+        }
     }]
 
     def _real_extract(self, url):
@@ -107,7 +117,7 @@ def _real_extract(self, url):
                 video = True
 
         if mobj is None:
-            mobj = re.search(r'(?P<type>)"sources": (?P<json_data>\[[^\]]+\]),', webpage)
+            mobj = re.search(r'(?P<type>)"(?:sources|files|renditions)":\s*(?P<json_data>\[[^\]]+\])', webpage)
             if mobj is None:
                 mobj = re.search(
                     r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
@@ -121,7 +131,8 @@ def _real_extract(self, url):
             urls_info = self._parse_json(
                 mobj.group('json_data'), video_id, transform_source=js_to_json)
             youtube = mobj.group('type') == 'YouTube'
-            video = mobj.group('type') == 'Video' or urls_info[0]['contentType'] == 'video/mp4'
+            video = mobj.group('type') == 'Video' or traverse_obj(
+                urls_info, (0, ('contentType', 'MIMEType')), get_all=False) == 'video/mp4'
 
         if not isinstance(urls_info, list):
             urls_info = [urls_info]

From a2be9781fbf4d7e4db245c277ca2ecc41cf3a7b2 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Tue, 27 Jun 2023 16:50:02 -0500
Subject: [PATCH 413/871] [extractor/Douyin] Fix extraction from webpage

Closes #7431
Authored by: bashonly
---
 yt_dlp/extractor/tiktok.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index 9c6d74007..2f491c317 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -1015,18 +1015,16 @@ def _real_extract(self, url):
             self.to_screen(f'{e}; trying with webpage')
 
         webpage = self._download_webpage(url, video_id)
-        render_data_json = self._search_regex(
-            r'<script [^>]*\bid=[\'"]RENDER_DATA[\'"][^>]*>(%7B.+%7D)</script>',
-            webpage, 'render data', default=None)
-        if not render_data_json:
+        render_data = self._search_json(
+            r'<script [^>]*\bid=[\'"]RENDER_DATA[\'"][^>]*>', webpage, 'render data', video_id,
+            contains_pattern=r'%7B(?s:.+)%7D', fatal=False, transform_source=compat_urllib_parse_unquote)
+        if not render_data:
             # TODO: Run verification challenge code to generate signature cookies
             cookies = self._get_cookies(self._WEBPAGE_HOST)
             expected = not cookies.get('s_v_web_id') or not cookies.get('ttwid')
             raise ExtractorError(
                 'Fresh cookies (not necessarily logged in) are needed', expected=expected)
 
-        render_data = self._parse_json(
-            render_data_json, video_id, transform_source=compat_urllib_parse_unquote)
         return self._parse_aweme_video_web(get_first(render_data, ('aweme', 'detail')), url, video_id)
 
 

From fcbc9ed760be6e3455bbadfaf277b4504b06f068 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Thu, 29 Jun 2023 23:26:27 +0000
Subject: [PATCH 414/871] [extractor/youtube:tab] Support shorts-only playlists
 (#7425)

Fixes https://github.com/yt-dlp/yt-dlp/issues/7424

Authored by: coletdjnz
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
---
 yt_dlp/extractor/youtube.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index d5607975e..967914c0f 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4898,7 +4898,8 @@ def _extract_entries(self, parent_renderer, continuation_list):
                     'videoRenderer': lambda x: [self._video_entry(x)],
                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
-                    'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
+                    'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)],
+                    'richGridRenderer': lambda x: self._extract_entries(x, continuation_list),
                 }
                 for key, renderer in isr_content.items():
                     if key not in known_renderers:
@@ -6390,6 +6391,28 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'channel_is_verified': True,
         },
         'playlist_mincount': 10,
+    }, {
+        # Playlist with only shorts, shown as reel renderers
+        # FIXME: future: YouTube currently doesn't give continuation for this,
+        # may do in future.
+        'url': 'https://www.youtube.com/playlist?list=UUxqPAgubo4coVn9Lx1FuKcg',
+        'info_dict': {
+            'id': 'UUxqPAgubo4coVn9Lx1FuKcg',
+            'channel_url': 'https://www.youtube.com/channel/UCxqPAgubo4coVn9Lx1FuKcg',
+            'view_count': int,
+            'uploader_id': '@BangyShorts',
+            'description': '',
+            'uploader_url': 'https://www.youtube.com/@BangyShorts',
+            'channel_id': 'UCxqPAgubo4coVn9Lx1FuKcg',
+            'channel': 'Bangy Shorts',
+            'uploader': 'Bangy Shorts',
+            'tags': [],
+            'availability': 'public',
+            'modified_date': '20230626',
+            'title': 'Uploads from Bangy Shorts',
+        },
+        'playlist_mincount': 100,
+        'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
     }]
 
     @classmethod

From af1fd12f675220df6793fc019dff320bc76e8080 Mon Sep 17 00:00:00 2001
From: urectanc <5403400+urectanc@users.noreply.github.com>
Date: Sat, 1 Jul 2023 03:27:07 +0900
Subject: [PATCH 415/871] [extractor/stacommu] Add extractors (#7432)

Authored by: urectanc
---
 README.md                           |   2 +-
 yt_dlp/extractor/_extractors.py     |   4 +
 yt_dlp/extractor/stacommu.py        | 148 ++++++++++++++++++++++++++++
 yt_dlp/extractor/wrestleuniverse.py |  33 ++++---
 4 files changed, 173 insertions(+), 14 deletions(-)
 create mode 100644 yt_dlp/extractor/stacommu.py

diff --git a/README.md b/README.md
index d89bb204e..066ff9052 100644
--- a/README.md
+++ b/README.md
@@ -1855,7 +1855,7 @@ #### rokfinchannel
 #### twitter
 * `legacy_api`: Force usage of the legacy Twitter API instead of the GraphQL API for tweet extraction. Has no effect if login cookies are passed
 
-#### wrestleuniverse
+#### stacommu, wrestleuniverse
 * `device_id`: UUID value assigned by the website and used to enforce device limits for paid livestream content. Can be found in browser local storage
 
 #### twitch
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 06340fcd8..76a7fef23 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1855,6 +1855,10 @@
     SRGSSRPlayIE,
 )
 from .srmediathek import SRMediathekIE
+from .stacommu import (
+    StacommuLiveIE,
+    StacommuVODIE,
+)
 from .stanfordoc import StanfordOpenClassroomIE
 from .startv import StarTVIE
 from .steam import (
diff --git a/yt_dlp/extractor/stacommu.py b/yt_dlp/extractor/stacommu.py
new file mode 100644
index 000000000..6f58f06dc
--- /dev/null
+++ b/yt_dlp/extractor/stacommu.py
@@ -0,0 +1,148 @@
+import time
+
+from .wrestleuniverse import WrestleUniverseBaseIE
+from ..utils import (
+    int_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class StacommuBaseIE(WrestleUniverseBaseIE):
+    _NETRC_MACHINE = 'stacommu'
+    _API_HOST = 'api.stacommu.jp'
+    _LOGIN_QUERY = {'key': 'AIzaSyCR9czxhH2eWuijEhTNWBZ5MCcOYEUTAhg'}
+    _LOGIN_HEADERS = {
+        'Accept': '*/*',
+        'Content-Type': 'application/json',
+        'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web',
+        'Referer': 'https://www.stacommu.jp/',
+        'Origin': 'https://www.stacommu.jp',
+    }
+
+    @WrestleUniverseBaseIE._TOKEN.getter
+    def _TOKEN(self):
+        if self._REAL_TOKEN and self._TOKEN_EXPIRY <= int(time.time()):
+            self._refresh_token()
+
+        return self._REAL_TOKEN
+
+    def _get_formats(self, data, path, video_id=None):
+        if not traverse_obj(data, path) and not data.get('canWatch') and not self._TOKEN:
+            self.raise_login_required(method='password')
+        return super()._get_formats(data, path, video_id)
+
+    def _extract_hls_key(self, data, path, decrypt):
+        encryption_data = traverse_obj(data, path)
+        if traverse_obj(encryption_data, ('encryptType', {int})) == 0:
+            return None
+        return traverse_obj(encryption_data, {'key': ('key', {decrypt}), 'iv': ('iv', {decrypt})})
+
+
+class StacommuVODIE(StacommuBaseIE):
+    _VALID_URL = r'https?://www\.stacommu\.jp/videos/episodes/(?P<id>[\da-zA-Z]+)'
+    _TESTS = [{
+        # not encrypted
+        'url': 'https://www.stacommu.jp/videos/episodes/aXcVKjHyAENEjard61soZZ',
+        'info_dict': {
+            'id': 'aXcVKjHyAENEjard61soZZ',
+            'ext': 'mp4',
+            'title': 'スタコミュAWARDの裏側、ほぼ全部見せます！〜晴れ舞台の直前ドキドキ編〜',
+            'description': 'md5:6400275c57ae75c06da36b06f96beb1c',
+            'timestamp': 1679652000,
+            'upload_date': '20230324',
+            'thumbnail': 'https://image.stacommu.jp/6eLobQan8PFtBoU4RL4uGg/6eLobQan8PFtBoU4RL4uGg',
+            'cast': 'count:11',
+            'duration': 250,
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }, {
+        # encrypted; requires a premium account
+        'url': 'https://www.stacommu.jp/videos/episodes/3hybMByUvzMEqndSeu5LpD',
+        'info_dict': {
+            'id': '3hybMByUvzMEqndSeu5LpD',
+            'ext': 'mp4',
+            'title': 'スタプラフェス2023〜裏側ほぼ全部見せます〜＃10',
+            'description': 'md5:85494488ccf1dfa1934accdeadd7b340',
+            'timestamp': 1682506800,
+            'upload_date': '20230426',
+            'thumbnail': 'https://image.stacommu.jp/eMdXtEefR4kEyJJMpAFi7x/eMdXtEefR4kEyJJMpAFi7x',
+            'cast': 'count:55',
+            'duration': 312,
+            'hls_aes': {
+                'key': '6bbaf241b8e1fd9f59ecf546a70e4ae7',
+                'iv': '1fc9002a23166c3bb1d240b953d09de9',
+            },
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }]
+
+    _API_PATH = 'videoEpisodes'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_info = self._download_metadata(
+            url, video_id, 'ja', ('dehydratedState', 'queries', 0, 'state', 'data'))
+        hls_info, decrypt = self._call_encrypted_api(
+            video_id, ':watch', 'stream information', data={'method': 1})
+
+        return {
+            'id': video_id,
+            'formats': self._get_formats(hls_info, ('protocolHls', 'url', {url_or_none}), video_id),
+            'hls_aes': self._extract_hls_key(hls_info, 'protocolHls', decrypt),
+            **traverse_obj(video_info, {
+                'title': ('displayName', {str}),
+                'description': ('description', {str}),
+                'timestamp': ('watchStartTime', {int_or_none}),
+                'thumbnail': ('keyVisualUrl', {url_or_none}),
+                'cast': ('casts', ..., 'displayName', {str}),
+                'duration': ('duration', {int}),
+            }),
+        }
+
+
+class StacommuLiveIE(StacommuBaseIE):
+    _VALID_URL = r'https?://www\.stacommu\.jp/live/(?P<id>[\da-zA-Z]+)'
+    _TESTS = [{
+        'url': 'https://www.stacommu.jp/live/d2FJ3zLnndegZJCAEzGM3m',
+        'info_dict': {
+            'id': 'd2FJ3zLnndegZJCAEzGM3m',
+            'ext': 'mp4',
+            'title': '仲村悠菜 2023/05/04',
+            'timestamp': 1683195647,
+            'upload_date': '20230504',
+            'thumbnail': 'https://image.stacommu.jp/pHGF57SPEHE2ke83FS92FN/pHGF57SPEHE2ke83FS92FN',
+            'duration': 5322,
+            'hls_aes': {
+                'key': 'efbb3ec0b8246f61adf1764c5a51213a',
+                'iv': '80621d19a1f19167b64cedb415b05d1c',
+            },
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }]
+
+    _API_PATH = 'events'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_info = self._call_api(video_id, msg='video information', query={'al': 'ja'}, auth=False)
+        hls_info, decrypt = self._call_encrypted_api(
+            video_id, ':watchArchive', 'stream information', data={'method': 1})
+
+        return {
+            'id': video_id,
+            'formats': self._get_formats(hls_info, ('hls', 'urls', ..., {url_or_none}), video_id),
+            'hls_aes': self._extract_hls_key(hls_info, 'hls', decrypt),
+            **traverse_obj(video_info, {
+                'title': ('displayName', {str}),
+                'timestamp': ('startTime', {int_or_none}),
+                'thumbnail': ('keyVisualUrl', {url_or_none}),
+                'duration': ('duration', {int_or_none}),
+            }),
+        }
diff --git a/yt_dlp/extractor/wrestleuniverse.py b/yt_dlp/extractor/wrestleuniverse.py
index b12b0f0a9..99a8f0120 100644
--- a/yt_dlp/extractor/wrestleuniverse.py
+++ b/yt_dlp/extractor/wrestleuniverse.py
@@ -14,12 +14,14 @@
     try_call,
     url_or_none,
     urlencode_postdata,
+    variadic,
 )
 
 
 class WrestleUniverseBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'wrestleuniverse'
     _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
+    _API_HOST = 'api.wrestle-universe.com'
     _API_PATH = None
     _REAL_TOKEN = None
     _TOKEN_EXPIRY = None
@@ -67,24 +69,28 @@ def _perform_login(self, username, password):
                 'returnSecureToken': True,
                 'email': username,
                 'password': password,
-            }, separators=(',', ':')).encode())
+            }, separators=(',', ':')).encode(), expected_status=400)
+        token = traverse_obj(login, ('idToken', {str}))
+        if not token:
+            raise ExtractorError(
+                f'Unable to log in: {traverse_obj(login, ("error", "message"))}', expected=True)
         self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str}))
         if not self._REFRESH_TOKEN:
             self.report_warning('No refresh token was granted')
-        self._TOKEN = traverse_obj(login, ('idToken', {str}))
+        self._TOKEN = token
 
     def _real_initialize(self):
-        if WrestleUniverseBaseIE._DEVICE_ID:
+        if self._DEVICE_ID:
             return
 
-        WrestleUniverseBaseIE._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key='WrestleUniverse')[0]
-        if not WrestleUniverseBaseIE._DEVICE_ID:
-            WrestleUniverseBaseIE._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
-            if WrestleUniverseBaseIE._DEVICE_ID:
+        self._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key=self._NETRC_MACHINE)[0]
+        if not self._DEVICE_ID:
+            self._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
+            if self._DEVICE_ID:
                 return
-            WrestleUniverseBaseIE._DEVICE_ID = str(uuid.uuid4())
+            self._DEVICE_ID = str(uuid.uuid4())
 
-        self.cache.store(self._NETRC_MACHINE, 'device_id', WrestleUniverseBaseIE._DEVICE_ID)
+        self.cache.store(self._NETRC_MACHINE, 'device_id', self._DEVICE_ID)
 
     def _refresh_token(self):
         refresh = self._download_json(
@@ -108,10 +114,10 @@ def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={
         if data:
             headers['Content-Type'] = 'application/json;charset=utf-8'
             data = json.dumps(data, separators=(',', ':')).encode()
-        if auth:
+        if auth and self._TOKEN:
             headers['Authorization'] = f'Bearer {self._TOKEN}'
         return self._download_json(
-            f'https://api.wrestle-universe.com/v1/{self._API_PATH}/{video_id}{param}', video_id,
+            f'https://{self._API_HOST}/v1/{self._API_PATH}/{video_id}{param}', video_id,
             note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
             data=data, headers=headers, query=query, fatal=fatal)
 
@@ -137,12 +143,13 @@ def decrypt(data):
         }, query=query, fatal=fatal)
         return api_json, decrypt
 
-    def _download_metadata(self, url, video_id, lang, props_key):
+    def _download_metadata(self, url, video_id, lang, props_keys):
         metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
         if not metadata:
             webpage = self._download_webpage(url, video_id)
             nextjs_data = self._search_nextjs_data(webpage, video_id)
-            metadata = traverse_obj(nextjs_data, ('props', 'pageProps', props_key, {dict})) or {}
+            metadata = traverse_obj(nextjs_data, (
+                'props', 'pageProps', *variadic(props_keys, (str, bytes, dict, set)), {dict})) or {}
         return metadata
 
     def _get_formats(self, data, path, video_id=None):

From 8776349ef6b1f644584a92dfa00a05208a48edc4 Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Sun, 2 Jul 2023 15:31:00 -0400
Subject: [PATCH 416/871] [extractor/vk] VKPlay, VKPlayLive: Add extractors
 (#7358)

Closes #7107
Authored by: c-basalt
---
 yt_dlp/extractor/_extractors.py |   2 +
 yt_dlp/extractor/vk.py          | 139 ++++++++++++++++++++++++++++++++
 2 files changed, 141 insertions(+)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 76a7fef23..6f1873383 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2272,6 +2272,8 @@
     VKIE,
     VKUserVideosIE,
     VKWallPostIE,
+    VKPlayIE,
+    VKPlayLiveIE,
 )
 from .vocaroo import VocarooIE
 from .vodlocker import VodlockerIE
diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py
index 16ca954f2..575369028 100644
--- a/yt_dlp/extractor/vk.py
+++ b/yt_dlp/extractor/vk.py
@@ -11,11 +11,13 @@
 from .youtube import YoutubeIE
 from ..utils import (
     ExtractorError,
+    UserNotLive,
     clean_html,
     get_element_by_class,
     get_element_html_by_id,
     int_or_none,
     join_nonempty,
+    parse_resolution,
     str_or_none,
     str_to_int,
     try_call,
@@ -25,6 +27,7 @@
     url_or_none,
     urlencode_postdata,
     urljoin,
+    traverse_obj,
 )
 
 
@@ -701,3 +704,139 @@ def _real_extract(self, url):
         return self.playlist_result(
             entries, post_id, join_nonempty(uploader, f'Wall post {post_id}', delim=' - '),
             clean_html(get_element_by_class('wall_post_text', webpage)))
+
+
+class VKPlayBaseIE(InfoExtractor):
+    _RESOLUTIONS = {
+        'tiny': '256x144',
+        'lowest': '426x240',
+        'low': '640x360',
+        'medium': '852x480',
+        'high': '1280x720',
+        'full_hd': '1920x1080',
+        'quad_hd': '2560x1440',
+    }
+
+    def _extract_from_initial_state(self, url, video_id, path):
+        webpage = self._download_webpage(url, video_id)
+        video_info = traverse_obj(self._search_json(
+            r'<script[^>]+\bid="initial-state"[^>]*>', webpage, 'initial state', video_id),
+            path, expected_type=dict)
+        if not video_info:
+            raise ExtractorError('Unable to extract video info from html inline initial state')
+        return video_info
+
+    def _extract_formats(self, stream_info, video_id):
+        formats = []
+        for stream in traverse_obj(stream_info, (
+                'data', 0, 'playerUrls', lambda _, v: url_or_none(v['url']) and v['type'])):
+            url = stream['url']
+            format_id = str_or_none(stream['type'])
+            if format_id in ('hls', 'live_hls', 'live_playback_hls') or '.m3u8' in url:
+                formats.extend(self._extract_m3u8_formats(url, video_id, m3u8_id=format_id, fatal=False))
+            elif format_id == 'dash':
+                formats.extend(self._extract_mpd_formats(url, video_id, mpd_id=format_id, fatal=False))
+            elif format_id in ('live_dash', 'live_playback_dash'):
+                self.write_debug(f'Not extracting unsupported format "{format_id}"')
+            else:
+                formats.append({
+                    'url': url,
+                    'ext': 'mp4',
+                    'format_id': format_id,
+                    **parse_resolution(self._RESOLUTIONS.get(format_id)),
+                })
+        return formats
+
+    def _extract_common_meta(self, stream_info):
+        return traverse_obj(stream_info, {
+            'id': ('id', {str_or_none}),
+            'title': ('title', {str}),
+            'release_timestamp': ('startTime', {int_or_none}),
+            'thumbnail': ('previewUrl', {url_or_none}),
+            'view_count': ('count', 'views', {int_or_none}),
+            'like_count': ('count', 'likes', {int_or_none}),
+            'categories': ('category', 'title', {str}, {lambda x: [x] if x else None}),
+            'uploader': (('user', ('blog', 'owner')), 'nick', {str}),
+            'uploader_id': (('user', ('blog', 'owner')), 'id', {str_or_none}),
+            'duration': ('duration', {int_or_none}),
+            'is_live': ('isOnline', {bool}),
+            'concurrent_view_count': ('count', 'viewers', {int_or_none}),
+        }, get_all=False)
+
+
+class VKPlayIE(VKPlayBaseIE):
+    _VALID_URL = r'https?://vkplay\.live/(?P<username>[^/]+)/record/(?P<id>[a-f0-9\-]+)'
+    _TESTS = [{
+        'url': 'https://vkplay.live/zitsmann/record/f5e6e3b5-dc52-4d14-965d-0680dd2882da',
+        'info_dict': {
+            'id': 'f5e6e3b5-dc52-4d14-965d-0680dd2882da',
+            'ext': 'mp4',
+            'title': 'Atomic Heart (пробуем!) спасибо подписчику EKZO!',
+            'uploader': 'ZitsmanN',
+            'uploader_id': '13159830',
+            'release_timestamp': 1683461378,
+            'release_date': '20230507',
+            'thumbnail': r're:https://images.vkplay.live/public_video_stream/record/f5e6e3b5-dc52-4d14-965d-0680dd2882da/preview\?change_time=\d+',
+            'duration': 10608,
+            'view_count': int,
+            'like_count': int,
+            'categories': ['Atomic Heart'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        username, video_id = self._match_valid_url(url).groups()
+
+        record_info = traverse_obj(self._download_json(
+            f'https://api.vkplay.live/v1/blog/{username}/public_video_stream/record/{video_id}', video_id, fatal=False),
+            ('data', 'record', {dict}))
+        if not record_info:
+            record_info = self._extract_from_initial_state(url, video_id, ('record', 'currentRecord', 'data'))
+
+        return {
+            **self._extract_common_meta(record_info),
+            'id': video_id,
+            'formats': self._extract_formats(record_info, video_id),
+        }
+
+
+class VKPlayLiveIE(VKPlayBaseIE):
+    _VALID_URL = r'https?://vkplay\.live/(?P<id>[^/]+)/?(?:[#?]|$)'
+    _TESTS = [{
+        'url': 'https://vkplay.live/bayda',
+        'info_dict': {
+            'id': 'f02c321e-427b-408d-b12f-ae34e53e0ea2',
+            'ext': 'mp4',
+            'title': r're:эскапизм крута .*',
+            'uploader': 'Bayda',
+            'uploader_id': 12279401,
+            'release_timestamp': 1687209962,
+            'release_date': '20230619',
+            'thumbnail': r're:https://images.vkplay.live/public_video_stream/12279401/preview\?change_time=\d+',
+            'view_count': int,
+            'concurrent_view_count': int,
+            'like_count': int,
+            'categories': ['EVE Online'],
+            'live_status': 'is_live',
+        },
+        'skip': 'livestream',
+        'params': {'skip_download': True},
+    }]
+
+    def _real_extract(self, url):
+        username = self._match_id(url)
+
+        stream_info = self._download_json(
+            f'https://api.vkplay.live/v1/blog/{username}/public_video_stream', username, fatal=False)
+        if not stream_info:
+            stream_info = self._extract_from_initial_state(url, username, ('stream', 'stream', 'data', 'stream'))
+
+        formats = self._extract_formats(stream_info, username)
+        if not formats and not traverse_obj(stream_info, ('isOnline', {bool})):
+            raise UserNotLive(video_id=username)
+
+        return {
+            **self._extract_common_meta(stream_info),
+            'formats': formats,
+        }

From 4dc4d8473c085900edc841c87c20041233d25b1f Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Mon, 3 Jul 2023 10:47:10 +0000
Subject: [PATCH 417/871] [extractor/youtube] Ignore incomplete data for
 comment threads by default (#7475)

For both `--ignore-errors` and `--ignore-errors only_download`. Pass `--no-ignore-errors` to not ignore.

Closes https://github.com/yt-dlp/yt-dlp/issues/7474

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 967914c0f..2c64f8e84 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3426,7 +3426,9 @@ def extract_thread(contents):
                         # Pinned comments may appear a second time in newest first sort
                         # See: https://github.com/yt-dlp/yt-dlp/issues/6712
                         continue
-                    self.report_warning('Detected YouTube comments looping. Stopping comment extraction as we probably cannot get any more.')
+                    self.report_warning(
+                        'Detected YouTube comments looping. Stopping comment extraction '
+                        f'{"for this thread" if parent else ""} as we probably cannot get any more.')
                     yield
                 else:
                     tracker['seen_comment_ids'].add(comment['id'])
@@ -3517,12 +3519,18 @@ def extract_thread(contents):
                 # Ignore incomplete data error for replies if retries didn't work.
                 # This is to allow any other parent comments and comment threads to be downloaded.
                 # See: https://github.com/yt-dlp/yt-dlp/issues/4669
-                if 'incomplete data' in str(e).lower() and parent and self.get_param('ignoreerrors') is True:
-                    self.report_warning(
-                        'Received incomplete data for a comment reply thread and retrying did not help. '
-                        'Ignoring to let other comments be downloaded.')
-                else:
-                    raise
+                if 'incomplete data' in str(e).lower() and parent:
+                    if self.get_param('ignoreerrors') in (True, 'only_download'):
+                        self.report_warning(
+                            'Received incomplete data for a comment reply thread and retrying did not help. '
+                            'Ignoring to let other comments be downloaded. Pass --no-ignore-errors to not ignore.')
+                        return
+                    else:
+                        raise ExtractorError(
+                            'Incomplete data received for comment reply thread. '
+                            'Pass --ignore-errors to ignore and allow rest of comments to download.',
+                            expected=True)
+                raise
             is_forced_continuation = False
             continuation = None
             for continuation_items in traverse_obj(response, continuation_items_path, expected_type=list, default=[]):

From 3b7f5300c577fef40464d46d4e4037a69d51fe82 Mon Sep 17 00:00:00 2001
From: RfadnjdExt <40250666+RfadnjdExt@users.noreply.github.com>
Date: Wed, 5 Jul 2023 09:17:13 +0700
Subject: [PATCH 418/871] [extractor/googledrive] Fix source format extraction
 (#7395)

Closes #7344
Authored by: RfadnjdExt
---
 yt_dlp/extractor/googledrive.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/yt_dlp/extractor/googledrive.py b/yt_dlp/extractor/googledrive.py
index 9e2ccde00..8a4cd1690 100644
--- a/yt_dlp/extractor/googledrive.py
+++ b/yt_dlp/extractor/googledrive.py
@@ -5,7 +5,9 @@
 from ..utils import (
     ExtractorError,
     determine_ext,
+    extract_attributes,
     get_element_by_class,
+    get_element_html_by_id,
     int_or_none,
     lowercase_escape,
     try_get,
@@ -34,6 +36,7 @@ class GoogleDriveIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Big Buck Bunny.mp4',
             'duration': 45,
+            'thumbnail': 'https://drive.google.com/thumbnail?id=0ByeS4oOUV-49Zzh4R1J6R09zazQ',
         }
     }, {
         # video can't be watched anonymously due to view count limit reached,
@@ -207,10 +210,10 @@ def get_value(key):
                 'export': 'download',
             })
 
-        def request_source_file(source_url, kind):
+        def request_source_file(source_url, kind, data=None):
             return self._request_webpage(
                 source_url, video_id, note='Requesting %s file' % kind,
-                errnote='Unable to request %s file' % kind, fatal=False)
+                errnote='Unable to request %s file' % kind, fatal=False, data=data)
         urlh = request_source_file(source_url, 'source')
         if urlh:
             def add_source_format(urlh):
@@ -237,14 +240,10 @@ def add_source_format(urlh):
                     urlh, url, video_id, note='Downloading confirmation page',
                     errnote='Unable to confirm download', fatal=False)
                 if confirmation_webpage:
-                    confirm = self._search_regex(
-                        r'confirm=([^&"\']+)', confirmation_webpage,
-                        'confirmation code', default=None)
-                    if confirm:
-                        confirmed_source_url = update_url_query(source_url, {
-                            'confirm': confirm,
-                        })
-                        urlh = request_source_file(confirmed_source_url, 'confirmed source')
+                    confirmed_source_url = extract_attributes(
+                        get_element_html_by_id('download-form', confirmation_webpage) or '').get('action')
+                    if confirmed_source_url:
+                        urlh = request_source_file(confirmed_source_url, 'confirmed source', data=b'')
                         if urlh and urlh.headers.get('Content-Disposition'):
                             add_source_format(urlh)
                     else:

From 1cffd621cb371f1563563cfb2fe37d137e8a7bee Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 4 Jul 2023 22:05:52 -0500
Subject: [PATCH 419/871] [extractor/twitter:spaces] Fix extraction (#7512)

Closes #7455
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 38 +++++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index f854d9c4a..1fb9524da 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -889,8 +889,10 @@ class TwitterIE(TwitterBaseIE):
             'uploader_id': 'MoniqueCamarra',
             'live_status': 'was_live',
             'release_timestamp': 1658417414,
-            'description': 'md5:acce559345fd49f129c20dbcda3f1201',
-            'timestamp': 1658407771464,
+            'description': 'md5:4dc8e972f1d8b3c6580376fabb02a3ad',
+            'timestamp': 1658407771,
+            'release_date': '20220721',
+            'upload_date': '20220721',
         },
         'add_ie': ['TwitterSpaces'],
         'params': {'skip_download': 'm3u8'},
@@ -1436,7 +1438,10 @@ class TwitterSpacesIE(TwitterBaseIE):
             'uploader': r're:Lucio Di Gaetano.*?',
             'uploader_id': 'luciodigaetano',
             'live_status': 'was_live',
-            'timestamp': 1659877956397,
+            'timestamp': 1659877956,
+            'upload_date': '20220807',
+            'release_timestamp': 1659904215,
+            'release_date': '20220807',
         },
         'params': {'skip_download': 'm3u8'},
     }]
@@ -1482,26 +1487,31 @@ def _real_extract(self, url):
 
         metadata = space_data['metadata']
         live_status = try_call(lambda: self.SPACE_STATUS[metadata['state'].lower()])
+        is_live = live_status == 'is_live'
 
         formats = []
         if live_status == 'is_upcoming':
             self.raise_no_formats('Twitter Space not started yet', expected=True)
-        elif live_status == 'post_live':
-            self.raise_no_formats('Twitter Space ended but not downloadable yet', expected=True)
-        else:
-            source = self._call_api(
-                f'live_video_stream/status/{metadata["media_key"]}', metadata['media_key'])['source']
-
-            # XXX: Native downloader does not work
+        elif not is_live and not metadata.get('is_space_available_for_replay'):
+            self.raise_no_formats('Twitter Space ended and replay is disabled', expected=True)
+        elif metadata.get('media_key'):
+            source = traverse_obj(
+                self._call_api(f'live_video_stream/status/{metadata["media_key"]}', metadata['media_key']),
+                ('source', ('noRedirectPlaybackUrl', 'location'), {url_or_none}), get_all=False)
             formats = self._extract_m3u8_formats(
-                traverse_obj(source, 'noRedirectPlaybackUrl', 'location'),
-                metadata['media_key'], 'm4a', 'm3u8', live=live_status == 'is_live',
-                headers={'Referer': 'https://twitter.com/'})
+                source, metadata['media_key'], 'm4a', live=is_live, fatal=False,
+                headers={'Referer': 'https://twitter.com/'}) if source else []
             for fmt in formats:
                 fmt.update({'vcodec': 'none', 'acodec': 'aac'})
+                if not is_live:
+                    fmt['container'] = 'm4a_dash'
 
         participants = ', '.join(traverse_obj(
             space_data, ('participants', 'speakers', ..., 'display_name'))) or 'nobody yet'
+
+        if not formats and live_status == 'post_live':
+            self.raise_no_formats('Twitter Space ended but not downloadable yet', expected=True)
+
         return {
             'id': space_id,
             'title': metadata.get('title'),
@@ -1513,7 +1523,7 @@ def _real_extract(self, url):
             'live_status': live_status,
             'release_timestamp': try_call(
                 lambda: int_or_none(metadata['scheduled_start'], scale=1000)),
-            'timestamp': metadata.get('created_at'),
+            'timestamp': int_or_none(metadata.get('created_at'), scale=1000),
             'formats': formats,
         }
 

From 49296437a8e5fa91dacb5446e51ab588474c85d3 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Wed, 5 Jul 2023 11:27:36 -0500
Subject: [PATCH 420/871] [extractor/twitter] Fix unauthenticated extraction
 (#7476)

Closes #7473
Authored by: bashonly
---
 README.md                   |   3 -
 yt_dlp/extractor/twitter.py | 180 ++++++++++++++----------------------
 2 files changed, 70 insertions(+), 113 deletions(-)

diff --git a/README.md b/README.md
index 066ff9052..4fb3e450d 100644
--- a/README.md
+++ b/README.md
@@ -1852,9 +1852,6 @@ #### tiktok
 #### rokfinchannel
 * `tab`: Which tab to download - one of `new`, `top`, `videos`, `podcasts`, `streams`, `stacks`
 
-#### twitter
-* `legacy_api`: Force usage of the legacy Twitter API instead of the GraphQL API for tweet extraction. Has no effect if login cookies are passed
-
 #### stacommu, wrestleuniverse
 * `device_id`: UUID value assigned by the website and used to enforce device limits for paid livestream content. Can be found in browser local storage
 
diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 1fb9524da..eaf9be526 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1,5 +1,6 @@
 import json
 import re
+import urllib.error
 
 from .common import InfoExtractor
 from .periscope import PeriscopeBaseIE, PeriscopeIE
@@ -34,7 +35,6 @@ class TwitterBaseIE(InfoExtractor):
     _GRAPHQL_API_BASE = 'https://twitter.com/i/api/graphql/'
     _BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?(?:twitter\.com|twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid\.onion)/'
     _AUTH = {'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'}
-    _guest_token = None
     _flow_token = None
 
     _LOGIN_INIT_DATA = json.dumps({
@@ -145,14 +145,6 @@ def _search_dimensions_in_video_url(a_format, video_url):
     def is_logged_in(self):
         return bool(self._get_cookies(self._API_BASE).get('auth_token'))
 
-    def _fetch_guest_token(self, headers, display_id):
-        headers.pop('x-guest-token', None)
-        self._guest_token = traverse_obj(self._download_json(
-            f'{self._API_BASE}guest/activate.json', display_id,
-            'Downloading guest token', data=b'', headers=headers), 'guest_token')
-        if not self._guest_token:
-            raise ExtractorError('Could not retrieve guest token')
-
     def _set_base_headers(self):
         headers = self._AUTH.copy()
         csrf_token = try_call(lambda: self._get_cookies(self._API_BASE)['ct0'].value)
@@ -183,12 +175,15 @@ def _perform_login(self, username, password):
         if self.is_logged_in:
             return
 
-        self._request_webpage('https://twitter.com/', None, 'Requesting cookies')
+        webpage = self._download_webpage('https://twitter.com/', None, 'Downloading login page')
         headers = self._set_base_headers()
-        self._fetch_guest_token(headers, None)
+        guest_token = self._search_regex(
+            r'\.cookie\s*=\s*["\']gt=(\d+);', webpage, 'gt', default=None) or self._download_json(
+            f'{self._API_BASE}guest/activate.json', None, 'Downloading guest token',
+            data=b'', headers=headers)['guest_token']
         headers.update({
             'content-type': 'application/json',
-            'x-guest-token': self._guest_token,
+            'x-guest-token': guest_token,
             'x-twitter-client-language': 'en',
             'x-twitter-active-user': 'yes',
             'Referer': 'https://twitter.com/',
@@ -285,37 +280,24 @@ def input_dict(subtask_id, text):
         self.report_login()
 
     def _call_api(self, path, video_id, query={}, graphql=False):
-        headers = self._set_base_headers()
-        if self.is_logged_in:
-            headers.update({
+        if not self.is_logged_in:
+            self.raise_login_required()
+
+        result = self._download_json(
+            (self._GRAPHQL_API_BASE if graphql else self._API_BASE) + path, video_id,
+            f'Downloading {"GraphQL" if graphql else "legacy API"} JSON', headers={
+                **self._set_base_headers(),
                 'x-twitter-auth-type': 'OAuth2Session',
                 'x-twitter-client-language': 'en',
                 'x-twitter-active-user': 'yes',
-            })
+            }, query=query, expected_status={400, 401, 403, 404} if graphql else {403})
 
-        for first_attempt in (True, False):
-            if not self.is_logged_in:
-                if not self._guest_token:
-                    self._fetch_guest_token(headers, video_id)
-                headers['x-guest-token'] = self._guest_token
+        if result.get('errors'):
+            errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
+            raise ExtractorError(
+                f'Error(s) while querying API: {errors or "Unknown error"}', expected=True)
 
-            allowed_status = {400, 401, 403, 404} if graphql else {403}
-            result = self._download_json(
-                (self._GRAPHQL_API_BASE if graphql else self._API_BASE) + path,
-                video_id, headers=headers, query=query, expected_status=allowed_status,
-                note=f'Downloading {"GraphQL" if graphql else "legacy API"} JSON')
-
-            if result.get('errors'):
-                errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
-                if not self.is_logged_in and first_attempt and 'bad guest token' in errors.lower():
-                    self.to_screen('Guest token has expired. Refreshing guest token')
-                    self._guest_token = None
-                    continue
-
-                raise ExtractorError(
-                    f'Error(s) while querying API: {errors or "Unknown error"}', expected=True)
-
-            return result
+        return result
 
     def _build_graphql_query(self, media_id):
         raise NotImplementedError('Method must be implemented to support GraphQL')
@@ -457,6 +439,7 @@ class TwitterIE(TwitterBaseIE):
     _VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)(?:/(?:video|photo)/(?P<index>\d+))?'
 
     _TESTS = [{
+        # comment_count, repost_count, view_count are only available with auth (applies to all tests)
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
         'info_dict': {
             'id': '643211870443208704',
@@ -471,10 +454,7 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1442188653,
             'upload_date': '20150913',
             'uploader_url': 'https://twitter.com/freethenipple',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': [],
             'age_limit': 18,
         },
@@ -505,8 +485,6 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1447395772,
             'upload_date': '20151113',
             'uploader_url': 'https://twitter.com/starwars',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
             'tags': ['TV', 'StarWars', 'TheForceAwakens'],
             'age_limit': 0,
@@ -550,10 +528,7 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1455777459,
             'upload_date': '20160218',
             'uploader_url': 'https://twitter.com/jaydingeer',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': ['Damndaniel'],
             'age_limit': 0,
         },
@@ -591,10 +566,7 @@ class TwitterIE(TwitterBaseIE):
             'upload_date': '20160412',
             'uploader_url': 'https://twitter.com/CaptainAmerica',
             'thumbnail': r're:^https?://.*\.jpg',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -641,10 +613,7 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1505803395,
             'upload_date': '20170919',
             'uploader_url': 'https://twitter.com/Prefet971',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': ['Maria'],
             'age_limit': 0,
         },
@@ -667,10 +636,7 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1527623489,
             'upload_date': '20180529',
             'uploader_url': 'https://twitter.com/LisPower1',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -692,10 +658,7 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1548184644,
             'upload_date': '20190122',
             'uploader_url': 'https://twitter.com/Twitter',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -713,6 +676,7 @@ class TwitterIE(TwitterBaseIE):
             'view_count': int,
         },
         'add_ie': ['TwitterBroadcast'],
+        'skip': 'Requires authentication',
     }, {
         # unified card
         'url': 'https://twitter.com/BrooklynNets/status/1349794411333394432?s=20',
@@ -729,8 +693,6 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1610651040,
             'upload_date': '20210114',
             'uploader_url': 'https://twitter.com/BrooklynNets',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
             'tags': [],
             'age_limit': 0,
@@ -753,10 +715,7 @@ class TwitterIE(TwitterBaseIE):
             'thumbnail': r're:^https?://.*\.jpg',
             'duration': 30.03,
             'timestamp': 1665025050,
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -765,15 +724,13 @@ class TwitterIE(TwitterBaseIE):
         'url': 'https://twitter.com/UltimaShadowX/status/1577719286659006464',
         'info_dict': {
             'id': '1577719286659006464',
-            'title': 'Ultima | #\u0432\u029f\u043c - Test',
+            'title': 'Ultima📛 | #вʟм - Test',
             'description': 'Test https://t.co/Y3KEZD7Dad',
-            'uploader': 'Ultima | #\u0432\u029f\u043c',
+            'uploader': 'Ultima📛 | #вʟм',
             'uploader_id': 'UltimaShadowX',
             'uploader_url': 'https://twitter.com/UltimaShadowX',
             'upload_date': '20221005',
             'timestamp': 1664992565,
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
             'tags': [],
             'age_limit': 0,
@@ -795,10 +752,7 @@ class TwitterIE(TwitterBaseIE):
             'duration': 21.321,
             'timestamp': 1664477766,
             'upload_date': '20220929',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': ['HurricaneIan'],
             'age_limit': 0,
         },
@@ -825,6 +779,20 @@ class TwitterIE(TwitterBaseIE):
         },
         'skip': 'Requires authentication',
     }, {
+        # Single Vimeo video result without auth
+        'url': 'https://twitter.com/Srirachachau/status/1395079556562706435',
+        'info_dict': {
+            'id': '551578322',
+            'ext': 'mp4',
+            'title': 'Dusty & The Mayor',
+            'uploader': 'Michael Chau',
+            'uploader_id': 'user29061007',
+            'uploader_url': 'https://vimeo.com/user29061007',
+            'duration': 478,
+            'thumbnail': 'https://i.vimeocdn.com/video/1139658575-0dfdce6e9a2401fe09feb24bf0d14e6f24a53c12f447ff688ace61009ad4c1ba-d_1280',
+        },
+    }, {
+        # Playlist result only with auth
         'url': 'https://twitter.com/Srirachachau/status/1395079556562706435',
         'playlist_mincount': 2,
         'info_dict': {
@@ -842,6 +810,7 @@ class TwitterIE(TwitterBaseIE):
             'uploader_url': 'https://twitter.com/Srirachachau',
             'timestamp': 1621447860,
         },
+        'skip': 'Requires authentication',
     }, {
         'url': 'https://twitter.com/DavidToons_/status/1578353380363501568',
         'playlist_mincount': 2,
@@ -860,6 +829,7 @@ class TwitterIE(TwitterBaseIE):
             'upload_date': '20221007',
             'age_limit': 0,
         },
+        'skip': 'Requires authentication',
     }, {
         'url': 'https://twitter.com/primevideouk/status/1578401165338976258',
         'playlist_count': 2,
@@ -873,8 +843,6 @@ class TwitterIE(TwitterBaseIE):
             'upload_date': '20221007',
             'age_limit': 0,
             'uploader_url': 'https://twitter.com/primevideouk',
-            'comment_count': int,
-            'repost_count': int,
             'like_count': int,
             'tags': ['TheRingsOfPower'],
         },
@@ -896,6 +864,7 @@ class TwitterIE(TwitterBaseIE):
         },
         'add_ie': ['TwitterSpaces'],
         'params': {'skip_download': 'm3u8'},
+        'skip': 'Requires authentication',
     }, {
         # URL specifies video number but --yes-playlist
         'url': 'https://twitter.com/CTVJLaidlaw/status/1600649710662213632/video/1',
@@ -905,9 +874,7 @@ class TwitterIE(TwitterBaseIE):
             'title': 'md5:be05989b0722e114103ed3851a0ffae2',
             'timestamp': 1670459604.0,
             'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
-            'comment_count': int,
             'uploader_id': 'CTVJLaidlaw',
-            'repost_count': int,
             'tags': ['colorectalcancer', 'cancerjourney', 'imnotaquitter'],
             'upload_date': '20221208',
             'age_limit': 0,
@@ -926,14 +893,11 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1670459604.0,
             'uploader_id': 'CTVJLaidlaw',
             'uploader': 'Jocelyn Laidlaw',
-            'repost_count': int,
-            'comment_count': int,
             'tags': ['colorectalcancer', 'cancerjourney', 'imnotaquitter'],
             'duration': 102.226,
             'uploader_url': 'https://twitter.com/CTVJLaidlaw',
             'display_id': '1600649710662213632',
             'like_count': int,
-            'view_count': int,
             'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
             'upload_date': '20221208',
             'age_limit': 0,
@@ -959,9 +923,6 @@ class TwitterIE(TwitterBaseIE):
             'age_limit': 18,
             'tags': [],
             'like_count': int,
-            'repost_count': int,
-            'comment_count': int,
-            'view_count': int,
         },
     }, {
         'url': 'https://twitter.com/hlo_again/status/1599108751385972737/video/2',
@@ -974,10 +935,7 @@ class TwitterIE(TwitterBaseIE):
             'like_count': int,
             'uploader_id': 'hlo_again',
             'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1599108643743473680/pu/img/UG3xjov4rgg5sbYM.jpg?name=orig',
-            'repost_count': int,
             'duration': 9.531,
-            'comment_count': int,
-            'view_count': int,
             'upload_date': '20221203',
             'age_limit': 0,
             'timestamp': 1670092210.0,
@@ -994,14 +952,11 @@ class TwitterIE(TwitterBaseIE):
             'ext': 'mp4',
             'uploader_url': 'https://twitter.com/MunTheShinobi',
             'description': 'This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525 https://t.co/cNsA0MoOml',
-            'view_count': int,
             'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1600009362759733248/pu/img/XVhFQivj75H_YxxV.jpg?name=orig',
             'age_limit': 0,
             'uploader': 'Mün The Shinobi',
-            'repost_count': int,
             'upload_date': '20221206',
             'title': 'Mün The Shinobi - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
-            'comment_count': int,
             'like_count': int,
             'tags': [],
             'uploader_id': 'MunTheShinobi',
@@ -1009,14 +964,14 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1670306984.0,
         },
     }, {
-        # url to retweet id, legacy API
+        # url to retweet id
         'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
         'info_dict': {
             'id': '1623274794488659969',
             'display_id': '1623739803874349067',
             'ext': 'mp4',
             'title': 'Johnny Bullets - Me after going viral to over 30million people:    Whoopsie-daisy',
-            'description': 'md5:e873616a4a8fe0f93e71872678a672f3',
+            'description': 'md5:224d62f54b0cdef8e33d4c56c41ac503',
             'uploader': 'Johnny Bullets',
             'uploader_id': 'Johnnybull3ts',
             'uploader_url': 'https://twitter.com/Johnnybull3ts',
@@ -1027,10 +982,7 @@ class TwitterIE(TwitterBaseIE):
             'upload_date': '20230208',
             'thumbnail': r're:https://pbs\.twimg\.com/ext_tw_video_thumb/.+',
             'like_count': int,
-            'repost_count': int,
-            'comment_count': int,
         },
-        'params': {'extractor_args': {'twitter': {'legacy_api': ['']}}},
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -1081,8 +1033,6 @@ def _graphql_to_legacy(self, data, twid):
 
         if 'tombstone' in result:
             cause = remove_end(traverse_obj(result, ('tombstone', 'text', 'text', {str})), '. Learn more')
-            if cause and 'adult content' in cause:
-                self.raise_login_required(cause)
             raise ExtractorError(f'Twitter API says: {cause or "Unknown error"}', expected=True)
 
         status = result.get('legacy', {})
@@ -1138,19 +1088,22 @@ def _build_graphql_query(self, media_id):
 
     def _real_extract(self, url):
         twid, selected_index = self._match_valid_url(url).group('id', 'index')
-        if self._configuration_arg('legacy_api') and not self.is_logged_in:
-            status = traverse_obj(self._call_api(f'statuses/show/{twid}.json', twid, {
-                'cards_platform': 'Web-12',
-                'include_cards': 1,
-                'include_reply_count': 1,
-                'include_user_entities': 0,
-                'tweet_mode': 'extended',
-            }), 'retweeted_status', None)
+        if not self.is_logged_in:
+            try:
+                status = self._download_json(
+                    'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
+                    headers={'User-Agent': 'Googlebot'}, query={'id': twid})
+                self.to_screen(f'Some metadata is missing without authentication. {self._login_hint()}')
+            except ExtractorError as e:
+                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+                    self.raise_login_required('Requested tweet may only be available when logged in')
+                raise
         else:
-            result = self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid)
-            status = self._graphql_to_legacy(result, twid)
+            status = self._graphql_to_legacy(
+                self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid), twid)
 
-        title = description = status['full_text'].replace('\n', ' ')
+        title = description = traverse_obj(
+            status, (('full_text', 'text'), {lambda x: x.replace('\n', ' ')}), get_all=False) or ''
         # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
         title = re.sub(r'\s+(https?://[^ ]+)', '', title)
         user = status.get('user') or {}
@@ -1176,12 +1129,16 @@ def _real_extract(self, url):
 
         def extract_from_video_info(media):
             media_id = traverse_obj(media, 'id_str', 'id', expected_type=str_or_none)
+            if not media_id:
+                # workaround for non-authenticated responses
+                media_id = traverse_obj(media, (
+                    'video_info', 'variants', ..., 'url',
+                    {lambda x: re.search(r'_video/(\d+)/', x)[1]}), get_all=False)
             self.write_debug(f'Extracting from video info: {media_id}')
-            video_info = media.get('video_info') or {}
 
             formats = []
             subtitles = {}
-            for variant in video_info.get('variants', []):
+            for variant in traverse_obj(media, ('video_info', 'variants', ...)):
                 fmts, subs = self._extract_variant_formats(variant, twid)
                 subtitles = self._merge_subtitles(subtitles, subs)
                 formats.extend(fmts)
@@ -1201,12 +1158,12 @@ def add_thumbnail(name, size):
                 add_thumbnail('orig', media.get('original_info') or {})
 
             return {
-                'id': media_id,
+                'id': media_id or twid,
                 'formats': formats,
                 'subtitles': subtitles,
                 'thumbnails': thumbnails,
                 'view_count': traverse_obj(media, ('mediaStats', 'viewCount', {int_or_none})),
-                'duration': float_or_none(video_info.get('duration_millis'), 1000),
+                'duration': float_or_none(traverse_obj(media, ('video_info', 'duration_millis')), 1000),
                 # The codec of http formats are unknown
                 '_format_sort_fields': ('res', 'br', 'size', 'proto'),
             }
@@ -1286,12 +1243,15 @@ def get_binding_value(k):
                 }
 
         videos = traverse_obj(status, (
-            (None, 'quoted_status'), 'extended_entities', 'media', lambda _, m: m['type'] != 'photo', {dict}))
+            ('mediaDetails', ((None, 'quoted_status'), 'extended_entities', 'media')),
+            lambda _, m: m['type'] != 'photo', {dict}))
 
         if self._yes_playlist(twid, selected_index, video_label='URL-specified video number'):
             selected_entries = (*map(extract_from_video_info, videos), *extract_from_card_info(status.get('card')))
         else:
-            desired_obj = traverse_obj(status, ('extended_entities', 'media', int(selected_index) - 1, {dict}))
+            desired_obj = traverse_obj(status, (
+                ('mediaDetails', ((None, 'quoted_status'), 'extended_entities', 'media')),
+                int(selected_index) - 1, {dict}), get_all=False)
             if not desired_obj:
                 raise ExtractorError(f'Video #{selected_index} is unavailable', expected=True)
             elif desired_obj.get('type') != 'video':

From 90db9a3c00ca80492c6a58c542e4cbf4c2710866 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Fri, 7 Jul 2023 01:32:41 +1200
Subject: [PATCH 421/871] [extractor/youtube:stories] Remove (#7459)

YouTube killed them

https://web.archive.org/web/20230630153050/https://support.google.com/youtube/thread/217640760
---
 yt_dlp/extractor/_extractors.py |  1 -
 yt_dlp/extractor/youtube.py     | 43 +--------------------------------
 2 files changed, 1 insertion(+), 43 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 6f1873383..c0a330dbe 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -15,7 +15,6 @@
     YoutubeSearchURLIE,
     YoutubeMusicSearchURLIE,
     YoutubeSubscriptionsIE,
-    YoutubeStoriesIE,
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeYtBeIE,
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 2c64f8e84..552ca099c 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -2499,29 +2499,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': '@abaointokyo',
             },
             'params': {'skip_download': True}
-        }, {
-            # Story. Requires specific player params to work.
-            'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
-            'info_dict': {
-                'id': 'vv8qTUWmulI',
-                'ext': 'mp4',
-                'availability': 'unlisted',
-                'view_count': int,
-                'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
-                'upload_date': '20220526',
-                'categories': ['Education'],
-                'title': 'Story',
-                'channel': 'IT\'S HISTORY',
-                'description': '',
-                'duration': 12,
-                'playable_in_embed': True,
-                'age_limit': 0,
-                'live_status': 'not_live',
-                'tags': [],
-                'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
-                'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
-            },
-            'skip': 'stories get removed after some period of time',
         }, {
             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
             'info_dict': {
@@ -3620,7 +3597,7 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
         yt_query = {
             'videoId': video_id,
         }
-        if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
+        if _split_innertube_client(client)[0] == 'android':
             yt_query['params'] = self._PLAYER_PARAMS
 
         yt_query.update(self._generate_player_context(sts))
@@ -4033,8 +4010,6 @@ def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
         webpage = None
         if 'webpage' not in self._configuration_arg('player_skip'):
             query = {'bpctr': '9999999999', 'has_verified': '1'}
-            if smuggled_data.get('is_story'):  # XXX: Deprecated
-                query['pp'] = self._PLAYER_PARAMS
             webpage = self._download_webpage(
                 webpage_url, video_id, fatal=False, query=query)
 
@@ -7145,22 +7120,6 @@ class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
     }]
 
 
-class YoutubeStoriesIE(InfoExtractor):
-    IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
-    IE_NAME = 'youtube:stories'
-    _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
-    _TESTS = [{
-        'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        playlist_id = f'RLTD{self._match_id(url)}'
-        return self.url_result(
-            smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
-            ie=YoutubeTabIE, video_id=playlist_id)
-
-
 class YoutubeShortsAudioPivotIE(InfoExtractor):
     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
     IE_NAME = 'youtube:shorts:pivot:audio'

From 6355b5f1e1e8e7f4ef866d71d51e03baf0e82f17 Mon Sep 17 00:00:00 2001
From: Jorge <46056498+jorgectf@users.noreply.github.com>
Date: Thu, 6 Jul 2023 16:51:46 +0200
Subject: [PATCH 422/871] [misc] Add CodeQL workflow (#7497)

---
 .github/workflows/codeql.yml | 65 ++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)
 create mode 100644 .github/workflows/codeql.yml

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 000000000..2821d90d0
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,65 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ 'master', 'gh-pages', 'release' ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ 'master' ]
+  schedule:
+    - cron: '59 11 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

From 662ef1e910b72e57957f06589925b2332ba52821 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 4 Jul 2023 18:46:32 +0530
Subject: [PATCH 423/871] [downloader/http] Avoid infinite loop when no data is
 received

Closes #7504
---
 yt_dlp/downloader/http.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index e785f0d4e..7c5daea85 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -339,15 +339,15 @@ def retry(e):
                 elif speed:
                     ctx.throttle_start = None
 
-            if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
-                ctx.resume_len = byte_counter
-                # ctx.block_size = block_size
-                raise NextFragment()
-
             if ctx.stream is None:
                 self.to_stderr('\n')
                 self.report_error('Did not get any data blocks')
                 return False
+
+            if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
+                ctx.resume_len = byte_counter
+                raise NextFragment()
+
             if ctx.tmpfilename != '-':
                 ctx.stream.close()
 

From 47bcd437247152e0af5b3ebc5592db7bb66855c2 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 6 Jul 2023 18:08:44 +0530
Subject: [PATCH 424/871] [outtmpl] Pad `playlist_index` etc even when with
 internal formatting

Closes #7501
---
 test/test_YoutubeDL.py |  2 +-
 yt_dlp/YoutubeDL.py    | 13 ++++++-------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index f495fa6d9..3fbcdd01f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -684,7 +684,7 @@ def test(tmpl, expected, *, info=None, **params):
         test('%(id)s.%(ext)s', '1234.mp4')
         test('%(duration_string)s', ('27:46:40', '27-46-40'))
         test('%(resolution)s', '1080p')
-        test('%(playlist_index)s', '001')
+        test('%(playlist_index|)s', '001')
         test('%(playlist_autonumber)s', '02')
         test('%(autonumber)s', '00001')
         test('%(autonumber+2)03d', '005', autonumber_start=3)
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 6dade0b2a..d4aff0743 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1271,21 +1271,20 @@ def create_key(outer_mobj):
                 return outer_mobj.group(0)
             key = outer_mobj.group('key')
             mobj = re.match(INTERNAL_FORMAT_RE, key)
-            initial_field = mobj.group('fields') if mobj else ''
-            value, replacement, default = None, None, na
+            value, replacement, default, last_field = None, None, na, ''
             while mobj:
                 mobj = mobj.groupdict()
                 default = mobj['default'] if mobj['default'] is not None else default
                 value = get_value(mobj)
-                replacement = mobj['replacement']
+                last_field, replacement = mobj['fields'], mobj['replacement']
                 if value is None and mobj['alternate']:
                     mobj = re.match(INTERNAL_FORMAT_RE, mobj['remaining'][1:])
                 else:
                     break
 
             fmt = outer_mobj.group('format')
-            if fmt == 's' and value is not None and key in field_size_compat_map.keys():
-                fmt = f'0{field_size_compat_map[key]:d}d'
+            if fmt == 's' and value is not None and last_field in field_size_compat_map.keys():
+                fmt = f'0{field_size_compat_map[last_field]:d}d'
 
             if None not in (value, replacement):
                 try:
@@ -1322,7 +1321,7 @@ def create_key(outer_mobj):
                 value = format_decimal_suffix(value, f'%{num_fmt}f%s' if num_fmt else '%d%s',
                                               factor=1024 if '#' in flags else 1000)
             elif fmt[-1] == 'S':  # filename sanitization
-                value, fmt = filename_sanitizer(initial_field, value, restricted='#' in flags), str_fmt
+                value, fmt = filename_sanitizer(last_field, value, restricted='#' in flags), str_fmt
             elif fmt[-1] == 'c':
                 if value:
                     value = str(value)[0]
@@ -1341,7 +1340,7 @@ def create_key(outer_mobj):
                 elif fmt[-1] == 'a':
                     value, fmt = ascii(value), str_fmt
                 if fmt[-1] in 'csra':
-                    value = sanitizer(initial_field, value)
+                    value = sanitizer(last_field, value)
 
             key = '%s\0%s' % (key.replace('%', '%\0'), outer_mobj.group('format'))
             TMPL_DICT[key] = value

From fa44802809d189fca0f4782263d48d6533384503 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 6 Jul 2023 17:34:51 +0530
Subject: [PATCH 425/871] [devscripts/make_changelog] Skip reverted commits

---
 devscripts/make_changelog.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index 0bcfa6ae7..eb0e3082f 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -252,6 +252,7 @@ class CommitRange:
         (?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
         ''', re.VERBOSE | re.DOTALL)
     EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE)
+    REVERT_RE = re.compile(r'(?i:Revert)\s+([\da-f]{40})')
     FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})')
     UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)')
 
@@ -279,7 +280,7 @@ def _get_commits_and_fixes(self, default_author):
             self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}',
             f'{self._start}..{self._end}' if self._start else self._end).stdout
 
-        commits = {}
+        commits, reverts = {}, {}
         fixes = defaultdict(list)
         lines = iter(result.splitlines(False))
         for i, commit_hash in enumerate(lines):
@@ -300,6 +301,11 @@ def _get_commits_and_fixes(self, default_author):
                 logger.debug(f'Reached Release commit, breaking: {commit}')
                 break
 
+            revert_match = self.REVERT_RE.fullmatch(commit.short)
+            if revert_match:
+                reverts[revert_match.group(1)] = commit
+                continue
+
             fix_match = self.FIXES_RE.search(commit.short)
             if fix_match:
                 commitish = fix_match.group(1)
@@ -307,6 +313,13 @@ def _get_commits_and_fixes(self, default_author):
 
             commits[commit.hash] = commit
 
+        for commitish, revert_commit in reverts.items():
+            reverted = commits.pop(commitish, None)
+            if reverted:
+                logger.debug(f'{commit} fully reverted {reverted}')
+            else:
+                commits[revert_commit.hash] = revert_commit
+
         for commitish, fix_commits in fixes.items():
             if commitish in commits:
                 hashes = ', '.join(commit.hash[:HASH_LENGTH] for commit in fix_commits)

From 337734d4a8a6500bc65434843db346b5cbd05e81 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 6 Jul 2023 20:09:42 +0530
Subject: [PATCH 426/871] [cleanup] Misc

---
 devscripts/make_changelog.py  | 7 ++++---
 setup.cfg                     | 1 -
 yt_dlp/YoutubeDL.py           | 2 +-
 yt_dlp/downloader/common.py   | 3 ++-
 yt_dlp/downloader/fragment.py | 4 +---
 yt_dlp/extractor/adobepass.py | 2 +-
 yt_dlp/extractor/iqiyi.py     | 2 +-
 yt_dlp/extractor/vshare.py    | 2 +-
 yt_dlp/extractor/youtube.py   | 2 +-
 yt_dlp/utils/__init__.py      | 1 +
 10 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index eb0e3082f..3ad4c5408 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -55,6 +55,7 @@ def commit_lookup(cls):
                     'dependencies',
                     'jsinterp',
                     'outtmpl',
+                    'formats',
                     'plugins',
                     'update',
                     'upstream',
@@ -68,9 +69,9 @@ def commit_lookup(cls):
                     'misc',
                     'test',
                 },
-                cls.EXTRACTOR: {'extractor'},
-                cls.DOWNLOADER: {'downloader'},
-                cls.POSTPROCESSOR: {'postprocessor'},
+                cls.EXTRACTOR: {'extractor', 'ie'},
+                cls.DOWNLOADER: {'downloader', 'fd'},
+                cls.POSTPROCESSOR: {'postprocessor', 'pp'},
             }.items()
             for name in names
         }
diff --git a/setup.cfg b/setup.cfg
index 68d9e516d..6deaa7971 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -8,7 +8,6 @@ ignore = E402,E501,E731,E741,W503
 max_line_length = 120
 per_file_ignores =
     devscripts/lazy_load_template.py: F401
-    yt_dlp/utils/__init__.py: F401, F403
 
 
 [autoflake]
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index d4aff0743..448a15bc9 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3687,7 +3687,7 @@ def render_formats_table(self, info_dict):
 
         def simplified_codec(f, field):
             assert field in ('acodec', 'vcodec')
-            codec = f.get(field, 'unknown')
+            codec = f.get(field)
             if not codec:
                 return 'unknown'
             elif codec != 'none':
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index a0219a350..8fe9d9993 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -255,7 +255,8 @@ def sanitize_open(self, filename, open_mode):
 
     @wrap_file_access('remove')
     def try_remove(self, filename):
-        os.remove(filename)
+        if os.path.isfile(filename):
+            os.remove(filename)
 
     @wrap_file_access('rename')
     def try_rename(self, old_filename, new_filename):
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 458167216..069815326 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -300,9 +300,7 @@ def frag_progress_hook(s):
     def _finish_frag_download(self, ctx, info_dict):
         ctx['dest_stream'].close()
         if self.__do_ytdl_file(ctx):
-            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
-            if os.path.isfile(ytdl_filename):
-                self.try_remove(ytdl_filename)
+            self.try_remove(self.ytdl_filename(ctx['filename']))
         elapsed = time.time() - ctx['started']
 
         to_file = ctx['tmpfilename'] != '-'
diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py
index 68a970f68..722a534ed 100644
--- a/yt_dlp/extractor/adobepass.py
+++ b/yt_dlp/extractor/adobepass.py
@@ -1473,7 +1473,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                     elif 'automatically signed in with' in provider_redirect_page:
                         # Seems like comcast is rolling up new way of automatically signing customers
                         oauth_redirect_url = self._html_search_regex(
-                            r'continue:\s*"(https://oauth.xfinity.com/oauth/authorize\?.+)"', provider_redirect_page,
+                            r'continue:\s*"(https://oauth\.xfinity\.com/oauth/authorize\?.+)"', provider_redirect_page,
                             'oauth redirect (signed)')
                         # Just need to process the request. No useful data comes back
                         self._download_webpage(oauth_redirect_url, video_id, 'Confirming auto login')
diff --git a/yt_dlp/extractor/iqiyi.py b/yt_dlp/extractor/iqiyi.py
index ebf49e835..fa602ba88 100644
--- a/yt_dlp/extractor/iqiyi.py
+++ b/yt_dlp/extractor/iqiyi.py
@@ -527,7 +527,7 @@ def _extract_vms_player_js(self, webpage, video_id):
         if player_js_cache:
             return player_js_cache
         webpack_js_url = self._proto_relative_url(self._search_regex(
-            r'<script src="((?:https?)?//stc.iqiyipic.com/_next/static/chunks/webpack-\w+\.js)"', webpage, 'webpack URL'))
+            r'<script src="((?:https?:)?//stc\.iqiyipic\.com/_next/static/chunks/webpack-\w+\.js)"', webpage, 'webpack URL'))
         webpack_js = self._download_webpage(webpack_js_url, video_id, note='Downloading webpack JS', errnote='Unable to download webpack JS')
         webpack_map = self._search_json(
             r'["\']\s*\+\s*', webpack_js, 'JS locations', video_id,
diff --git a/yt_dlp/extractor/vshare.py b/yt_dlp/extractor/vshare.py
index 1bc7ae4ba..443ed43cc 100644
--- a/yt_dlp/extractor/vshare.py
+++ b/yt_dlp/extractor/vshare.py
@@ -22,7 +22,7 @@ def _extract_packed(self, webpage):
         packed = self._search_regex(
             r'(eval\(function.+)', webpage, 'packed code')
         unpacked = decode_packed_codes(packed)
-        digits = self._search_regex(r'\[((?:\d+,?)+)\]', unpacked, 'digits')
+        digits = self._search_regex(r'\[([\d,]+)\]', unpacked, 'digits')
         digits = [int(digit) for digit in digits.split(',')]
         key_digit = self._search_regex(
             r'fromCharCode\(.+?(\d+)\)}', unpacked, 'key digit')
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 552ca099c..2a8106b45 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3117,7 +3117,7 @@ def _extract_n_function_name(self, jscode):
             return funcname
 
         return json.loads(js_to_json(self._search_regex(
-            rf'var {re.escape(funcname)}\s*=\s*(\[.+?\])[,;]', jscode,
+            rf'var {re.escape(funcname)}\s*=\s*(\[.+?\])\s*[,;]', jscode,
             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
 
     def _extract_n_function_code(self, video_id, player_url):
diff --git a/yt_dlp/utils/__init__.py b/yt_dlp/utils/__init__.py
index 74b39e2c7..2dd20ada2 100644
--- a/yt_dlp/utils/__init__.py
+++ b/yt_dlp/utils/__init__.py
@@ -1,3 +1,4 @@
+# flake8: noqa: F401, F403
 import warnings
 
 from ..compat.compat_utils import passthrough_module

From 906c0bdcd8974340d619e99ccd613c163eb0d0c2 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 6 Jul 2023 18:17:42 +0530
Subject: [PATCH 427/871] [formats] Fix best fallback for storyboards

Partial fix for #7478
---
 yt_dlp/YoutubeDL.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 448a15bc9..2c5014f87 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2766,11 +2766,8 @@ def is_wellformed(f):
             formats_to_download = list(format_selector({
                 'formats': formats,
                 'has_merged_format': any('none' not in (f.get('acodec'), f.get('vcodec')) for f in formats),
-                'incomplete_formats': (
-                    # All formats are video-only or
-                    all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats)
-                    # all formats are audio-only
-                    or all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats)),
+                'incomplete_formats': (all(f.get('vcodec') == 'none' for f in formats)  # No formats with video
+                                       or all(f.get('acodec') == 'none' for f in formats)),  # OR, No formats with audio
             }))
             if interactive_format_selection and not formats_to_download:
                 self.report_error('Requested format is not available', tb=False, is_error=False)

From bc344cd456380999c1ee74554dfd432a38f32ec7 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 6 Jul 2023 18:39:50 +0530
Subject: [PATCH 428/871] [core] Allow extractors to mark formats as
 potentially DRM (#7396)

This is useful for HLS where detecting whether the format is
actually DRM requires the child manifest to be downloaded.

Makes the error message when using `--test` inconsistent,
but doesn't really matter.
---
 yt_dlp/YoutubeDL.py        | 37 ++++++++++++++++++++++---------------
 yt_dlp/downloader/hls.py   | 30 ++++++++++++++++++++----------
 yt_dlp/extractor/common.py | 10 ++++------
 3 files changed, 46 insertions(+), 31 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 2c5014f87..cf0122d4b 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -983,6 +983,7 @@ def trouble(self, message=None, tb=None, is_error=True):
         ID='green',
         DELIM='blue',
         ERROR='red',
+        BAD_FORMAT='light red',
         WARNING='yellow',
         SUPPRESS='light black',
     )
@@ -2085,8 +2086,6 @@ def syntax_error(note, start):
         allow_multiple_streams = {'audio': self.params.get('allow_multiple_audio_streams', False),
                                   'video': self.params.get('allow_multiple_video_streams', False)}
 
-        check_formats = self.params.get('check_formats') == 'selected'
-
         def _parse_filter(tokens):
             filter_parts = []
             for type, string_, start, _, _ in tokens:
@@ -2259,10 +2258,19 @@ def _merge(formats_pair):
             return new_dict
 
         def _check_formats(formats):
-            if not check_formats:
+            if (self.params.get('check_formats') is not None
+                    or self.params.get('allow_unplayable_formats')):
                 yield from formats
                 return
-            yield from self._check_formats(formats)
+            elif self.params.get('check_formats') == 'selected':
+                yield from self._check_formats(formats)
+                return
+
+            for f in formats:
+                if f.get('has_drm'):
+                    yield from self._check_formats([f])
+                else:
+                    yield f
 
         def _build_selector_function(selector):
             if isinstance(selector, list):  # ,
@@ -2614,10 +2622,10 @@ def sanitize_numeric_fields(info):
         if field_preference:
             info_dict['_format_sort_fields'] = field_preference
 
-        # or None ensures --clean-infojson removes it
-        info_dict['_has_drm'] = any(f.get('has_drm') for f in formats) or None
+        info_dict['_has_drm'] = any(  # or None ensures --clean-infojson removes it
+            f.get('has_drm') and f['has_drm'] != 'maybe' for f in formats) or None
         if not self.params.get('allow_unplayable_formats'):
-            formats = [f for f in formats if not f.get('has_drm')]
+            formats = [f for f in formats if not f.get('has_drm') or f['has_drm'] == 'maybe']
 
         if formats and all(f.get('acodec') == f.get('vcodec') == 'none' for f in formats):
             self.report_warning(
@@ -3719,14 +3727,13 @@ def simplified_codec(f, field):
                 simplified_codec(f, 'acodec'),
                 format_field(f, 'abr', '\t%dk', func=round),
                 format_field(f, 'asr', '\t%s', func=format_decimal_suffix),
-                join_nonempty(
-                    self._format_out('UNSUPPORTED', 'light red') if f.get('ext') in ('f4f', 'f4m') else None,
-                    self._format_out('DRM', 'light red') if f.get('has_drm') else None,
-                    format_field(f, 'language', '[%s]'),
-                    join_nonempty(format_field(f, 'format_note'),
-                                  format_field(f, 'container', ignore=(None, f.get('ext'))),
-                                  delim=', '),
-                    delim=' '),
+                join_nonempty(format_field(f, 'language', '[%s]'), join_nonempty(
+                    self._format_out('UNSUPPORTED', self.Styles.BAD_FORMAT) if f.get('ext') in ('f4f', 'f4m') else None,
+                    (self._format_out('Maybe DRM', self.Styles.WARNING) if f.get('has_drm') == 'maybe'
+                     else self._format_out('DRM', self.Styles.BAD_FORMAT) if f.get('has_drm') else None),
+                    format_field(f, 'format_note'),
+                    format_field(f, 'container', ignore=(None, f.get('ext'))),
+                    delim=', '), delim=' '),
             ] for f in formats if f.get('preference') is None or f['preference'] >= -1000]
         header_line = self._list_format_headers(
             'ID', 'EXT', 'RESOLUTION', '\tFPS', 'HDR', 'CH', delim, '\tFILESIZE', '\tTBR', 'PROTO',
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index f2868dc52..ab7d496d4 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -28,7 +28,16 @@ class HlsFD(FragmentFD):
     FD_NAME = 'hlsnative'
 
     @staticmethod
-    def can_download(manifest, info_dict, allow_unplayable_formats=False):
+    def _has_drm(manifest):  # TODO: https://github.com/yt-dlp/yt-dlp/pull/5039
+        return bool(re.search('|'.join((
+            r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://',  # Apple FairPlay
+            r'#EXT-X-(?:SESSION-)?KEY:.*?KEYFORMAT="com\.apple\.streamingkeydelivery"',  # Apple FairPlay
+            r'#EXT-X-(?:SESSION-)?KEY:.*?KEYFORMAT="com\.microsoft\.playready"',  # Microsoft PlayReady
+            r'#EXT-X-FAXS-CM:',  # Adobe Flash Access
+        )), manifest))
+
+    @classmethod
+    def can_download(cls, manifest, info_dict, allow_unplayable_formats=False):
         UNSUPPORTED_FEATURES = [
             # r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
 
@@ -50,13 +59,15 @@ def can_download(manifest, info_dict, allow_unplayable_formats=False):
         ]
         if not allow_unplayable_formats:
             UNSUPPORTED_FEATURES += [
-                r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
+                r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1], but not necessarily DRM
             ]
 
         def check_results():
             yield not info_dict.get('is_live')
             for feature in UNSUPPORTED_FEATURES:
                 yield not re.search(feature, manifest)
+            if not allow_unplayable_formats:
+                yield not cls._has_drm(manifest)
         return all(check_results())
 
     def real_download(self, filename, info_dict):
@@ -81,14 +92,13 @@ def real_download(self, filename, info_dict):
                 message = ('Live HLS streams are not supported by the native downloader. If this is a livestream, '
                            f'please {install_ffmpeg}add "--downloader ffmpeg --hls-use-mpegts" to your command')
         if not can_download:
-            has_drm = re.search('|'.join([
-                r'#EXT-X-FAXS-CM:',  # Adobe Flash Access
-                r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://',  # Apple FairPlay
-            ]), s)
-            if has_drm and not self.params.get('allow_unplayable_formats'):
-                self.report_error(
-                    'This video is DRM protected; Try selecting another format with --format or '
-                    'add --check-formats to automatically fallback to the next best format')
+            if self._has_drm(s) and not self.params.get('allow_unplayable_formats'):
+                if info_dict.get('has_drm') and self.params.get('test'):
+                    self.to_screen(f'[{self.FD_NAME}] This format is DRM protected', skip_eol=True)
+                else:
+                    self.report_error(
+                        'This format is DRM protected; Try selecting another format with --format or '
+                        'add --check-formats to automatically fallback to the next best format', tb=False)
                 return False
             message = message or 'Unsupported features have been detected'
             fd = FFmpegFD(self.ydl, self.params)
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 3f7dcb82b..fe08839aa 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -26,6 +26,7 @@
 from ..compat import compat_etree_fromstring, compat_expanduser, compat_os_name
 from ..cookies import LenientSimpleCookie
 from ..downloader.f4m import get_base_url, remove_encrypted_media
+from ..downloader.hls import HlsFD
 from ..utils import (
     IDENTITY,
     JSON_LD_RE,
@@ -224,7 +225,8 @@ class InfoExtractor:
                                  width : height ratio as float.
                     * no_resume  The server does not support resuming the
                                  (HTTP or RTMP) download. Boolean.
-                    * has_drm    The format has DRM and cannot be downloaded. Boolean
+                    * has_drm    True if the format has DRM and cannot be downloaded.
+                                 'maybe' if the format may have DRM and has to be tested before download.
                     * extra_param_to_segment_url  A query string to append to each
                                  fragment's URL, or to update each existing query string
                                  with. Only applied by the native HLS/DASH downloaders.
@@ -1979,11 +1981,7 @@ def _parse_m3u8_formats_and_subtitles(
             errnote=None, fatal=True, data=None, headers={}, query={},
             video_id=None):
         formats, subtitles = [], {}
-
-        has_drm = re.search('|'.join([
-            r'#EXT-X-FAXS-CM:',  # Adobe Flash Access
-            r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://',  # Apple FairPlay
-        ]), m3u8_doc)
+        has_drm = HlsFD._has_drm(m3u8_doc)
 
         def format_url(url):
             return url if re.match(r'^https?://', url) else urllib.parse.urljoin(m3u8_url, url)

From 94ed638a437fc766699d440e978982e24ce6a30a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 23 Jun 2023 18:16:07 +0530
Subject: [PATCH 429/871] [ie/youtube] Avoid false DRM detection (#7396)

Some master manifests contain a mix of DRM and non-DRM formats
---
 yt_dlp/extractor/youtube.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 2a8106b45..73bfa662d 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3927,9 +3927,12 @@ def process_manifest_format(f, proto, client_name, itag):
             elif itag:
                 f['format_id'] = itag
 
+            if f.get('source_preference') is None:
+                f['source_preference'] = -1
+
             if itag in ('616', '235'):
                 f['format_note'] = join_nonempty(f.get('format_note'), 'Premium', delim=' ')
-                f['source_preference'] = (f.get('source_preference') or -1) + 100
+                f['source_preference'] += 100
 
             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
             if f['quality'] == -1 and f.get('height'):
@@ -3938,6 +3941,10 @@ def process_manifest_format(f, proto, client_name, itag):
                 f['format_note'] = join_nonempty(f.get('format_note'), client_name, delim=', ')
             if f.get('fps') and f['fps'] <= 1:
                 del f['fps']
+
+            if proto == 'hls' and f.get('has_drm'):
+                f['has_drm'] = 'maybe'
+                f['source_preference'] -= 5
             return True
 
         subtitles = {}
@@ -4037,6 +4044,10 @@ def _list_formats(self, video_id, microformats, video_details, player_responses,
                        else None)
         streaming_data = traverse_obj(player_responses, (..., 'streamingData'))
         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
+        if all(f.get('has_drm') for f in formats):
+            # If there are no formats that definitely don't have DRM, all have DRM
+            for f in formats:
+                f['has_drm'] = True
 
         return live_broadcast_details, live_status, streaming_data, formats, subtitles
 

From ad8902f616ad2541f9b9626738f1393fad89a64c Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 6 Jul 2023 19:35:49 +0530
Subject: [PATCH 430/871] [ie/vidlii] Handle relative URLs

Closes #7480
---
 yt_dlp/extractor/vidlii.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/yt_dlp/extractor/vidlii.py b/yt_dlp/extractor/vidlii.py
index 5933783ae..cde4274d9 100644
--- a/yt_dlp/extractor/vidlii.py
+++ b/yt_dlp/extractor/vidlii.py
@@ -70,6 +70,7 @@ def _real_extract(self, url):
             r'src\s*:\s*(["\'])(?P<url>(?:https?://)?(?:(?!\1).)+)\1',
             webpage) or []]
         for source in sources:
+            source = urljoin(url, source)
             height = int(self._search_regex(r'(\d+).mp4', source, 'height', default=360))
             if self._request_webpage(HEADRequest(source), video_id, f'Checking {height}p url', errnote=False):
                 formats.append({

From 1ceb657bdd254ad961489e5060f2ccc7d556b729 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Wed, 5 Jul 2023 15:16:28 -0500
Subject: [PATCH 431/871] [fd/external] Scope cookies

- ffmpeg: Calculate cookies from cookiejar and pass with `-cookies` arg instead of `-headers`
- aria2c, curl, wget: Write cookiejar to file and use external FD built-in cookiejar support
- httpie: Calculate cookies from cookiejar instead of `http_headers`
- axel: Calculate cookies from cookiejar and disable http redirection if cookies are passed
    - May break redirects, but axel simply don't have proper cookie support

Ref: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj

Authored by: bashonly, coletdjnz
---
 test/test_downloader_external.py | 133 +++++++++++++++++++++++++++++++
 yt_dlp/cookies.py                |   7 ++
 yt_dlp/downloader/external.py    |  41 +++++++++-
 3 files changed, 179 insertions(+), 2 deletions(-)
 create mode 100644 test/test_downloader_external.py

diff --git a/test/test_downloader_external.py b/test/test_downloader_external.py
new file mode 100644
index 000000000..e5b02ba5a
--- /dev/null
+++ b/test/test_downloader_external.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+# Allow direct execution
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import http.cookiejar
+
+from test.helper import FakeYDL
+from yt_dlp.downloader.external import (
+    Aria2cFD,
+    AxelFD,
+    CurlFD,
+    FFmpegFD,
+    HttpieFD,
+    WgetFD,
+)
+
+TEST_COOKIE = {
+    'version': 0,
+    'name': 'test',
+    'value': 'ytdlp',
+    'port': None,
+    'port_specified': False,
+    'domain': '.example.com',
+    'domain_specified': True,
+    'domain_initial_dot': False,
+    'path': '/',
+    'path_specified': True,
+    'secure': False,
+    'expires': None,
+    'discard': False,
+    'comment': None,
+    'comment_url': None,
+    'rest': {},
+}
+
+TEST_INFO = {'url': 'http://www.example.com/'}
+
+
+class TestHttpieFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = HttpieFD(ydl, {})
+            self.assertEqual(
+                downloader._make_cmd('test', TEST_INFO),
+                ['http', '--download', '--output', 'test', 'http://www.example.com/'])
+
+            # Test cookie header is added
+            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(**TEST_COOKIE))
+            self.assertEqual(
+                downloader._make_cmd('test', TEST_INFO),
+                ['http', '--download', '--output', 'test', 'http://www.example.com/', 'Cookie:test=ytdlp'])
+
+
+class TestAxelFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = AxelFD(ydl, {})
+            self.assertEqual(
+                downloader._make_cmd('test', TEST_INFO),
+                ['axel', '-o', 'test', '--', 'http://www.example.com/'])
+
+            # Test cookie header is added
+            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(**TEST_COOKIE))
+            self.assertEqual(
+                downloader._make_cmd('test', TEST_INFO),
+                ['axel', '-o', 'test', 'Cookie: test=ytdlp', '--max-redirect=0', '--', 'http://www.example.com/'])
+
+
+class TestWgetFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = WgetFD(ydl, {})
+            self.assertNotIn('--load-cookies', downloader._make_cmd('test', TEST_INFO))
+            # Test cookiejar tempfile arg is added
+            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(**TEST_COOKIE))
+            self.assertIn('--load-cookies', downloader._make_cmd('test', TEST_INFO))
+
+
+class TestCurlFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = CurlFD(ydl, {})
+            self.assertNotIn('--cookie-jar', downloader._make_cmd('test', TEST_INFO))
+            # Test cookiejar tempfile arg is added
+            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(**TEST_COOKIE))
+            self.assertIn('--cookie-jar', downloader._make_cmd('test', TEST_INFO))
+
+
+class TestAria2cFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = Aria2cFD(ydl, {})
+            downloader._make_cmd('test', TEST_INFO)
+            self.assertFalse(hasattr(downloader, '_cookies_tempfile'))
+
+            # Test cookiejar tempfile arg is added
+            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(**TEST_COOKIE))
+            cmd = downloader._make_cmd('test', TEST_INFO)
+            self.assertIn(f'--load-cookies={downloader._cookies_tempfile}', cmd)
+
+
+@unittest.skipUnless(FFmpegFD.available(), 'ffmpeg not found')
+class TestFFmpegFD(unittest.TestCase):
+    _args = []
+
+    def _test_cmd(self, args):
+        self._args = args
+
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = FFmpegFD(ydl, {})
+            downloader._debug_cmd = self._test_cmd
+
+            downloader._call_downloader('test', {**TEST_INFO, 'ext': 'mp4'})
+            self.assertEqual(self._args, [
+                'ffmpeg', '-y', '-hide_banner', '-i', 'http://www.example.com/',
+                '-c', 'copy', '-f', 'mp4', 'file:test'])
+
+            # Test cookies arg is added
+            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(**TEST_COOKIE))
+            downloader._call_downloader('test', {**TEST_INFO, 'ext': 'mp4'})
+            self.assertEqual(self._args, [
+                'ffmpeg', '-y', '-hide_banner', '-cookies', 'test=ytdlp; path=/; domain=.example.com;\r\n',
+                '-i', 'http://www.example.com/', '-c', 'copy', '-f', 'mp4', 'file:test'])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index f21e4f7e7..53fe0ec2d 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -1327,6 +1327,13 @@ def get_cookie_header(self, url):
         self.add_cookie_header(cookie_req)
         return cookie_req.get_header('Cookie')
 
+    def get_cookies_for_url(self, url):
+        """Generate a list of Cookie objects for a given url"""
+        # Policy `_now` attribute must be set before calling `_cookies_for_request`
+        # Ref: https://github.com/python/cpython/blob/3.7/Lib/http/cookiejar.py#L1360
+        self._policy._now = self._now = int(time.time())
+        return self._cookies_for_request(urllib.request.Request(escape_url(sanitize_url(url))))
+
     def clear(self, *args, **kwargs):
         with contextlib.suppress(KeyError):
             return super().clear(*args, **kwargs)
diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index f637a100b..d4045e58f 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -1,9 +1,10 @@
 import enum
 import json
-import os.path
+import os
 import re
 import subprocess
 import sys
+import tempfile
 import time
 import uuid
 
@@ -42,6 +43,7 @@ class ExternalFD(FragmentFD):
     def real_download(self, filename, info_dict):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
+        self._cookies_tempfile = None
 
         try:
             started = time.time()
@@ -54,6 +56,9 @@ def real_download(self, filename, info_dict):
             # should take place
             retval = 0
             self.to_screen('[%s] Interrupted by user' % self.get_basename())
+        finally:
+            if self._cookies_tempfile:
+                self.try_remove(self._cookies_tempfile)
 
         if retval == 0:
             status = {
@@ -125,6 +130,16 @@ def _configuration_args(self, keys=None, *args, **kwargs):
             self.get_basename(), self.params.get('external_downloader_args'), self.EXE_NAME,
             keys, *args, **kwargs)
 
+    def _write_cookies(self):
+        if not self.ydl.cookiejar.filename:
+            tmp_cookies = tempfile.NamedTemporaryFile(suffix='.cookies', delete=False)
+            tmp_cookies.close()
+            self._cookies_tempfile = tmp_cookies.name
+            self.to_screen(f'[download] Writing temporary cookies file to "{self._cookies_tempfile}"')
+        # real_download resets _cookies_tempfile; if it's None then save() will write to cookiejar.filename
+        self.ydl.cookiejar.save(self._cookies_tempfile)
+        return self.ydl.cookiejar.filename or self._cookies_tempfile
+
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
@@ -184,6 +199,8 @@ class CurlFD(ExternalFD):
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '--location', '-o', tmpfilename, '--compressed']
+        if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
+            cmd += ['--cookie-jar', self._write_cookies()]
         if info_dict.get('http_headers') is not None:
             for key, val in info_dict['http_headers'].items():
                 cmd += ['--header', f'{key}: {val}']
@@ -214,6 +231,9 @@ def _make_cmd(self, tmpfilename, info_dict):
         if info_dict.get('http_headers') is not None:
             for key, val in info_dict['http_headers'].items():
                 cmd += ['-H', f'{key}: {val}']
+        cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
+        if cookie_header:
+            cmd += [f'Cookie: {cookie_header}', '--max-redirect=0']
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -223,7 +243,9 @@ class WgetFD(ExternalFD):
     AVAILABLE_OPT = '--version'
 
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies', '--compression=auto']
+        cmd = [self.exe, '-O', tmpfilename, '-nv', '--compression=auto']
+        if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
+            cmd += ['--load-cookies', self._write_cookies()]
         if info_dict.get('http_headers') is not None:
             for key, val in info_dict['http_headers'].items():
                 cmd += ['--header', f'{key}: {val}']
@@ -279,6 +301,8 @@ def _make_cmd(self, tmpfilename, info_dict):
         else:
             cmd += ['--min-split-size', '1M']
 
+        if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
+            cmd += [f'--load-cookies={self._write_cookies()}']
         if info_dict.get('http_headers') is not None:
             for key, val in info_dict['http_headers'].items():
                 cmd += ['--header', f'{key}: {val}']
@@ -417,6 +441,14 @@ def _make_cmd(self, tmpfilename, info_dict):
         if info_dict.get('http_headers') is not None:
             for key, val in info_dict['http_headers'].items():
                 cmd += [f'{key}:{val}']
+
+        # httpie 3.1.0+ removes the Cookie header on redirect, so this should be safe for now. [1]
+        # If we ever need cookie handling for redirects, we can export the cookiejar into a session. [2]
+        # 1: https://github.com/httpie/httpie/security/advisories/GHSA-9w4w-cpc8-h2fq
+        # 2: https://httpie.io/docs/cli/sessions
+        cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
+        if cookie_header:
+            cmd += [f'Cookie:{cookie_header}']
         return cmd
 
 
@@ -527,6 +559,11 @@ def _call_downloader(self, tmpfilename, info_dict):
 
         selected_formats = info_dict.get('requested_formats') or [info_dict]
         for i, fmt in enumerate(selected_formats):
+            cookies = self.ydl.cookiejar.get_cookies_for_url(fmt['url'])
+            if cookies:
+                args.extend(['-cookies', ''.join(
+                    f'{cookie.name}={cookie.value}; path={cookie.path}; domain={cookie.domain};\r\n'
+                    for cookie in cookies)])
             if fmt.get('http_headers') and re.match(r'^https?://', fmt['url']):
                 # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
                 # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.

From f8b4bcc0a791274223723488bfbfc23ea3276641 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Tue, 6 Jun 2023 20:44:51 +1200
Subject: [PATCH 432/871] [core] Prevent `Cookie` leaks on HTTP redirect

Ref: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj

Authored by: coletdjnz
---
 test/test_http.py      | 31 +++++++++++++++++++++++++++++++
 yt_dlp/utils/_utils.py |  9 +++++++--
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/test/test_http.py b/test/test_http.py
index 3941a6e77..e4e66dce1 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -132,6 +132,11 @@ def do_GET(self):
             self._method('GET')
         elif self.path.startswith('/headers'):
             self._headers()
+        elif self.path.startswith('/308-to-headers'):
+            self.send_response(308)
+            self.send_header('Location', '/headers')
+            self.send_header('Content-Length', '0')
+            self.end_headers()
         elif self.path == '/trailing_garbage':
             payload = b'<html><video src="/vid.mp4" /></html>'
             self.send_response(200)
@@ -270,6 +275,7 @@ def do_req(redirect_status, method):
             self.assertEqual(do_req(303, 'PUT'), ('', 'GET'))
 
             # 301 and 302 turn POST only into a GET
+            # XXX: we should also test if the Content-Type and Content-Length headers are removed
             self.assertEqual(do_req(301, 'POST'), ('', 'GET'))
             self.assertEqual(do_req(301, 'HEAD'), ('', 'HEAD'))
             self.assertEqual(do_req(302, 'POST'), ('', 'GET'))
@@ -313,6 +319,31 @@ def test_cookiejar(self):
             data = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/headers')).read()
             self.assertIn(b'Cookie: test=ytdlp', data)
 
+    def test_passed_cookie_header(self):
+        # We should accept a Cookie header being passed as in normal headers and handle it appropriately.
+        with FakeYDL() as ydl:
+            # Specified Cookie header should be used
+            res = ydl.urlopen(
+                sanitized_Request(f'http://127.0.0.1:{self.http_port}/headers',
+                                  headers={'Cookie': 'test=test'})).read().decode('utf-8')
+            self.assertIn('Cookie: test=test', res)
+
+            # Specified Cookie header should be removed on any redirect
+            res = ydl.urlopen(
+                sanitized_Request(f'http://127.0.0.1:{self.http_port}/308-to-headers', headers={'Cookie': 'test=test'})).read().decode('utf-8')
+            self.assertNotIn('Cookie: test=test', res)
+
+            # Specified Cookie header should override global cookiejar for that request
+            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(
+                version=0, name='test', value='ytdlp', port=None, port_specified=False,
+                domain='127.0.0.1', domain_specified=True, domain_initial_dot=False, path='/',
+                path_specified=True, secure=False, expires=None, discard=False, comment=None,
+                comment_url=None, rest={}))
+
+            data = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'Cookie': 'test=test'})).read()
+            self.assertNotIn(b'Cookie: test=ytdlp', data)
+            self.assertIn(b'Cookie: test=test', data)
+
     def test_no_compression_compat_header(self):
         with FakeYDL() as ydl:
             data = ydl.urlopen(
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index f68cdb968..82d9ba4d5 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -1556,7 +1556,12 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
 
         new_method = req.get_method()
         new_data = req.data
-        remove_headers = []
+
+        # Technically the Cookie header should be in unredirected_hdrs,
+        # however in practice some may set it in normal headers anyway.
+        # We will remove it here to prevent any leaks.
+        remove_headers = ['Cookie']
+
         # A 303 must either use GET or HEAD for subsequent request
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.4
         if code == 303 and req.get_method() != 'HEAD':
@@ -1573,7 +1578,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
             new_data = None
             remove_headers.extend(['Content-Length', 'Content-Type'])
 
-        new_headers = {k: v for k, v in req.headers.items() if k.lower() not in remove_headers}
+        new_headers = {k: v for k, v in req.headers.items() if k.title() not in remove_headers}
 
         return urllib.request.Request(
             newurl, headers=new_headers, origin_req_host=req.origin_req_host,

From 3121512228487c9c690d3d39bfd2579addf96e07 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Thu, 6 Jul 2023 21:51:04 +0530
Subject: [PATCH 433/871] [core] Change how `Cookie` headers are handled

Cookies are now saved and loaded under `cookies` key in the info dict
instead of `http_headers.Cookie`. Cookies passed in headers are
auto-scoped to the input URLs with a warning.

Ref: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj

Authored by: Grub4K
---
 test/test_YoutubeDL.py      | 56 ++++++++++++++++++++++++++
 yt_dlp/YoutubeDL.py         | 80 +++++++++++++++++++++++++++++++++++--
 yt_dlp/downloader/common.py |  7 +++-
 3 files changed, 139 insertions(+), 4 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 3fbcdd01f..c15c7704c 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -1213,6 +1213,62 @@ def _real_extract(self, url):
         self.assertEqual(downloaded['extractor'], 'Video')
         self.assertEqual(downloaded['extractor_key'], 'Video')
 
+    def test_header_cookies(self):
+        from http.cookiejar import Cookie
+
+        ydl = FakeYDL()
+        ydl.report_warning = lambda *_, **__: None
+
+        def cookie(name, value, version=None, domain='', path='', secure=False, expires=None):
+            return Cookie(
+                version or 0, name, value, None, False,
+                domain, bool(domain), bool(domain), path, bool(path),
+                secure, expires, False, None, None, rest={})
+
+        _test_url = 'https://yt.dlp/test'
+
+        def test(encoded_cookies, cookies, headers=False, round_trip=None, error=None):
+            def _test():
+                ydl.cookiejar.clear()
+                ydl._load_cookies(encoded_cookies, from_headers=headers)
+                if headers:
+                    ydl._apply_header_cookies(_test_url)
+                data = {'url': _test_url}
+                ydl._calc_headers(data)
+                self.assertCountEqual(
+                    map(vars, ydl.cookiejar), map(vars, cookies),
+                    'Extracted cookiejar.Cookie is not the same')
+                if not headers:
+                    self.assertEqual(
+                        data.get('cookies'), round_trip or encoded_cookies,
+                        'Cookie is not the same as round trip')
+                ydl.__dict__['_YoutubeDL__header_cookies'] = []
+
+            with self.subTest(msg=encoded_cookies):
+                if not error:
+                    _test()
+                    return
+                with self.assertRaisesRegex(Exception, error):
+                    _test()
+
+        test('test=value; Domain=.yt.dlp', [cookie('test', 'value', domain='.yt.dlp')])
+        test('test=value', [cookie('test', 'value')], error='Unscoped cookies are not allowed')
+        test('cookie1=value1; Domain=.yt.dlp; Path=/test; cookie2=value2; Domain=.yt.dlp; Path=/', [
+            cookie('cookie1', 'value1', domain='.yt.dlp', path='/test'),
+            cookie('cookie2', 'value2', domain='.yt.dlp', path='/')])
+        test('test=value; Domain=.yt.dlp; Path=/test; Secure; Expires=9999999999', [
+            cookie('test', 'value', domain='.yt.dlp', path='/test', secure=True, expires=9999999999)])
+        test('test="value; "; path=/test; domain=.yt.dlp', [
+            cookie('test', 'value; ', domain='.yt.dlp', path='/test')],
+            round_trip='test="value\\073 "; Domain=.yt.dlp; Path=/test')
+        test('name=; Domain=.yt.dlp', [cookie('name', '', domain='.yt.dlp')],
+             round_trip='name=""; Domain=.yt.dlp')
+
+        test('test=value', [cookie('test', 'value', domain='.yt.dlp')], headers=True)
+        test('cookie1=value; Domain=.yt.dlp; cookie2=value', [], headers=True, error='Invalid syntax')
+        ydl.deprecated_feature = ydl.report_error
+        test('test=value', [], headers=True, error='Passing cookies as a header is a potential security risk')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index cf0122d4b..7f5571666 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1,9 +1,11 @@
 import collections
 import contextlib
+import copy
 import datetime
 import errno
 import fileinput
 import functools
+import http.cookiejar
 import io
 import itertools
 import json
@@ -25,7 +27,7 @@
 from .cache import Cache
 from .compat import urllib  # isort: split
 from .compat import compat_os_name, compat_shlex_quote
-from .cookies import load_cookies
+from .cookies import LenientSimpleCookie, load_cookies
 from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name
 from .downloader.rtmp import rtmpdump_version
 from .extractor import gen_extractor_classes, get_info_extractor
@@ -673,6 +675,9 @@ def process_color_policy(stream):
         if auto_init and auto_init != 'no_verbose_header':
             self.print_debug_header()
 
+        self.__header_cookies = []
+        self._load_cookies(traverse_obj(self.params.get('http_headers'), 'cookie', casesense=False))  # compat
+
         def check_deprecated(param, option, suggestion):
             if self.params.get(param) is not None:
                 self.report_warning(f'{option} is deprecated. Use {suggestion} instead')
@@ -1625,8 +1630,60 @@ def progress(msg):
                 self.to_screen('')
             raise
 
+    def _load_cookies(self, data, *, from_headers=True):
+        """Loads cookies from a `Cookie` header
+
+        This tries to work around the security vulnerability of passing cookies to every domain.
+        See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
+        The unscoped cookies are saved for later to be stored in the jar with a limited scope.
+
+        @param data         The Cookie header as string to load the cookies from
+        @param from_headers If `False`, allows Set-Cookie syntax in the cookie string (at least a domain will be required)
+        """
+        for cookie in LenientSimpleCookie(data).values():
+            if from_headers and any(cookie.values()):
+                raise ValueError('Invalid syntax in Cookie Header')
+
+            domain = cookie.get('domain') or ''
+            expiry = cookie.get('expires')
+            if expiry == '':  # 0 is valid
+                expiry = None
+            prepared_cookie = http.cookiejar.Cookie(
+                cookie.get('version') or 0, cookie.key, cookie.value, None, False,
+                domain, True, True, cookie.get('path') or '', bool(cookie.get('path')),
+                cookie.get('secure') or False, expiry, False, None, None, {})
+
+            if domain:
+                self.cookiejar.set_cookie(prepared_cookie)
+            elif from_headers:
+                self.deprecated_feature(
+                    'Passing cookies as a header is a potential security risk; '
+                    'they will be scoped to the domain of the downloaded urls. '
+                    'Please consider loading cookies from a file or browser instead.')
+                self.__header_cookies.append(prepared_cookie)
+            else:
+                self.report_error('Unscoped cookies are not allowed; please specify some sort of scoping',
+                                  tb=False, is_error=False)
+
+    def _apply_header_cookies(self, url):
+        """Applies stray header cookies to the provided url
+
+        This loads header cookies and scopes them to the domain provided in `url`.
+        While this is not ideal, it helps reduce the risk of them being sent
+        to an unintended destination while mostly maintaining compatibility.
+        """
+        parsed = urllib.parse.urlparse(url)
+        if not parsed.hostname:
+            return
+
+        for cookie in map(copy.copy, self.__header_cookies):
+            cookie.domain = f'.{parsed.hostname}'
+            self.cookiejar.set_cookie(cookie)
+
     @_handle_extraction_exceptions
     def __extract_info(self, url, ie, download, extra_info, process):
+        self._apply_header_cookies(url)
+
         try:
             ie_result = ie.extract(url)
         except UserNotLive as e:
@@ -2414,9 +2471,24 @@ def _calc_headers(self, info_dict):
         if 'Youtubedl-No-Compression' in res:  # deprecated
             res.pop('Youtubedl-No-Compression', None)
             res['Accept-Encoding'] = 'identity'
-        cookies = self.cookiejar.get_cookie_header(info_dict['url'])
+        cookies = self.cookiejar.get_cookies_for_url(info_dict['url'])
         if cookies:
-            res['Cookie'] = cookies
+            encoder = LenientSimpleCookie()
+            values = []
+            for cookie in cookies:
+                _, value = encoder.value_encode(cookie.value)
+                values.append(f'{cookie.name}={value}')
+                if cookie.domain:
+                    values.append(f'Domain={cookie.domain}')
+                if cookie.path:
+                    values.append(f'Path={cookie.path}')
+                if cookie.secure:
+                    values.append('Secure')
+                if cookie.expires:
+                    values.append(f'Expires={cookie.expires}')
+                if cookie.version:
+                    values.append(f'Version={cookie.version}')
+            info_dict['cookies'] = '; '.join(values)
 
         if 'X-Forwarded-For' not in res:
             x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
@@ -3423,6 +3495,8 @@ def download_with_info_file(self, info_filename):
             infos = [self.sanitize_info(info, self.params.get('clean_infojson', True))
                      for info in variadic(json.loads('\n'.join(f)))]
         for info in infos:
+            self._load_cookies(info.get('cookies'), from_headers=False)
+            self._load_cookies(traverse_obj(info.get('http_headers'), 'Cookie', casesense=False))  # compat
             try:
                 self.__download_wrapper(self.process_ie_result)(info, download=True)
             except (DownloadError, EntryNotInPlaylist, ReExtractInfo) as e:
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index 8fe9d9993..2c404ee90 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -32,6 +32,7 @@
     timetuple_from_msec,
     try_call,
 )
+from ..utils.traversal import traverse_obj
 
 
 class FileDownloader:
@@ -419,7 +420,6 @@ def download(self, filename, info_dict, subtitle=False):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
         """
-
         nooverwrites_and_exists = (
             not self.params.get('overwrites', True)
             and os.path.exists(encodeFilename(filename))
@@ -453,6 +453,11 @@ def download(self, filename, info_dict, subtitle=False):
             self.to_screen(f'[download] Sleeping {sleep_interval:.2f} seconds ...')
             time.sleep(sleep_interval)
 
+        # Filter the `Cookie` header from the info_dict to prevent leaks.
+        # See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
+        info_dict['http_headers'] = dict(traverse_obj(info_dict, (
+            'http_headers', {dict.items}, lambda _, pair: pair[0].lower() != 'cookie'))) or None
+
         ret = self.real_download(filename, info_dict)
         self._finish_multiline_status()
         return ret, True

From b532a3481046e1eabb6232ee8196fb696c356ff6 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 6 Jul 2023 19:18:35 +0530
Subject: [PATCH 434/871] [docs] Minor fixes

Closes #7515
---
 Changelog.md                       |  6 ++++--
 README.md                          | 14 +++++++-------
 devscripts/changelog_override.json | 20 ++++++++++++++------
 3 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/Changelog.md b/Changelog.md
index c340b74c9..d7efa5d25 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -9,6 +9,8 @@ ### 2023.06.22
 #### Core changes
 - [Fix bug in db3ad8a67661d7b234a6954d9c6a4a9b1749f5eb](https://github.com/yt-dlp/yt-dlp/commit/d7cd97e8d8d42b500fea9abb2aa4ac9b0f98b2ad) by [pukkandan](https://github.com/pukkandan)
 - [Improve `--download-sections`](https://github.com/yt-dlp/yt-dlp/commit/b4e0d75848e9447cee2cd3646ce54d4744a7ff56) by [pukkandan](https://github.com/pukkandan)
+    - Support negative time-ranges
+    - Add `*from-url` to obey time-ranges in URL
 - [Indicate `filesize` approximated from `tbr` better](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) by [pukkandan](https://github.com/pukkandan)
 
 #### Extractor changes
@@ -19,7 +21,7 @@ #### Extractor changes
 - **nebula**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/3f756c8c4095b942cf49788eb0862ceaf57847f2) ([#7156](https://github.com/yt-dlp/yt-dlp/issues/7156)) by [Lamieur](https://github.com/Lamieur), [rohieb](https://github.com/rohieb)
 - **rheinmaintv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/98cb1eda7a4cf67c96078980dbd63e6c06ad7f7c) ([#7311](https://github.com/yt-dlp/yt-dlp/issues/7311)) by [barthelmannk](https://github.com/barthelmannk)
 - **youtube**
-    - [Add `ios` to default clients used](https://github.com/yt-dlp/yt-dlp/commit/1e75d97db21152acc764b30a688e516f04b8a142)
+    - [Add `ios` to default clients used](https://github.com/yt-dlp/yt-dlp/commit/1e75d97db21152acc764b30a688e516f04b8a142) by [pukkandan](https://github.com/pukkandan)
         - IOS is affected neither by 403 nor by nsig so helps mitigate them preemptively
         - IOS also has higher bit-rate 'premium' formats though they are not labeled as such
     - [Improve description parsing performance](https://github.com/yt-dlp/yt-dlp/commit/71dc18fa29263a1ff0472c23d81bfc8dd4422d48) ([#7315](https://github.com/yt-dlp/yt-dlp/issues/7315)) by [berkanteber](https://github.com/berkanteber), [pukkandan](https://github.com/pukkandan)
@@ -27,7 +29,7 @@ #### Extractor changes
     - [Workaround 403 for android formats](https://github.com/yt-dlp/yt-dlp/commit/81ca451480051d7ce1a31c017e005358345a9149) by [pukkandan](https://github.com/pukkandan)
 
 #### Misc. changes
-- [Revert "Add automatic duplicate issue detection"](https://github.com/yt-dlp/yt-dlp/commit/a4486bfc1dc7057efca9dd3fe70d7fa25c56f700)
+- [Revert "Add automatic duplicate issue detection"](https://github.com/yt-dlp/yt-dlp/commit/a4486bfc1dc7057efca9dd3fe70d7fa25c56f700) by [pukkandan](https://github.com/pukkandan)
 - **cleanup**
     - Miscellaneous
         - [7f9c6a6](https://github.com/yt-dlp/yt-dlp/commit/7f9c6a63b16e145495479e9f666f5b9e2ee69e2f) by [bashonly](https://github.com/bashonly)
diff --git a/README.md b/README.md
index 4fb3e450d..0526fe418 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 [![License: Unlicense](https://img.shields.io/badge/-Unlicense-blue.svg?style=for-the-badge)](LICENSE "License")
 [![CI Status](https://img.shields.io/github/actions/workflow/status/yt-dlp/yt-dlp/core.yml?branch=master&label=Tests&style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/actions "CI Status")
 [![Commits](https://img.shields.io/github/commit-activity/m/yt-dlp/yt-dlp?label=commits&style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/commits "Commit History")
-[![Last Commit](https://img.shields.io/github/last-commit/yt-dlp/yt-dlp/master?label=&style=for-the-badge&display_timestamp=committer)](https://github.com/yt-dlp/yt-dlp/commits "Commit History")
+[![Last Commit](https://img.shields.io/github/last-commit/yt-dlp/yt-dlp/master?label=&style=for-the-badge&display_timestamp=committer)](https://github.com/yt-dlp/yt-dlp/pulse/monthly "Last activity")
 
 </div>
 <!-- MANPAGE: END EXCLUDED SECTION -->
@@ -76,7 +76,7 @@
 
 # NEW FEATURES
 
-* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@42f2d4**](https://github.com/yt-dlp/yt-dlp/commit/42f2d4) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
+* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@42f2d4**](https://github.com/ytdl-org/youtube-dl/commit/07af47960f3bb262ead02490ce65c8c45c01741e) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 
@@ -1323,7 +1323,7 @@ # OUTPUT TEMPLATE
  - `extractor` (string): Name of the extractor
  - `extractor_key` (string): Key name of the extractor
  - `epoch` (numeric): Unix epoch of when the information extraction was completed
- - `autonumber` (numeric): Number that will be increased with each download, starting at `--autonumber-start`
+ - `autonumber` (numeric): Number that will be increased with each download, starting at `--autonumber-start`, padded with leading zeros to 5 digits
  - `video_autonumber` (numeric): Number that will be increased with each video
  - `n_entries` (numeric): Total number of extracted items in the playlist
  - `playlist_id` (string): Identifier of the playlist that contains the video
@@ -1509,7 +1509,7 @@ # FORMAT SELECTION
 
 ## Filtering Formats
 
-You can also filter the video formats by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).
+You can also filter the video formats by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"` since filters without a selector are interpreted as `best`).
 
 The following numeric meta fields can be used with comparisons `<`, `<=`, `>`, `>=`, `=` (equals), `!=` (not equals):
 
@@ -1545,7 +1545,7 @@ ## Filtering Formats
 
 **Note**: None of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by the website. Any other field made available by the extractor can also be used for filtering.
 
-Formats for which the value is not known are excluded unless you put a question mark (`?`) after the operator. You can combine format filters, so `-f "[height<=?720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. You can also use the filters with `all` to download all formats that satisfy the filter, e.g. `-f "all[vcodec=none]"` selects all audio-only formats.
+Formats for which the value is not known are excluded unless you put a question mark (`?`) after the operator. You can combine format filters, so `-f "bv[height<=?720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. You can also use the filters with `all` to download all formats that satisfy the filter, e.g. `-f "all[vcodec=none]"` selects all audio-only formats.
 
 Format selectors can also be grouped using parentheses; e.g. `-f "(mp4,webm)[height<480]"` will download the best pre-merged mp4 and webm formats with a height lower than 480.
 
@@ -1805,7 +1805,7 @@ #### youtube
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
 * `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all`
     * E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total
-* `formats`: Change the types of formats to return. `dashy` (convert http to DASH), `duplicate` (identical content but different URLs or protocol; includes `dashy`), `incomplete` (cannot be downloaded completely - live dash and post-live m3u8)
+* `formats`: Change the types of formats to return. `dashy` (convert HTTP to DASH), `duplicate` (identical content but different URLs or protocol; includes `dashy`), `incomplete` (cannot be downloaded completely - live dash and post-live m3u8)
 * `innertube_host`: Innertube API host to use for all API requests; e.g. `studio.youtube.com`, `youtubei.googleapis.com`. Note that cookies exported from one subdomain will not work on others
 * `innertube_key`: Innertube API key to use for all API requests
 
@@ -1950,7 +1950,7 @@ # EMBEDDING YT-DLP
     ydl.download(URLS)
 ```
 
-Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L184).
+Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L183) or `help(yt_dlp.YoutubeDL)` in a Python shell. If you are already familiar with the CLI, you can use [`devscripts/cli_to_api.py`](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) to translate any CLI switches to `YoutubeDL` params.
 
 **Tip**: If you are porting your code from youtube-dl to yt-dlp, one important point to look out for is that we do not guarantee the return value of `YoutubeDL.extract_info` to be json serializable, or even be a dictionary. It will be dictionary-like, but if you want to ensure it is a serializable dictionary, pass it through `YoutubeDL.sanitize_info` as shown in the [example below](#extracting-information)
 
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index df80f45e0..f573a7463 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -1,12 +1,12 @@
 [
     {
         "action": "add",
-        "when": "776d1c3f0c9b00399896dd2e40e78e9a43218109",
+        "when": "29cb20bd563c02671b31dd840139e93dd37150a1",
         "short": "[priority] **A new release type has been added!**\n    * [`nightly`](https://github.com/yt-dlp/yt-dlp/releases/tag/nightly) builds will be made after each push, containing the latest fixes (but also possibly bugs).\n    * When using `--update`/`-U`, a release binary will only update to its current channel (either `stable` or `nightly`).\n    * The `--update-to` option has been added allowing the user more control over program upgrades (or downgrades).\n    * `--update-to` can change the release channel (`stable`, `nightly`) and also upgrade or downgrade to specific tags.\n    * **Usage**: `--update-to CHANNEL`, `--update-to TAG`, `--update-to CHANNEL@TAG`"
     },
     {
         "action": "add",
-        "when": "776d1c3f0c9b00399896dd2e40e78e9a43218109",
+        "when": "5038f6d713303e0967d002216e7a88652401c22a",
         "short": "[priority] **YouTube throttling fixes!**"
     },
     {
@@ -38,13 +38,15 @@
     },
     {
         "action": "change",
-        "when": "7b37e8b23691613f331bd4ebc9d639dd6f93c972",
-        "short": "Improve `--download-sections`\n    - Support negative time-ranges\n    - Add `*from-url` to obey time-ranges in URL"
+        "when": "b4e0d75848e9447cee2cd3646ce54d4744a7ff56",
+        "short": "Improve `--download-sections`\n    - Support negative time-ranges\n    - Add `*from-url` to obey time-ranges in URL",
+        "authors": ["pukkandan"]
     },
     {
         "action": "change",
         "when": "1e75d97db21152acc764b30a688e516f04b8a142",
-        "short": "[extractor/youtube] Add `ios` to default clients used\n        - IOS is affected neither by 403 nor by nsig so helps mitigate them preemptively\n        - IOS also has higher bit-rate 'premium' formats though they are not labeled as such"
+        "short": "[extractor/youtube] Add `ios` to default clients used\n        - IOS is affected neither by 403 nor by nsig so helps mitigate them preemptively\n        - IOS also has higher bit-rate 'premium' formats though they are not labeled as such",
+        "authors": ["pukkandan"]
     },
     {
         "action": "change",
@@ -55,6 +57,12 @@
     {
         "action": "change",
         "when": "a4486bfc1dc7057efca9dd3fe70d7fa25c56f700",
-        "short": "[misc] Revert \"Add automatic duplicate issue detection\""
+        "short": "[misc] Revert \"Add automatic duplicate issue detection\"",
+        "authors": ["pukkandan"]
+    },
+    {
+        "action": "add",
+        "when": "1ceb657bdd254ad961489e5060f2ccc7d556b729",
+        "short": "[priority] Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)\n    - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains\n    - Cookies are scoped when passed to external downloaders\n    - Add `cookie` field to info.json and deprecate `http_headers.Cookie`"
     }
 ]

From cc0619f62d6da52689797483e96b29290b0c0873 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@example.com>
Date: Thu, 6 Jul 2023 18:57:59 +0000
Subject: [PATCH 435/871] Release 2023.07.06

Created by: pukkandan

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |  8 +--
 .../ISSUE_TEMPLATE/2_site_support_request.yml |  8 +--
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |  8 +--
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |  8 +--
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |  8 +--
 .github/ISSUE_TEMPLATE/6_question.yml         |  8 +--
 CONTRIBUTORS                                  |  7 +++
 Changelog.md                                  | 52 +++++++++++++++++++
 supportedsites.md                             |  6 ++-
 yt_dlp/version.py                             |  4 +-
 10 files changed, 90 insertions(+), 27 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index a00a11f27..dd1b33dde 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -64,7 +64,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -72,8 +72,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.22, Current version: 2023.06.22
-        yt-dlp is up to date (2023.06.22)
+        Latest version: 2023.07.06, Current version: 2023.07.06
+        yt-dlp is up to date (2023.07.06)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index fc1f41ead..4f4378924 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -76,7 +76,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -84,8 +84,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.22, Current version: 2023.06.22
-        yt-dlp is up to date (2023.06.22)
+        Latest version: 2023.07.06, Current version: 2023.07.06
+        yt-dlp is up to date (2023.07.06)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index ed51dfa97..05b4dd23b 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -72,7 +72,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -80,8 +80,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.22, Current version: 2023.06.22
-        yt-dlp is up to date (2023.06.22)
+        Latest version: 2023.07.06, Current version: 2023.07.06
+        yt-dlp is up to date (2023.07.06)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index 1c997f3e2..880f1014c 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -57,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -65,8 +65,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.22, Current version: 2023.06.22
-        yt-dlp is up to date (2023.06.22)
+        Latest version: 2023.07.06, Current version: 2023.07.06
+        yt-dlp is up to date (2023.07.06)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index 1638945bf..acb11795f 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -53,7 +53,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -61,7 +61,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.22, Current version: 2023.06.22
-        yt-dlp is up to date (2023.06.22)
+        Latest version: 2023.07.06, Current version: 2023.07.06
+        yt-dlp is up to date (2023.07.06)
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index d27bd5742..a2563e975 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.06.22** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -59,7 +59,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.06.22 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -67,7 +67,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.06.22, Current version: 2023.06.22
-        yt-dlp is up to date (2023.06.22)
+        Latest version: 2023.07.06, Current version: 2023.07.06
+        yt-dlp is up to date (2023.07.06)
         <more lines>
       render: shell
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 0864f16c4..6ccd08931 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -460,3 +460,10 @@ berkanteber
 OverlordQ
 rexlambert22
 Ti4eeT4e
+AmanSal1
+bbilly1
+meliber
+nnoboa
+rdamas
+RfadnjdExt
+urectanc
diff --git a/Changelog.md b/Changelog.md
index d7efa5d25..622ae68b9 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,58 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.07.06
+
+#### Important changes
+- Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)
+    - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains
+    - Cookies are scoped when passed to external downloaders
+    - Add `cookie` field to info.json and deprecate `http_headers.Cookie`
+
+#### Core changes
+- [Allow extractors to mark formats as potentially DRM](https://github.com/yt-dlp/yt-dlp/commit/bc344cd456380999c1ee74554dfd432a38f32ec7) ([#7396](https://github.com/yt-dlp/yt-dlp/issues/7396)) by [pukkandan](https://github.com/pukkandan)
+- [Bugfix for b4e0d75848e9447cee2cd3646ce54d4744a7ff56](https://github.com/yt-dlp/yt-dlp/commit/e59e20744eb32ce4b6ea0dece7c673be8376a710) by [pukkandan](https://github.com/pukkandan)
+- [Change how `Cookie` headers are handled](https://github.com/yt-dlp/yt-dlp/commit/3121512228487c9c690d3d39bfd2579addf96e07) by [Grub4K](https://github.com/Grub4K)
+- [Prevent `Cookie` leaks on HTTP redirect](https://github.com/yt-dlp/yt-dlp/commit/f8b4bcc0a791274223723488bfbfc23ea3276641) by [coletdjnz](https://github.com/coletdjnz)
+- **formats**: [Fix best fallback for storyboards](https://github.com/yt-dlp/yt-dlp/commit/906c0bdcd8974340d619e99ccd613c163eb0d0c2) by [pukkandan](https://github.com/pukkandan)
+- **outtmpl**: [Pad `playlist_index` etc even when with internal formatting](https://github.com/yt-dlp/yt-dlp/commit/47bcd437247152e0af5b3ebc5592db7bb66855c2) by [pukkandan](https://github.com/pukkandan)
+- **utils**: clean_podcast_url: [Handle protocol in redirect URL](https://github.com/yt-dlp/yt-dlp/commit/91302ed349f34dc26cc1d661bb45a4b71f4417f7) by [pukkandan](https://github.com/pukkandan)
+
+#### Extractor changes
+- **abc**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/8f05fbae2a79ce0713077ccc68b354e63216bf20) ([#7434](https://github.com/yt-dlp/yt-dlp/issues/7434)) by [meliber](https://github.com/meliber)
+- **AdultSwim**: [Extract subtitles from m3u8](https://github.com/yt-dlp/yt-dlp/commit/5e16cf92eb496b7c1541a6b1d727cb87542984db) ([#7421](https://github.com/yt-dlp/yt-dlp/issues/7421)) by [nnoboa](https://github.com/nnoboa)
+- **crunchyroll**: music: [Fix `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/5b4b92769afcc398475e481bfa839f1158902fe9) ([#7439](https://github.com/yt-dlp/yt-dlp/issues/7439)) by [AmanSal1](https://github.com/AmanSal1), [rdamas](https://github.com/rdamas)
+- **Douyin**: [Fix extraction from webpage](https://github.com/yt-dlp/yt-dlp/commit/a2be9781fbf4d7e4db245c277ca2ecc41cf3a7b2) by [bashonly](https://github.com/bashonly)
+- **googledrive**: [Fix source format extraction](https://github.com/yt-dlp/yt-dlp/commit/3b7f5300c577fef40464d46d4e4037a69d51fe82) ([#7395](https://github.com/yt-dlp/yt-dlp/issues/7395)) by [RfadnjdExt](https://github.com/RfadnjdExt)
+- **kick**: [Fix `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/ef8509c300ea50da86aea447eb214d3d6f6db6bb) by [bashonly](https://github.com/bashonly)
+- **qdance**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/f0a1ff118145b6449982ba401f9a9f656ecd8062) ([#7420](https://github.com/yt-dlp/yt-dlp/issues/7420)) by [bashonly](https://github.com/bashonly)
+- **sbs**: [Python 3.7 compat](https://github.com/yt-dlp/yt-dlp/commit/f393bbe724b1fc6c7f754a5da507e807b2b40ad2) by [pukkandan](https://github.com/pukkandan)
+- **stacommu**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/af1fd12f675220df6793fc019dff320bc76e8080) ([#7432](https://github.com/yt-dlp/yt-dlp/issues/7432)) by [urectanc](https://github.com/urectanc)
+- **twitter**
+    - [Fix unauthenticated extraction](https://github.com/yt-dlp/yt-dlp/commit/49296437a8e5fa91dacb5446e51ab588474c85d3) ([#7476](https://github.com/yt-dlp/yt-dlp/issues/7476)) by [bashonly](https://github.com/bashonly)
+    - spaces: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/1cffd621cb371f1563563cfb2fe37d137e8a7bee) ([#7512](https://github.com/yt-dlp/yt-dlp/issues/7512)) by [bashonly](https://github.com/bashonly)
+- **vidlii**: [Handle relative URLs](https://github.com/yt-dlp/yt-dlp/commit/ad8902f616ad2541f9b9626738f1393fad89a64c) by [pukkandan](https://github.com/pukkandan)
+- **vk**: VKPlay, VKPlayLive: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/8776349ef6b1f644584a92dfa00a05208a48edc4) ([#7358](https://github.com/yt-dlp/yt-dlp/issues/7358)) by [c-basalt](https://github.com/c-basalt)
+- **youtube**
+    - [Add extractor-arg `formats`](https://github.com/yt-dlp/yt-dlp/commit/58786a10f212bd63f9ad1d0b4d9e4d31c3b385e2) by [pukkandan](https://github.com/pukkandan)
+    - [Avoid false DRM detection](https://github.com/yt-dlp/yt-dlp/commit/94ed638a437fc766699d440e978982e24ce6a30a) ([#7396](https://github.com/yt-dlp/yt-dlp/issues/7396)) by [pukkandan](https://github.com/pukkandan)
+    - [Fix comments' `is_favorited`](https://github.com/yt-dlp/yt-dlp/commit/89bed013741a776506f60380b7fd89d27d0710b4) ([#7390](https://github.com/yt-dlp/yt-dlp/issues/7390)) by [bbilly1](https://github.com/bbilly1)
+    - [Ignore incomplete data for comment threads by default](https://github.com/yt-dlp/yt-dlp/commit/4dc4d8473c085900edc841c87c20041233d25b1f) ([#7475](https://github.com/yt-dlp/yt-dlp/issues/7475)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Process `post_live` over 2 hours](https://github.com/yt-dlp/yt-dlp/commit/d949c10c45bfc359bdacd52e6a180169b8128958) by [pukkandan](https://github.com/pukkandan)
+    - stories: [Remove](https://github.com/yt-dlp/yt-dlp/commit/90db9a3c00ca80492c6a58c542e4cbf4c2710866) ([#7459](https://github.com/yt-dlp/yt-dlp/issues/7459)) by [pukkandan](https://github.com/pukkandan)
+    - tab: [Support shorts-only playlists](https://github.com/yt-dlp/yt-dlp/commit/fcbc9ed760be6e3455bbadfaf277b4504b06f068) ([#7425](https://github.com/yt-dlp/yt-dlp/issues/7425)) by [coletdjnz](https://github.com/coletdjnz)
+
+#### Downloader changes
+- **aria2c**: [Add `--no-conf`](https://github.com/yt-dlp/yt-dlp/commit/8a8af356e3bba98a7f7d333aff0777d5d92130c8) by [pukkandan](https://github.com/pukkandan)
+- **external**: [Scope cookies](https://github.com/yt-dlp/yt-dlp/commit/1ceb657bdd254ad961489e5060f2ccc7d556b729) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz)
+- **http**: [Avoid infinite loop when no data is received](https://github.com/yt-dlp/yt-dlp/commit/662ef1e910b72e57957f06589925b2332ba52821) by [pukkandan](https://github.com/pukkandan)
+
+#### Misc. changes
+- [Add CodeQL workflow](https://github.com/yt-dlp/yt-dlp/commit/6355b5f1e1e8e7f4ef866d71d51e03baf0e82f17) ([#7497](https://github.com/yt-dlp/yt-dlp/issues/7497)) by [pukkandan](https://github.com/pukkandan)
+- **cleanup**: Miscellaneous: [337734d](https://github.com/yt-dlp/yt-dlp/commit/337734d4a8a6500bc65434843db346b5cbd05e81) by [pukkandan](https://github.com/pukkandan)
+- **docs**: [Minor fixes](https://github.com/yt-dlp/yt-dlp/commit/b532a3481046e1eabb6232ee8196fb696c356ff6) by [pukkandan](https://github.com/pukkandan)
+- **make_changelog**: [Skip reverted commits](https://github.com/yt-dlp/yt-dlp/commit/fa44802809d189fca0f4782263d48d6533384503) by [pukkandan](https://github.com/pukkandan)
+
 ### 2023.06.22
 
 #### Core changes
diff --git a/supportedsites.md b/supportedsites.md
index 7d99d9e22..379d28ef3 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -1136,6 +1136,7 @@ # Supported sites
  - **puhutv:serie**
  - **Puls4**
  - **Pyvideo**
+ - **QDance**: [*qdance*](## "netrc machine")
  - **QingTing**
  - **qqmusic**: QQ音乐
  - **qqmusic:album**: QQ音乐 - 专辑
@@ -1363,6 +1364,8 @@ # Supported sites
  - **sr:mediathek**: Saarländischer Rundfunk
  - **SRGSSR**
  - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
+ - **StacommuLive**: [*stacommu*](## "netrc machine")
+ - **StacommuVOD**: [*stacommu*](## "netrc machine")
  - **StagePlusVODConcert**: [*stageplus*](## "netrc machine")
  - **stanfordoc**: Stanford Open ClassRoom
  - **StarTrek**
@@ -1647,6 +1650,8 @@ # Supported sites
  - **vk**: [*vk*](## "netrc machine") VK
  - **vk:uservideos**: [*vk*](## "netrc machine") VK - User's Videos
  - **vk:wallpost**: [*vk*](## "netrc machine")
+ - **VKPlay**
+ - **VKPlayLive**
  - **vm.tiktok**
  - **Vocaroo**
  - **Vodlocker**
@@ -1800,7 +1805,6 @@ # Supported sites
  - **youtube:​search:date**: YouTube search, newest videos first; "ytsearchdate:" prefix
  - **youtube:search_url**: YouTube search URLs with sorting and filter support
  - **youtube:​shorts:pivot:audio**: YouTube Shorts audio pivot (Shorts using audio of a given video)
- - **youtube:stories**: YouTube channel stories; "ytstories:" prefix
  - **youtube:subscriptions**: YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)
  - **youtube:tab**: YouTube Tabs
  - **youtube:user**: YouTube user videos; "ytuser:" prefix
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index 434f36f48..67cfe44ef 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.06.22'
+__version__ = '2023.07.06'
 
-RELEASE_GIT_HEAD = '812cdfa06c33a40e73a8e04b3e6f42c084666a43'
+RELEASE_GIT_HEAD = 'b532a3481046e1eabb6232ee8196fb696c356ff6'
 
 VARIANT = None
 

From b03fa7834579a01cc5fba48c0e73488a16683d48 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 6 Jul 2023 02:00:23 +0530
Subject: [PATCH 436/871] Revert 49296437a8e5fa91dacb5446e51ab588474c85d3

---
 README.md                   |   3 +
 yt_dlp/extractor/twitter.py | 180 ++++++++++++++++++++++--------------
 2 files changed, 113 insertions(+), 70 deletions(-)

diff --git a/README.md b/README.md
index 0526fe418..655cd41f5 100644
--- a/README.md
+++ b/README.md
@@ -1852,6 +1852,9 @@ #### tiktok
 #### rokfinchannel
 * `tab`: Which tab to download - one of `new`, `top`, `videos`, `podcasts`, `streams`, `stacks`
 
+#### twitter
+* `legacy_api`: Force usage of the legacy Twitter API instead of the GraphQL API for tweet extraction. Has no effect if login cookies are passed
+
 #### stacommu, wrestleuniverse
 * `device_id`: UUID value assigned by the website and used to enforce device limits for paid livestream content. Can be found in browser local storage
 
diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index eaf9be526..1fb9524da 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1,6 +1,5 @@
 import json
 import re
-import urllib.error
 
 from .common import InfoExtractor
 from .periscope import PeriscopeBaseIE, PeriscopeIE
@@ -35,6 +34,7 @@ class TwitterBaseIE(InfoExtractor):
     _GRAPHQL_API_BASE = 'https://twitter.com/i/api/graphql/'
     _BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?(?:twitter\.com|twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid\.onion)/'
     _AUTH = {'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'}
+    _guest_token = None
     _flow_token = None
 
     _LOGIN_INIT_DATA = json.dumps({
@@ -145,6 +145,14 @@ def _search_dimensions_in_video_url(a_format, video_url):
     def is_logged_in(self):
         return bool(self._get_cookies(self._API_BASE).get('auth_token'))
 
+    def _fetch_guest_token(self, headers, display_id):
+        headers.pop('x-guest-token', None)
+        self._guest_token = traverse_obj(self._download_json(
+            f'{self._API_BASE}guest/activate.json', display_id,
+            'Downloading guest token', data=b'', headers=headers), 'guest_token')
+        if not self._guest_token:
+            raise ExtractorError('Could not retrieve guest token')
+
     def _set_base_headers(self):
         headers = self._AUTH.copy()
         csrf_token = try_call(lambda: self._get_cookies(self._API_BASE)['ct0'].value)
@@ -175,15 +183,12 @@ def _perform_login(self, username, password):
         if self.is_logged_in:
             return
 
-        webpage = self._download_webpage('https://twitter.com/', None, 'Downloading login page')
+        self._request_webpage('https://twitter.com/', None, 'Requesting cookies')
         headers = self._set_base_headers()
-        guest_token = self._search_regex(
-            r'\.cookie\s*=\s*["\']gt=(\d+);', webpage, 'gt', default=None) or self._download_json(
-            f'{self._API_BASE}guest/activate.json', None, 'Downloading guest token',
-            data=b'', headers=headers)['guest_token']
+        self._fetch_guest_token(headers, None)
         headers.update({
             'content-type': 'application/json',
-            'x-guest-token': guest_token,
+            'x-guest-token': self._guest_token,
             'x-twitter-client-language': 'en',
             'x-twitter-active-user': 'yes',
             'Referer': 'https://twitter.com/',
@@ -280,24 +285,37 @@ def input_dict(subtask_id, text):
         self.report_login()
 
     def _call_api(self, path, video_id, query={}, graphql=False):
-        if not self.is_logged_in:
-            self.raise_login_required()
-
-        result = self._download_json(
-            (self._GRAPHQL_API_BASE if graphql else self._API_BASE) + path, video_id,
-            f'Downloading {"GraphQL" if graphql else "legacy API"} JSON', headers={
-                **self._set_base_headers(),
+        headers = self._set_base_headers()
+        if self.is_logged_in:
+            headers.update({
                 'x-twitter-auth-type': 'OAuth2Session',
                 'x-twitter-client-language': 'en',
                 'x-twitter-active-user': 'yes',
-            }, query=query, expected_status={400, 401, 403, 404} if graphql else {403})
+            })
 
-        if result.get('errors'):
-            errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
-            raise ExtractorError(
-                f'Error(s) while querying API: {errors or "Unknown error"}', expected=True)
+        for first_attempt in (True, False):
+            if not self.is_logged_in:
+                if not self._guest_token:
+                    self._fetch_guest_token(headers, video_id)
+                headers['x-guest-token'] = self._guest_token
 
-        return result
+            allowed_status = {400, 401, 403, 404} if graphql else {403}
+            result = self._download_json(
+                (self._GRAPHQL_API_BASE if graphql else self._API_BASE) + path,
+                video_id, headers=headers, query=query, expected_status=allowed_status,
+                note=f'Downloading {"GraphQL" if graphql else "legacy API"} JSON')
+
+            if result.get('errors'):
+                errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
+                if not self.is_logged_in and first_attempt and 'bad guest token' in errors.lower():
+                    self.to_screen('Guest token has expired. Refreshing guest token')
+                    self._guest_token = None
+                    continue
+
+                raise ExtractorError(
+                    f'Error(s) while querying API: {errors or "Unknown error"}', expected=True)
+
+            return result
 
     def _build_graphql_query(self, media_id):
         raise NotImplementedError('Method must be implemented to support GraphQL')
@@ -439,7 +457,6 @@ class TwitterIE(TwitterBaseIE):
     _VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)(?:/(?:video|photo)/(?P<index>\d+))?'
 
     _TESTS = [{
-        # comment_count, repost_count, view_count are only available with auth (applies to all tests)
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
         'info_dict': {
             'id': '643211870443208704',
@@ -454,7 +471,10 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1442188653,
             'upload_date': '20150913',
             'uploader_url': 'https://twitter.com/freethenipple',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 18,
         },
@@ -485,6 +505,8 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1447395772,
             'upload_date': '20151113',
             'uploader_url': 'https://twitter.com/starwars',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
             'tags': ['TV', 'StarWars', 'TheForceAwakens'],
             'age_limit': 0,
@@ -528,7 +550,10 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1455777459,
             'upload_date': '20160218',
             'uploader_url': 'https://twitter.com/jaydingeer',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': ['Damndaniel'],
             'age_limit': 0,
         },
@@ -566,7 +591,10 @@ class TwitterIE(TwitterBaseIE):
             'upload_date': '20160412',
             'uploader_url': 'https://twitter.com/CaptainAmerica',
             'thumbnail': r're:^https?://.*\.jpg',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -613,7 +641,10 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1505803395,
             'upload_date': '20170919',
             'uploader_url': 'https://twitter.com/Prefet971',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': ['Maria'],
             'age_limit': 0,
         },
@@ -636,7 +667,10 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1527623489,
             'upload_date': '20180529',
             'uploader_url': 'https://twitter.com/LisPower1',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -658,7 +692,10 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1548184644,
             'upload_date': '20190122',
             'uploader_url': 'https://twitter.com/Twitter',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -676,7 +713,6 @@ class TwitterIE(TwitterBaseIE):
             'view_count': int,
         },
         'add_ie': ['TwitterBroadcast'],
-        'skip': 'Requires authentication',
     }, {
         # unified card
         'url': 'https://twitter.com/BrooklynNets/status/1349794411333394432?s=20',
@@ -693,6 +729,8 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1610651040,
             'upload_date': '20210114',
             'uploader_url': 'https://twitter.com/BrooklynNets',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
             'tags': [],
             'age_limit': 0,
@@ -715,7 +753,10 @@ class TwitterIE(TwitterBaseIE):
             'thumbnail': r're:^https?://.*\.jpg',
             'duration': 30.03,
             'timestamp': 1665025050,
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': [],
             'age_limit': 0,
         },
@@ -724,13 +765,15 @@ class TwitterIE(TwitterBaseIE):
         'url': 'https://twitter.com/UltimaShadowX/status/1577719286659006464',
         'info_dict': {
             'id': '1577719286659006464',
-            'title': 'Ultima📛 | #вʟм - Test',
+            'title': 'Ultima | #\u0432\u029f\u043c - Test',
             'description': 'Test https://t.co/Y3KEZD7Dad',
-            'uploader': 'Ultima📛 | #вʟм',
+            'uploader': 'Ultima | #\u0432\u029f\u043c',
             'uploader_id': 'UltimaShadowX',
             'uploader_url': 'https://twitter.com/UltimaShadowX',
             'upload_date': '20221005',
             'timestamp': 1664992565,
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
             'tags': [],
             'age_limit': 0,
@@ -752,7 +795,10 @@ class TwitterIE(TwitterBaseIE):
             'duration': 21.321,
             'timestamp': 1664477766,
             'upload_date': '20220929',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
+            'view_count': int,
             'tags': ['HurricaneIan'],
             'age_limit': 0,
         },
@@ -779,20 +825,6 @@ class TwitterIE(TwitterBaseIE):
         },
         'skip': 'Requires authentication',
     }, {
-        # Single Vimeo video result without auth
-        'url': 'https://twitter.com/Srirachachau/status/1395079556562706435',
-        'info_dict': {
-            'id': '551578322',
-            'ext': 'mp4',
-            'title': 'Dusty & The Mayor',
-            'uploader': 'Michael Chau',
-            'uploader_id': 'user29061007',
-            'uploader_url': 'https://vimeo.com/user29061007',
-            'duration': 478,
-            'thumbnail': 'https://i.vimeocdn.com/video/1139658575-0dfdce6e9a2401fe09feb24bf0d14e6f24a53c12f447ff688ace61009ad4c1ba-d_1280',
-        },
-    }, {
-        # Playlist result only with auth
         'url': 'https://twitter.com/Srirachachau/status/1395079556562706435',
         'playlist_mincount': 2,
         'info_dict': {
@@ -810,7 +842,6 @@ class TwitterIE(TwitterBaseIE):
             'uploader_url': 'https://twitter.com/Srirachachau',
             'timestamp': 1621447860,
         },
-        'skip': 'Requires authentication',
     }, {
         'url': 'https://twitter.com/DavidToons_/status/1578353380363501568',
         'playlist_mincount': 2,
@@ -829,7 +860,6 @@ class TwitterIE(TwitterBaseIE):
             'upload_date': '20221007',
             'age_limit': 0,
         },
-        'skip': 'Requires authentication',
     }, {
         'url': 'https://twitter.com/primevideouk/status/1578401165338976258',
         'playlist_count': 2,
@@ -843,6 +873,8 @@ class TwitterIE(TwitterBaseIE):
             'upload_date': '20221007',
             'age_limit': 0,
             'uploader_url': 'https://twitter.com/primevideouk',
+            'comment_count': int,
+            'repost_count': int,
             'like_count': int,
             'tags': ['TheRingsOfPower'],
         },
@@ -864,7 +896,6 @@ class TwitterIE(TwitterBaseIE):
         },
         'add_ie': ['TwitterSpaces'],
         'params': {'skip_download': 'm3u8'},
-        'skip': 'Requires authentication',
     }, {
         # URL specifies video number but --yes-playlist
         'url': 'https://twitter.com/CTVJLaidlaw/status/1600649710662213632/video/1',
@@ -874,7 +905,9 @@ class TwitterIE(TwitterBaseIE):
             'title': 'md5:be05989b0722e114103ed3851a0ffae2',
             'timestamp': 1670459604.0,
             'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
+            'comment_count': int,
             'uploader_id': 'CTVJLaidlaw',
+            'repost_count': int,
             'tags': ['colorectalcancer', 'cancerjourney', 'imnotaquitter'],
             'upload_date': '20221208',
             'age_limit': 0,
@@ -893,11 +926,14 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1670459604.0,
             'uploader_id': 'CTVJLaidlaw',
             'uploader': 'Jocelyn Laidlaw',
+            'repost_count': int,
+            'comment_count': int,
             'tags': ['colorectalcancer', 'cancerjourney', 'imnotaquitter'],
             'duration': 102.226,
             'uploader_url': 'https://twitter.com/CTVJLaidlaw',
             'display_id': '1600649710662213632',
             'like_count': int,
+            'view_count': int,
             'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
             'upload_date': '20221208',
             'age_limit': 0,
@@ -923,6 +959,9 @@ class TwitterIE(TwitterBaseIE):
             'age_limit': 18,
             'tags': [],
             'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+            'view_count': int,
         },
     }, {
         'url': 'https://twitter.com/hlo_again/status/1599108751385972737/video/2',
@@ -935,7 +974,10 @@ class TwitterIE(TwitterBaseIE):
             'like_count': int,
             'uploader_id': 'hlo_again',
             'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1599108643743473680/pu/img/UG3xjov4rgg5sbYM.jpg?name=orig',
+            'repost_count': int,
             'duration': 9.531,
+            'comment_count': int,
+            'view_count': int,
             'upload_date': '20221203',
             'age_limit': 0,
             'timestamp': 1670092210.0,
@@ -952,11 +994,14 @@ class TwitterIE(TwitterBaseIE):
             'ext': 'mp4',
             'uploader_url': 'https://twitter.com/MunTheShinobi',
             'description': 'This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525 https://t.co/cNsA0MoOml',
+            'view_count': int,
             'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1600009362759733248/pu/img/XVhFQivj75H_YxxV.jpg?name=orig',
             'age_limit': 0,
             'uploader': 'Mün The Shinobi',
+            'repost_count': int,
             'upload_date': '20221206',
             'title': 'Mün The Shinobi - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
+            'comment_count': int,
             'like_count': int,
             'tags': [],
             'uploader_id': 'MunTheShinobi',
@@ -964,14 +1009,14 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1670306984.0,
         },
     }, {
-        # url to retweet id
+        # url to retweet id, legacy API
         'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
         'info_dict': {
             'id': '1623274794488659969',
             'display_id': '1623739803874349067',
             'ext': 'mp4',
             'title': 'Johnny Bullets - Me after going viral to over 30million people:    Whoopsie-daisy',
-            'description': 'md5:224d62f54b0cdef8e33d4c56c41ac503',
+            'description': 'md5:e873616a4a8fe0f93e71872678a672f3',
             'uploader': 'Johnny Bullets',
             'uploader_id': 'Johnnybull3ts',
             'uploader_url': 'https://twitter.com/Johnnybull3ts',
@@ -982,7 +1027,10 @@ class TwitterIE(TwitterBaseIE):
             'upload_date': '20230208',
             'thumbnail': r're:https://pbs\.twimg\.com/ext_tw_video_thumb/.+',
             'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
         },
+        'params': {'extractor_args': {'twitter': {'legacy_api': ['']}}},
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -1033,6 +1081,8 @@ def _graphql_to_legacy(self, data, twid):
 
         if 'tombstone' in result:
             cause = remove_end(traverse_obj(result, ('tombstone', 'text', 'text', {str})), '. Learn more')
+            if cause and 'adult content' in cause:
+                self.raise_login_required(cause)
             raise ExtractorError(f'Twitter API says: {cause or "Unknown error"}', expected=True)
 
         status = result.get('legacy', {})
@@ -1088,22 +1138,19 @@ def _build_graphql_query(self, media_id):
 
     def _real_extract(self, url):
         twid, selected_index = self._match_valid_url(url).group('id', 'index')
-        if not self.is_logged_in:
-            try:
-                status = self._download_json(
-                    'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
-                    headers={'User-Agent': 'Googlebot'}, query={'id': twid})
-                self.to_screen(f'Some metadata is missing without authentication. {self._login_hint()}')
-            except ExtractorError as e:
-                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
-                    self.raise_login_required('Requested tweet may only be available when logged in')
-                raise
+        if self._configuration_arg('legacy_api') and not self.is_logged_in:
+            status = traverse_obj(self._call_api(f'statuses/show/{twid}.json', twid, {
+                'cards_platform': 'Web-12',
+                'include_cards': 1,
+                'include_reply_count': 1,
+                'include_user_entities': 0,
+                'tweet_mode': 'extended',
+            }), 'retweeted_status', None)
         else:
-            status = self._graphql_to_legacy(
-                self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid), twid)
+            result = self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid)
+            status = self._graphql_to_legacy(result, twid)
 
-        title = description = traverse_obj(
-            status, (('full_text', 'text'), {lambda x: x.replace('\n', ' ')}), get_all=False) or ''
+        title = description = status['full_text'].replace('\n', ' ')
         # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
         title = re.sub(r'\s+(https?://[^ ]+)', '', title)
         user = status.get('user') or {}
@@ -1129,16 +1176,12 @@ def _real_extract(self, url):
 
         def extract_from_video_info(media):
             media_id = traverse_obj(media, 'id_str', 'id', expected_type=str_or_none)
-            if not media_id:
-                # workaround for non-authenticated responses
-                media_id = traverse_obj(media, (
-                    'video_info', 'variants', ..., 'url',
-                    {lambda x: re.search(r'_video/(\d+)/', x)[1]}), get_all=False)
             self.write_debug(f'Extracting from video info: {media_id}')
+            video_info = media.get('video_info') or {}
 
             formats = []
             subtitles = {}
-            for variant in traverse_obj(media, ('video_info', 'variants', ...)):
+            for variant in video_info.get('variants', []):
                 fmts, subs = self._extract_variant_formats(variant, twid)
                 subtitles = self._merge_subtitles(subtitles, subs)
                 formats.extend(fmts)
@@ -1158,12 +1201,12 @@ def add_thumbnail(name, size):
                 add_thumbnail('orig', media.get('original_info') or {})
 
             return {
-                'id': media_id or twid,
+                'id': media_id,
                 'formats': formats,
                 'subtitles': subtitles,
                 'thumbnails': thumbnails,
                 'view_count': traverse_obj(media, ('mediaStats', 'viewCount', {int_or_none})),
-                'duration': float_or_none(traverse_obj(media, ('video_info', 'duration_millis')), 1000),
+                'duration': float_or_none(video_info.get('duration_millis'), 1000),
                 # The codec of http formats are unknown
                 '_format_sort_fields': ('res', 'br', 'size', 'proto'),
             }
@@ -1243,15 +1286,12 @@ def get_binding_value(k):
                 }
 
         videos = traverse_obj(status, (
-            ('mediaDetails', ((None, 'quoted_status'), 'extended_entities', 'media')),
-            lambda _, m: m['type'] != 'photo', {dict}))
+            (None, 'quoted_status'), 'extended_entities', 'media', lambda _, m: m['type'] != 'photo', {dict}))
 
         if self._yes_playlist(twid, selected_index, video_label='URL-specified video number'):
             selected_entries = (*map(extract_from_video_info, videos), *extract_from_card_info(status.get('card')))
         else:
-            desired_obj = traverse_obj(status, (
-                ('mediaDetails', ((None, 'quoted_status'), 'extended_entities', 'media')),
-                int(selected_index) - 1, {dict}), get_all=False)
+            desired_obj = traverse_obj(status, ('extended_entities', 'media', int(selected_index) - 1, {dict}))
             if not desired_obj:
                 raise ExtractorError(f'Video #{selected_index} is unavailable', expected=True)
             elif desired_obj.get('type') != 'video':

From 92315c03774cfabb3a921884326beb4b981f786b Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 6 Jul 2023 14:39:51 -0500
Subject: [PATCH 437/871] [extractor/twitter] Fix GraphQL and legacy API
 (#7516)

Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 191 +++++++++++++++++++++++-------------
 1 file changed, 124 insertions(+), 67 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 1fb9524da..fc157ac22 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -11,6 +11,7 @@
 from ..utils import (
     ExtractorError,
     dict_get,
+    filter_dict,
     float_or_none,
     format_field,
     int_or_none,
@@ -33,8 +34,8 @@ class TwitterBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitter.com/1.1/'
     _GRAPHQL_API_BASE = 'https://twitter.com/i/api/graphql/'
     _BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?(?:twitter\.com|twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid\.onion)/'
-    _AUTH = {'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'}
-    _guest_token = None
+    _AUTH = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
+    _LEGACY_AUTH = 'AAAAAAAAAAAAAAAAAAAAAIK1zgAAAAAA2tUWuhGZ2JceoId5GwYWU5GspY4%3DUq7gzFoCZs1QfwGoVdvSac3IniczZEYXIcDyumCauIXpcAPorE'
     _flow_token = None
 
     _LOGIN_INIT_DATA = json.dumps({
@@ -145,20 +146,21 @@ def _search_dimensions_in_video_url(a_format, video_url):
     def is_logged_in(self):
         return bool(self._get_cookies(self._API_BASE).get('auth_token'))
 
-    def _fetch_guest_token(self, headers, display_id):
-        headers.pop('x-guest-token', None)
-        self._guest_token = traverse_obj(self._download_json(
-            f'{self._API_BASE}guest/activate.json', display_id,
-            'Downloading guest token', data=b'', headers=headers), 'guest_token')
-        if not self._guest_token:
+    def _fetch_guest_token(self, display_id):
+        guest_token = traverse_obj(self._download_json(
+            f'{self._API_BASE}guest/activate.json', display_id, 'Downloading guest token', data=b'',
+            headers=self._set_base_headers(legacy=display_id and self._configuration_arg('legacy_api'))),
+            ('guest_token', {str}))
+        if not guest_token:
             raise ExtractorError('Could not retrieve guest token')
+        return guest_token
 
-    def _set_base_headers(self):
-        headers = self._AUTH.copy()
-        csrf_token = try_call(lambda: self._get_cookies(self._API_BASE)['ct0'].value)
-        if csrf_token:
-            headers['x-csrf-token'] = csrf_token
-        return headers
+    def _set_base_headers(self, legacy=False):
+        bearer_token = self._LEGACY_AUTH if legacy and not self.is_logged_in else self._AUTH
+        return filter_dict({
+            'Authorization': f'Bearer {bearer_token}',
+            'x-csrf-token': try_call(lambda: self._get_cookies(self._API_BASE)['ct0'].value),
+        })
 
     def _call_login_api(self, note, headers, query={}, data=None):
         response = self._download_json(
@@ -183,17 +185,18 @@ def _perform_login(self, username, password):
         if self.is_logged_in:
             return
 
-        self._request_webpage('https://twitter.com/', None, 'Requesting cookies')
-        headers = self._set_base_headers()
-        self._fetch_guest_token(headers, None)
-        headers.update({
+        webpage = self._download_webpage('https://twitter.com/', None, 'Downloading login page')
+        guest_token = self._search_regex(
+            r'\.cookie\s*=\s*["\']gt=(\d+);', webpage, 'gt', default=None) or self._fetch_guest_token(None)
+        headers = {
+            **self._set_base_headers(),
             'content-type': 'application/json',
-            'x-guest-token': self._guest_token,
+            'x-guest-token': guest_token,
             'x-twitter-client-language': 'en',
             'x-twitter-active-user': 'yes',
             'Referer': 'https://twitter.com/',
             'Origin': 'https://twitter.com',
-        })
+        }
 
         def build_login_json(*subtask_inputs):
             return json.dumps({
@@ -285,37 +288,26 @@ def input_dict(subtask_id, text):
         self.report_login()
 
     def _call_api(self, path, video_id, query={}, graphql=False):
-        headers = self._set_base_headers()
-        if self.is_logged_in:
-            headers.update({
-                'x-twitter-auth-type': 'OAuth2Session',
-                'x-twitter-client-language': 'en',
-                'x-twitter-active-user': 'yes',
-            })
+        headers = self._set_base_headers(legacy=not graphql and self._configuration_arg('legacy_api'))
+        headers.update({
+            'x-twitter-auth-type': 'OAuth2Session',
+            'x-twitter-client-language': 'en',
+            'x-twitter-active-user': 'yes',
+        } if self.is_logged_in else {
+            'x-guest-token': self._fetch_guest_token(video_id)
+        })
+        allowed_status = {400, 401, 403, 404} if graphql else {403}
+        result = self._download_json(
+            (self._GRAPHQL_API_BASE if graphql else self._API_BASE) + path,
+            video_id, headers=headers, query=query, expected_status=allowed_status,
+            note=f'Downloading {"GraphQL" if graphql else "legacy API"} JSON')
 
-        for first_attempt in (True, False):
-            if not self.is_logged_in:
-                if not self._guest_token:
-                    self._fetch_guest_token(headers, video_id)
-                headers['x-guest-token'] = self._guest_token
+        if result.get('errors'):
+            errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
+            raise ExtractorError(
+                f'Error(s) while querying API: {errors or "Unknown error"}', expected=True)
 
-            allowed_status = {400, 401, 403, 404} if graphql else {403}
-            result = self._download_json(
-                (self._GRAPHQL_API_BASE if graphql else self._API_BASE) + path,
-                video_id, headers=headers, query=query, expected_status=allowed_status,
-                note=f'Downloading {"GraphQL" if graphql else "legacy API"} JSON')
-
-            if result.get('errors'):
-                errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
-                if not self.is_logged_in and first_attempt and 'bad guest token' in errors.lower():
-                    self.to_screen('Guest token has expired. Refreshing guest token')
-                    self._guest_token = None
-                    continue
-
-                raise ExtractorError(
-                    f'Error(s) while querying API: {errors or "Unknown error"}', expected=True)
-
-            return result
+        return result
 
     def _build_graphql_query(self, media_id):
         raise NotImplementedError('Method must be implemented to support GraphQL')
@@ -765,9 +757,9 @@ class TwitterIE(TwitterBaseIE):
         'url': 'https://twitter.com/UltimaShadowX/status/1577719286659006464',
         'info_dict': {
             'id': '1577719286659006464',
-            'title': 'Ultima | #\u0432\u029f\u043c - Test',
+            'title': 'Ultima📛 | #вʟм - Test',
             'description': 'Test https://t.co/Y3KEZD7Dad',
-            'uploader': 'Ultima | #\u0432\u029f\u043c',
+            'uploader': 'Ultima📛 | #вʟм',
             'uploader_id': 'UltimaShadowX',
             'uploader_url': 'https://twitter.com/UltimaShadowX',
             'upload_date': '20221005',
@@ -825,6 +817,7 @@ class TwitterIE(TwitterBaseIE):
         },
         'skip': 'Requires authentication',
     }, {
+        # Playlist result only with auth
         'url': 'https://twitter.com/Srirachachau/status/1395079556562706435',
         'playlist_mincount': 2,
         'info_dict': {
@@ -896,6 +889,7 @@ class TwitterIE(TwitterBaseIE):
         },
         'add_ie': ['TwitterSpaces'],
         'params': {'skip_download': 'm3u8'},
+        'skip': 'Requires authentication',
     }, {
         # URL specifies video number but --yes-playlist
         'url': 'https://twitter.com/CTVJLaidlaw/status/1600649710662213632/video/1',
@@ -1009,14 +1003,14 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1670306984.0,
         },
     }, {
-        # url to retweet id, legacy API
+        # url to retweet id w/ legacy api
         'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
         'info_dict': {
             'id': '1623274794488659969',
             'display_id': '1623739803874349067',
             'ext': 'mp4',
             'title': 'Johnny Bullets - Me after going viral to over 30million people:    Whoopsie-daisy',
-            'description': 'md5:e873616a4a8fe0f93e71872678a672f3',
+            'description': 'md5:b06864cd3dc2554821cc327f5348485a',
             'uploader': 'Johnny Bullets',
             'uploader_id': 'Johnnybull3ts',
             'uploader_url': 'https://twitter.com/Johnnybull3ts',
@@ -1028,9 +1022,31 @@ class TwitterIE(TwitterBaseIE):
             'thumbnail': r're:https://pbs\.twimg\.com/ext_tw_video_thumb/.+',
             'like_count': int,
             'repost_count': int,
-            'comment_count': int,
         },
         'params': {'extractor_args': {'twitter': {'legacy_api': ['']}}},
+    }, {
+        # orig tweet w/ graphql
+        'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
+        'info_dict': {
+            'id': '1623274794488659969',
+            'display_id': '1623739803874349067',
+            'ext': 'mp4',
+            'title': '@selfisekai@hackerspace.pl 🐀 - RT @Johnnybull3ts: Me after going viral to over 30million people:    Whoopsie-daisy',
+            'description': 'md5:9258bdbb54793bdc124fe1cd47e96c6a',
+            'uploader': '@selfisekai@hackerspace.pl 🐀',
+            'uploader_id': 'liberdalau',
+            'uploader_url': 'https://twitter.com/liberdalau',
+            'age_limit': 0,
+            'tags': [],
+            'duration': 8.033,
+            'timestamp': 1675964711.0,
+            'upload_date': '20230209',
+            'thumbnail': r're:https://pbs\.twimg\.com/ext_tw_video_thumb/.+',
+            'like_count': int,
+            'view_count': int,
+            'repost_count': int,
+            'comment_count': int,
+        },
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -1073,17 +1089,21 @@ def _graphql_to_legacy(self, data, twid):
         result = traverse_obj(data, (
             'threaded_conversation_with_injections_v2', 'instructions', 0, 'entries',
             lambda _, v: v['entryId'] == f'tweet-{twid}', 'content', 'itemContent',
-            'tweet_results', 'result', ('tweet', None),
-        ), expected_type=dict, default={}, get_all=False)
+            'tweet_results', 'result', ('tweet', None), {dict},
+        ), default={}, get_all=False) if self.is_logged_in else traverse_obj(
+            data, ('tweetResult', 'result', {dict}), default={})
 
-        if result.get('__typename') not in ('Tweet', 'TweetTombstone', None):
+        if result.get('__typename') not in ('Tweet', 'TweetTombstone', 'TweetUnavailable', None):
             self.report_warning(f'Unknown typename: {result.get("__typename")}', twid, only_once=True)
 
         if 'tombstone' in result:
             cause = remove_end(traverse_obj(result, ('tombstone', 'text', 'text', {str})), '. Learn more')
-            if cause and 'adult content' in cause:
-                self.raise_login_required(cause)
             raise ExtractorError(f'Twitter API says: {cause or "Unknown error"}', expected=True)
+        elif result.get('__typename') == 'TweetUnavailable':
+            reason = result.get('reason')
+            if reason == 'NsfwLoggedOut':
+                self.raise_login_required('NSFW tweet requires authentication')
+            raise ExtractorError(reason or 'Requested tweet is unavailable', expected=True)
 
         status = result.get('legacy', {})
         status.update(traverse_obj(result, {
@@ -1134,11 +1154,42 @@ def _build_graphql_query(self, media_id):
                 'verified_phone_label_enabled': False,
                 'vibe_api_enabled': True,
             },
+        } if self.is_logged_in else {
+            'variables': {
+                'tweetId': media_id,
+                'withCommunity': False,
+                'includePromotedContent': False,
+                'withVoice': False,
+            },
+            'features': {
+                'creator_subscriptions_tweet_preview_api_enabled': True,
+                'tweetypie_unmention_optimization_enabled': True,
+                'responsive_web_edit_tweet_api_enabled': True,
+                'graphql_is_translatable_rweb_tweet_is_translatable_enabled': True,
+                'view_counts_everywhere_api_enabled': True,
+                'longform_notetweets_consumption_enabled': True,
+                'responsive_web_twitter_article_tweet_consumption_enabled': False,
+                'tweet_awards_web_tipping_enabled': False,
+                'freedom_of_speech_not_reach_fetch_enabled': True,
+                'standardized_nudges_misinfo': True,
+                'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': True,
+                'longform_notetweets_rich_text_read_enabled': True,
+                'longform_notetweets_inline_media_enabled': True,
+                'responsive_web_graphql_exclude_directive_enabled': True,
+                'verified_phone_label_enabled': False,
+                'responsive_web_media_download_video_enabled': False,
+                'responsive_web_graphql_skip_user_profile_image_extensions_enabled': False,
+                'responsive_web_graphql_timeline_navigation_enabled': True,
+                'responsive_web_enhance_cards_enabled': False
+            },
+            'fieldToggles': {
+                'withArticleRichContentState': False
+            }
         }
 
     def _real_extract(self, url):
         twid, selected_index = self._match_valid_url(url).group('id', 'index')
-        if self._configuration_arg('legacy_api') and not self.is_logged_in:
+        if not self.is_logged_in and self._configuration_arg('legacy_api'):
             status = traverse_obj(self._call_api(f'statuses/show/{twid}.json', twid, {
                 'cards_platform': 'Web-12',
                 'include_cards': 1,
@@ -1146,11 +1197,15 @@ def _real_extract(self, url):
                 'include_user_entities': 0,
                 'tweet_mode': 'extended',
             }), 'retweeted_status', None)
+        elif not self.is_logged_in:
+            status = self._graphql_to_legacy(
+                self._call_graphql_api('2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId', twid), twid)
         else:
-            result = self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid)
-            status = self._graphql_to_legacy(result, twid)
+            status = self._graphql_to_legacy(
+                self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid), twid)
 
-        title = description = status['full_text'].replace('\n', ' ')
+        title = description = traverse_obj(
+            status, (('full_text', 'text'), {lambda x: x.replace('\n', ' ')}), get_all=False) or ''
         # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
         title = re.sub(r'\s+(https?://[^ ]+)', '', title)
         user = status.get('user') or {}
@@ -1177,11 +1232,10 @@ def _real_extract(self, url):
         def extract_from_video_info(media):
             media_id = traverse_obj(media, 'id_str', 'id', expected_type=str_or_none)
             self.write_debug(f'Extracting from video info: {media_id}')
-            video_info = media.get('video_info') or {}
 
             formats = []
             subtitles = {}
-            for variant in video_info.get('variants', []):
+            for variant in traverse_obj(media, ('video_info', 'variants', ...)):
                 fmts, subs = self._extract_variant_formats(variant, twid)
                 subtitles = self._merge_subtitles(subtitles, subs)
                 formats.extend(fmts)
@@ -1206,7 +1260,7 @@ def add_thumbnail(name, size):
                 'subtitles': subtitles,
                 'thumbnails': thumbnails,
                 'view_count': traverse_obj(media, ('mediaStats', 'viewCount', {int_or_none})),
-                'duration': float_or_none(video_info.get('duration_millis'), 1000),
+                'duration': float_or_none(traverse_obj(media, ('video_info', 'duration_millis')), 1000),
                 # The codec of http formats are unknown
                 '_format_sort_fields': ('res', 'br', 'size', 'proto'),
             }
@@ -1291,7 +1345,8 @@ def get_binding_value(k):
         if self._yes_playlist(twid, selected_index, video_label='URL-specified video number'):
             selected_entries = (*map(extract_from_video_info, videos), *extract_from_card_info(status.get('card')))
         else:
-            desired_obj = traverse_obj(status, ('extended_entities', 'media', int(selected_index) - 1, {dict}))
+            desired_obj = traverse_obj(status, (
+                (None, 'quoted_status'), 'extended_entities', 'media', int(selected_index) - 1, {dict}), get_all=False)
             if not desired_obj:
                 raise ExtractorError(f'Video #{selected_index} is unavailable', expected=True)
             elif desired_obj.get('type') != 'video':
@@ -1481,6 +1536,8 @@ def _build_graphql_query(self, space_id):
 
     def _real_extract(self, url):
         space_id = self._match_id(url)
+        if not self.is_logged_in:
+            self.raise_login_required('Twitter Spaces require authentication')
         space_data = self._call_graphql_api('HPEisOmj1epUNLCWTYhUWw/AudioSpaceById', space_id)['audioSpace']
         if not space_data:
             raise ExtractorError('Twitter Space not found', expected=True)

From bdd0b75e3f41ff35440eda6d395008beef19ef2f Mon Sep 17 00:00:00 2001
From: GD-Slime <82302542+GD-Slime@users.noreply.github.com>
Date: Sun, 9 Jul 2023 06:26:03 +0800
Subject: [PATCH 438/871] [ie/BiliBiliBangumi] Fix extractors (#7337)

- Overhaul BiliBiliBangumi extractor for the site's new API
- Add BiliBiliBangumiSeason extractor
- Refactor BiliBiliBangumiMedia extractor

Closes #6701, Closes #7400
Authored by: GD-Slime
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/bilibili.py    | 129 +++++++++++++++++++++-----------
 2 files changed, 85 insertions(+), 45 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index c0a330dbe..1e7f165ab 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -214,6 +214,7 @@
 from .bilibili import (
     BiliBiliIE,
     BiliBiliBangumiIE,
+    BiliBiliBangumiSeasonIE,
     BiliBiliBangumiMediaIE,
     BiliBiliSearchIE,
     BilibiliCategoryIE,
diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 6629fbc08..e8714a33a 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -18,6 +18,7 @@
     float_or_none,
     format_field,
     int_or_none,
+    join_nonempty,
     make_archive_id,
     merge_dicts,
     mimetype2ext,
@@ -135,6 +136,17 @@ def _get_all_children(self, reply):
         for children in map(self._get_all_children, traverse_obj(reply, ('replies', ...))):
             yield from children
 
+    def _get_episodes_from_season(self, ss_id, url):
+        season_info = self._download_json(
+            'https://api.bilibili.com/pgc/web/season/section', ss_id,
+            note='Downloading season info', query={'season_id': ss_id},
+            headers={'Referer': url, **self.geo_verification_headers()})
+
+        for entry in traverse_obj(season_info, (
+                'result', 'main_section', 'episodes',
+                lambda _, v: url_or_none(v['share_url']) and v['id'])):
+            yield self.url_result(entry['share_url'], BiliBiliBangumiIE, f'ep{entry["id"]}')
+
 
 class BiliBiliIE(BilibiliBaseIE):
     _VALID_URL = r'https?://www\.bilibili\.com/(?:video/|festival/\w+\?(?:[^#]*&)?bvid=)[aAbB][vV](?P<id>[^/?#&]+)'
@@ -403,76 +415,93 @@ def _real_extract(self, url):
 
 
 class BiliBiliBangumiIE(BilibiliBaseIE):
-    _VALID_URL = r'(?x)https?://www\.bilibili\.com/bangumi/play/(?P<id>(?:ss|ep)\d+)'
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/bangumi/play/(?P<id>ep\d+)'
 
     _TESTS = [{
-        'url': 'https://www.bilibili.com/bangumi/play/ss897',
+        'url': 'https://www.bilibili.com/bangumi/play/ep267851',
         'info_dict': {
-            'id': 'ss897',
+            'id': '267851',
             'ext': 'mp4',
-            'series': '神的记事本',
-            'season': '神的记事本',
-            'season_id': 897,
+            'series': '鬼灭之刃',
+            'series_id': '4358',
+            'season': '鬼灭之刃',
+            'season_id': '26801',
             'season_number': 1,
-            'episode': '你与旅行包',
-            'episode_number': 2,
-            'title': '神的记事本：第2话 你与旅行包',
-            'duration': 1428.487,
-            'timestamp': 1310809380,
-            'upload_date': '20110716',
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'episode': '残酷',
+            'episode_id': '267851',
+            'episode_number': 1,
+            'title': '1 残酷',
+            'duration': 1425.256,
+            'timestamp': 1554566400,
+            'upload_date': '20190406',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
         },
-    }, {
-        'url': 'https://www.bilibili.com/bangumi/play/ep508406',
-        'only_matching': True,
+        'skip': 'According to the copyright owner\'s request, you may only watch the video after you are premium member.'
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        episode_id = video_id[2:]
         webpage = self._download_webpage(url, video_id)
 
         if '您所在的地区无法观看本片' in webpage:
             raise GeoRestrictedError('This video is restricted')
-        elif ('开通大会员观看' in webpage and '__playinfo__' not in webpage
-                or '正在观看预览，大会员免费看全片' in webpage):
+        elif '正在观看预览，大会员免费看全片' in webpage:
             self.raise_login_required('This video is for premium members only')
 
-        play_info = self._search_json(r'window\.__playinfo__\s*=', webpage, 'play info', video_id)['data']
+        headers = {'Referer': url, **self.geo_verification_headers()}
+        play_info = self._download_json(
+            'https://api.bilibili.com/pgc/player/web/v2/playurl', video_id,
+            'Extracting episode', query={'fnval': '4048', 'ep_id': episode_id},
+            headers=headers)
+        premium_only = play_info.get('code') == -10403
+        play_info = traverse_obj(play_info, ('result', 'video_info', {dict})) or {}
+
         formats = self.extract_formats(play_info)
-        if (not formats and '成为大会员抢先看' in webpage
-                and play_info.get('durl') and not play_info.get('dash')):
+        if not formats and (premium_only or '成为大会员抢先看' in webpage or '开通大会员观看' in webpage):
             self.raise_login_required('This video is for premium members only')
 
-        initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', video_id)
+        bangumi_info = self._download_json(
+            'https://api.bilibili.com/pgc/view/web/season', video_id, 'Get episode details',
+            query={'ep_id': episode_id}, headers=headers)['result']
 
-        season_id = traverse_obj(initial_state, ('mediaInfo', 'season_id'))
+        episode_number, episode_info = next((
+            (idx, ep) for idx, ep in enumerate(traverse_obj(
+                bangumi_info, ('episodes', ..., {dict})), 1)
+            if str_or_none(ep.get('id')) == episode_id), (1, {}))
+
+        season_id = bangumi_info.get('season_id')
         season_number = season_id and next((
             idx + 1 for idx, e in enumerate(
-                traverse_obj(initial_state, ('mediaInfo', 'seasons', ...)))
+                traverse_obj(bangumi_info, ('seasons', ...)))
             if e.get('season_id') == season_id
         ), None)
 
+        aid = episode_info.get('aid')
+
         return {
             'id': video_id,
             'formats': formats,
-            'title': traverse_obj(initial_state, 'h1Title'),
-            'episode': traverse_obj(initial_state, ('epInfo', 'long_title')),
-            'episode_number': int_or_none(traverse_obj(initial_state, ('epInfo', 'title'))),
-            'series': traverse_obj(initial_state, ('mediaInfo', 'series')),
-            'season': traverse_obj(initial_state, ('mediaInfo', 'season_title')),
-            'season_id': season_id,
+            **traverse_obj(bangumi_info, {
+                'series': ('series', 'series_title', {str}),
+                'series_id': ('series', 'series_id', {str_or_none}),
+                'thumbnail': ('square_cover', {url_or_none}),
+            }),
+            'title': join_nonempty('title', 'long_title', delim=' ', from_dict=episode_info),
+            'episode': episode_info.get('long_title'),
+            'episode_id': episode_id,
+            'episode_number': int_or_none(episode_info.get('title')) or episode_number,
+            'season_id': str_or_none(season_id),
             'season_number': season_number,
-            'thumbnail': traverse_obj(initial_state, ('epInfo', 'cover')),
-            'timestamp': traverse_obj(initial_state, ('epInfo', 'pub_time')),
+            'timestamp': int_or_none(episode_info.get('pub_time')),
             'duration': float_or_none(play_info.get('timelength'), scale=1000),
-            'subtitles': self.extract_subtitles(
-                video_id, initial_state, traverse_obj(initial_state, ('epInfo', 'cid'))),
-            '__post_extractor': self.extract_comments(traverse_obj(initial_state, ('epInfo', 'aid'))),
-            'http_headers': {'Referer': url, **self.geo_verification_headers()},
+            'subtitles': self.extract_subtitles(video_id, aid, episode_info.get('cid')),
+            '__post_extractor': self.extract_comments(aid),
+            'http_headers': headers,
         }
 
 
-class BiliBiliBangumiMediaIE(InfoExtractor):
+class BiliBiliBangumiMediaIE(BilibiliBaseIE):
     _VALID_URL = r'https?://www\.bilibili\.com/bangumi/media/md(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.bilibili.com/bangumi/media/md24097891',
@@ -485,16 +514,26 @@ class BiliBiliBangumiMediaIE(InfoExtractor):
     def _real_extract(self, url):
         media_id = self._match_id(url)
         webpage = self._download_webpage(url, media_id)
+        ss_id = self._search_json(
+            r'window\.__INITIAL_STATE__\s*=', webpage, 'initial_state', media_id)['mediaInfo']['season_id']
 
-        initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial_state', media_id)
-        episode_list = self._download_json(
-            'https://api.bilibili.com/pgc/web/season/section', media_id,
-            query={'season_id': initial_state['mediaInfo']['season_id']},
-            note='Downloading season info')['result']['main_section']['episodes']
+        return self.playlist_result(self._get_episodes_from_season(ss_id, url), media_id)
 
-        return self.playlist_result((
-            self.url_result(entry['share_url'], BiliBiliBangumiIE, entry['aid'])
-            for entry in episode_list), media_id)
+
+class BiliBiliBangumiSeasonIE(BilibiliBaseIE):
+    _VALID_URL = r'(?x)https?://www\.bilibili\.com/bangumi/play/ss(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.bilibili.com/bangumi/play/ss26801',
+        'info_dict': {
+            'id': '26801'
+        },
+        'playlist_mincount': 26
+    }]
+
+    def _real_extract(self, url):
+        ss_id = self._match_id(url)
+
+        return self.playlist_result(self._get_episodes_from_season(ss_id, url), ss_id)
 
 
 class BilibiliSpaceBaseIE(InfoExtractor):

From 325191d0c9bf3fe257b8a7c2eb95080f44f6ddfc Mon Sep 17 00:00:00 2001
From: Zprokkel <105783800+Zprokkel@users.noreply.github.com>
Date: Mon, 10 Jul 2023 15:15:47 +0200
Subject: [PATCH 439/871] [ie/vrt] Update token signing key (#7519)

Authored by: Zprokkel
---
 yt_dlp/extractor/vrt.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/vrt.py b/yt_dlp/extractor/vrt.py
index bacd3df29..005835712 100644
--- a/yt_dlp/extractor/vrt.py
+++ b/yt_dlp/extractor/vrt.py
@@ -44,9 +44,11 @@ class VRTBaseIE(GigyaBaseIE):
             'version': '2.7.4-prod-2023-04-19T06:05:45'
         }
     }
-    # From https://player.vrt.be/vrtnws/js/main.js & https://player.vrt.be/ketnet/js/main.fd1de01a40a1e3d842ea.js
+    # From https://player.vrt.be/vrtnws/js/main.js & https://player.vrt.be/ketnet/js/main.8cdb11341bcb79e4cd44.js
     _JWT_KEY_ID = '0-0Fp51UZykfaiCJrfTE3+oMI8zvDteYfPtR+2n1R+z8w='
-    _JWT_SIGNING_KEY = '2a9251d782700769fb856da5725daf38661874ca6f80ae7dc2b05ec1a81a24ae'
+    _JWT_SIGNING_KEY = 'b5f500d55cb44715107249ccd8a5c0136cfb2788dbb71b90a4f142423bacaf38'  # -dev
+    # player-stag.vrt.be key:    d23987504521ae6fbf2716caca6700a24bb1579477b43c84e146b279de5ca595
+    # player.vrt.be key:         2a9251d782700769fb856da5725daf38661874ca6f80ae7dc2b05ec1a81a24ae
 
     def _extract_formats_and_subtitles(self, data, video_id):
         if traverse_obj(data, 'drm'):

From 2af4eeb77246b8183aae75a0a8d19f18c08115b2 Mon Sep 17 00:00:00 2001
From: Mahmoud Abdel-Fattah <accounts@abdel-fattah.net>
Date: Tue, 11 Jul 2023 05:00:38 +0400
Subject: [PATCH 440/871] [utils] `clean_podcast_url`: Handle more trackers
 (#7556)

Authored by: mabdelfattah, bashonly
Closes #7544
---
 test/test_utils.py     |  2 ++
 yt_dlp/utils/_utils.py | 10 +++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index a22f25d73..bdbd2d879 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1835,6 +1835,8 @@ def test_iri_to_uri(self):
     def test_clean_podcast_url(self):
         self.assertEqual(clean_podcast_url('https://www.podtrac.com/pts/redirect.mp3/chtbl.com/track/5899E/traffic.megaphone.fm/HSW7835899191.mp3'), 'https://traffic.megaphone.fm/HSW7835899191.mp3')
         self.assertEqual(clean_podcast_url('https://play.podtrac.com/npr-344098539/edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3'), 'https://edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3')
+        self.assertEqual(clean_podcast_url('https://pdst.fm/e/2.gum.fm/chtbl.com/track/chrt.fm/track/34D33/pscrb.fm/rss/p/traffic.megaphone.fm/ITLLC7765286967.mp3?updated=1687282661'), 'https://traffic.megaphone.fm/ITLLC7765286967.mp3?updated=1687282661')
+        self.assertEqual(clean_podcast_url('https://pdst.fm/e/https://mgln.ai/e/441/www.buzzsprout.com/1121972/13019085-ep-252-the-deep-life-stack.mp3'), 'https://www.buzzsprout.com/1121972/13019085-ep-252-the-deep-life-stack.mp3')
 
     def test_LazyList(self):
         it = list(range(10))
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 82d9ba4d5..3023c33b2 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -5123,14 +5123,18 @@ def clean_podcast_url(url):
             (?:
                 chtbl\.com/track|
                 media\.blubrry\.com| # https://create.blubrry.com/resources/podcast-media-download-statistics/getting-started/
-                play\.podtrac\.com
-            )/[^/]+|
+                play\.podtrac\.com|
+                chrt\.fm/track|
+                mgln\.ai/e
+            )(?:/[^/.]+)?|
             (?:dts|www)\.podtrac\.com/(?:pts/)?redirect\.[0-9a-z]{3,4}| # http://analytics.podtrac.com/how-to-measure
             flex\.acast\.com|
             pd(?:
                 cn\.co| # https://podcorn.com/analytics-prefix/
                 st\.fm # https://podsights.com/docs/
-            )/e
+            )/e|
+            [0-9]\.gum\.fm|
+            pscrb\.fm/rss/p
         )/''', '', url)
     return re.sub(r'^\w+://(\w+://)', r'\1', url)
 

From 2cfe221fbbe46faa3f46552c08d947a51f424903 Mon Sep 17 00:00:00 2001
From: Aleri Kaisattera <73682764+alerikaisattera@users.noreply.github.com>
Date: Thu, 13 Jul 2023 20:17:05 +0600
Subject: [PATCH 441/871] [ie/streamanity] Remove (#7571)

Service is dead
Authored by: alerikaisattera
---
 yt_dlp/extractor/_extractors.py |  1 -
 yt_dlp/extractor/streamanity.py | 47 ---------------------------------
 2 files changed, 48 deletions(-)
 delete mode 100644 yt_dlp/extractor/streamanity.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 1e7f165ab..2af99b3da 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1871,7 +1871,6 @@
     StoryFireSeriesIE,
 )
 from .streamable import StreamableIE
-from .streamanity import StreamanityIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streamff import StreamFFIE
diff --git a/yt_dlp/extractor/streamanity.py b/yt_dlp/extractor/streamanity.py
deleted file mode 100644
index 6eaee52d9..000000000
--- a/yt_dlp/extractor/streamanity.py
+++ /dev/null
@@ -1,47 +0,0 @@
-from .common import InfoExtractor
-
-
-class StreamanityIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?streamanity\.com/video/(?P<id>[A-Za-z0-9]+)'
-    _TESTS = [{
-        'url': 'https://streamanity.com/video/9DFPTnuYi8f2',
-        'md5': '6ab171e8d4a02ad5dcbff6bea44cf5a1',
-        'info_dict': {
-            'id': '9DFPTnuYi8f2',
-            'ext': 'mp4',
-            'title': 'Bitcoin vs The Lighting Network',
-            'thumbnail': r're:https://res\.cloudinary\.com/.+\.png',
-            'description': '',
-            'uploader': 'Tom Bombadil (Freddy78)',
-        }
-    }, {
-        'url': 'https://streamanity.com/video/JktOUjSlfzTD',
-        'md5': '31f131e28abd3377c38be586a59532dc',
-        'info_dict': {
-            'id': 'JktOUjSlfzTD',
-            'ext': 'mp4',
-            'title': 'Share data when you see it',
-            'thumbnail': r're:https://res\.cloudinary\.com/.+\.png',
-            'description': 'Reposting as data should be public and stored on blockchain',
-            'uploader': 'digitalcurrencydaily',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        video_info = self._download_json(
-            f'https://app.streamanity.com/api/video/{video_id}', video_id)['data']['video']
-
-        formats = self._extract_m3u8_formats(
-            f'https://stream.mux.com/{video_info["play_id"]}.m3u8?token={video_info["token"]}',
-            video_id, ext='mp4', m3u8_id='hls')
-
-        return {
-            'id': video_id,
-            'title': video_info['title'],
-            'description': video_info.get('description'),
-            'uploader': video_info.get('author_name'),
-            'is_live': False,
-            'thumbnail': video_info.get('thumb'),
-            'formats': formats,
-        }

From 8a4cd12c8f8e93292e3e95200b9d17a3af39624c Mon Sep 17 00:00:00 2001
From: Neurognostic <donovan@tremura.email>
Date: Thu, 13 Jul 2023 16:39:21 -0400
Subject: [PATCH 442/871] [pp/EmbedThumbnail] Support `m4v` (#7583)

Authored by: Neurognostic
---
 yt_dlp/postprocessor/embedthumbnail.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/postprocessor/embedthumbnail.py b/yt_dlp/postprocessor/embedthumbnail.py
index 88a767132..d7be0b398 100644
--- a/yt_dlp/postprocessor/embedthumbnail.py
+++ b/yt_dlp/postprocessor/embedthumbnail.py
@@ -114,7 +114,7 @@ def run(self, info):
             self._report_run('ffmpeg', filename)
             self.run_ffmpeg(filename, temp_filename, options)
 
-        elif info['ext'] in ['m4a', 'mp4', 'mov']:
+        elif info['ext'] in ['m4a', 'mp4', 'm4v', 'mov']:
             prefer_atomicparsley = 'embed-thumbnail-atomicparsley' in self.get_param('compat_opts', [])
             # Method 1: Use mutagen
             if not mutagen or prefer_atomicparsley:
@@ -213,7 +213,7 @@ def run(self, info):
             temp_filename = filename
 
         else:
-            raise EmbedThumbnailPPError('Supported filetypes for thumbnail embedding are: mp3, mkv/mka, ogg/opus/flac, m4a/mp4/mov')
+            raise EmbedThumbnailPPError('Supported filetypes for thumbnail embedding are: mp3, mkv/mka, ogg/opus/flac, m4a/mp4/m4v/mov')
 
         if success and temp_filename != filename:
             os.replace(temp_filename, filename)

From 1bcb9fe8715b1f288efc322be3de409ee0597080 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Finn=20R=2E=20G=C3=A4rtner?=
 <65015656+FinnRG@users.noreply.github.com>
Date: Fri, 14 Jul 2023 20:09:02 +0200
Subject: [PATCH 443/871] [ie/piapro] Support `/content` URL (#7592)

Authored by: FinnRG
---
 yt_dlp/extractor/piapro.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/piapro.py b/yt_dlp/extractor/piapro.py
index d8d9c7801..eb5923d11 100644
--- a/yt_dlp/extractor/piapro.py
+++ b/yt_dlp/extractor/piapro.py
@@ -12,17 +12,22 @@
 
 class PiaproIE(InfoExtractor):
     _NETRC_MACHINE = 'piapro'
-    _VALID_URL = r'https?://piapro\.jp/t/(?P<id>\w+)/?'
+    _VALID_URL = r'https?://piapro\.jp/(?:t|content)/(?P<id>\w+)/?'
     _TESTS = [{
         'url': 'https://piapro.jp/t/NXYR',
-        'md5': 'a9d52f27d13bafab7ee34116a7dcfa77',
+        'md5': 'f7c0f760913fb1d44a1c45a4af793909',
         'info_dict': {
             'id': 'NXYR',
             'ext': 'mp3',
             'uploader': 'wowaka',
             'uploader_id': 'wowaka',
             'title': '裏表ラバーズ',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'http://www.nicovideo.jp/watch/sm8082467',
+            'duration': 189.0,
+            'timestamp': 1251785475,
+            'thumbnail': r're:^https?://.*\.(?:png|jpg)$',
+            'upload_date': '20090901',
+            'view_count': int,
         }
     }, {
         'note': 'There are break lines in description, mandating (?s) flag',
@@ -34,8 +39,16 @@ class PiaproIE(InfoExtractor):
             'title': '青に溶けた風船 / 初音ミク',
             'description': 'md5:d395a9bd151447631a5a1460bc7f9132',
             'uploader': 'シアン・キノ',
+            'duration': 229.0,
+            'timestamp': 1644030039,
+            'upload_date': '20220205',
+            'view_count': int,
+            'thumbnail': r're:^https?://.*\.(?:png|jpg)$',
             'uploader_id': 'cyankino',
         }
+    }, {
+        'url': 'https://piapro.jp/content/hcw0z3a169wtemz6',
+        'only_matching': True
     }]
 
     _login_status = False

From 1ba6fe9db5f660d5538588315c23ad6cf0371c5f Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 15 Jul 2023 15:20:24 +1200
Subject: [PATCH 444/871] [ie/youtube:tab] Detect looping feeds (#6621)

Closes https://github.com/yt-dlp/yt-dlp/issues/5555

Note: the first page may still be repeated, however this is better than nothing.

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 73bfa662d..826bbb20e 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4921,10 +4921,15 @@ def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
         yield from extract_entries(parent_renderer)
         continuation = continuation_list[0]
-
+        seen_continuations = set()
         for page_num in itertools.count(1):
             if not continuation:
                 break
+            continuation_token = continuation.get('continuation')
+            if continuation_token is not None and continuation_token in seen_continuations:
+                self.write_debug('Detected YouTube feed looping - assuming end of feed.')
+                break
+            seen_continuations.add(continuation_token)
             headers = self.generate_api_headers(
                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
             response = self._extract_response(

From 1b392f905d20ef1f1b300b180f867d43c9ce49b8 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 15 Jul 2023 11:41:08 +0530
Subject: [PATCH 445/871] [utils] Add temporary shim for logging

Related: #5680, #7517
---
 test/test_downloader_http.py | 12 +-----------
 yt_dlp/cookies.py            | 23 ++++-------------------
 yt_dlp/utils/_utils.py       | 30 ++++++++++++++++++++++++++++++
 3 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/test/test_downloader_http.py b/test/test_downloader_http.py
index 381b2583c..099ec2fff 100644
--- a/test/test_downloader_http.py
+++ b/test/test_downloader_http.py
@@ -16,6 +16,7 @@
 from yt_dlp import YoutubeDL
 from yt_dlp.downloader.http import HttpFD
 from yt_dlp.utils import encodeFilename
+from yt_dlp.utils._utils import _YDLLogger as FakeLogger
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -67,17 +68,6 @@ def do_GET(self):
             assert False
 
 
-class FakeLogger:
-    def debug(self, msg):
-        pass
-
-    def warning(self, msg):
-        pass
-
-    def error(self, msg):
-        pass
-
-
 class TestHttpFD(unittest.TestCase):
     def setUp(self):
         self.httpd = http.server.HTTPServer(
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index 53fe0ec2d..16f1918e6 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -41,30 +41,15 @@
     try_call,
     write_string,
 )
+from .utils._utils import _YDLLogger
 
 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
 SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
 
 
-class YDLLogger:
-    def __init__(self, ydl=None):
-        self._ydl = ydl
-
-    def debug(self, message):
-        if self._ydl:
-            self._ydl.write_debug(message)
-
-    def info(self, message):
-        if self._ydl:
-            self._ydl.to_screen(f'[Cookies] {message}')
-
-    def warning(self, message, only_once=False):
-        if self._ydl:
-            self._ydl.report_warning(message, only_once)
-
-    def error(self, message):
-        if self._ydl:
-            self._ydl.report_error(message)
+class YDLLogger(_YDLLogger):
+    def warning(self, message, only_once=False):  # compat
+        return super().warning(message, once=only_once)
 
     class ProgressBar(MultilinePrinter):
         _DELAY, _timer = 0.1, 0
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 3023c33b2..4af955743 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -5994,3 +5994,33 @@ def calculate_preference(self, format):
             format['tbr'] = try_call(lambda: format['vbr'] + format['abr']) or None
 
         return tuple(self._calculate_field_preference(format, field) for field in self._order)
+
+
+# XXX: Temporary
+class _YDLLogger:
+    def __init__(self, ydl=None):
+        self._ydl = ydl
+
+    def debug(self, message):
+        if self._ydl:
+            self._ydl.write_debug(message)
+
+    def info(self, message):
+        if self._ydl:
+            self._ydl.to_screen(message)
+
+    def warning(self, message, *, once=False):
+        if self._ydl:
+            self._ydl.report_warning(message, only_once=once)
+
+    def error(self, message, *, is_error=True):
+        if self._ydl:
+            self._ydl.report_error(message, is_error=is_error)
+
+    def stdout(self, message):
+        if self._ydl:
+            self._ydl.to_stdout(message)
+
+    def stderr(self, message):
+        if self._ydl:
+            self._ydl.to_stderr(message)

From c365dba8430ee33abda85d31f95128605bf240eb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 15 Jul 2023 14:30:08 +0530
Subject: [PATCH 446/871] [networking] Add module (#2861)

No actual changes - code is only moved around
---
 Makefile                                  |   2 +-
 devscripts/make_changelog.py              |   1 +
 test/{test_http.py => test_networking.py} |   0
 test/test_utils.py                        |  18 +-
 yt_dlp/YoutubeDL.py                       |  40 +-
 yt_dlp/networking/__init__.py             |   0
 yt_dlp/networking/_helper.py              | 139 +++++++
 yt_dlp/networking/_urllib.py              | 315 ++++++++++++++
 yt_dlp/networking/exceptions.py           |   9 +
 yt_dlp/utils/__init__.py                  |   5 +-
 yt_dlp/utils/_deprecated.py               |  19 +
 yt_dlp/utils/_utils.py                    | 479 +---------------------
 yt_dlp/utils/networking.py                |  60 +++
 13 files changed, 587 insertions(+), 500 deletions(-)
 rename test/{test_http.py => test_networking.py} (100%)
 create mode 100644 yt_dlp/networking/__init__.py
 create mode 100644 yt_dlp/networking/_helper.py
 create mode 100644 yt_dlp/networking/_urllib.py
 create mode 100644 yt_dlp/networking/exceptions.py
 create mode 100644 yt_dlp/utils/networking.py

diff --git a/Makefile b/Makefile
index b1ac0e7d6..c85b24c13 100644
--- a/Makefile
+++ b/Makefile
@@ -74,7 +74,7 @@ offlinetest: codetest
 	$(PYTHON) -m pytest -k "not download"
 
 # XXX: This is hard to maintain
-CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/compat/urllib yt_dlp/utils yt_dlp/dependencies
+CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/compat/urllib yt_dlp/utils yt_dlp/dependencies yt_dlp/networking
 yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
 	mkdir -p zip
 	for d in $(CODE_FOLDERS) ; do \
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index 3ad4c5408..157c66126 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -54,6 +54,7 @@ def commit_lookup(cls):
                     'core',
                     'dependencies',
                     'jsinterp',
+                    'networking',
                     'outtmpl',
                     'formats',
                     'plugins',
diff --git a/test/test_http.py b/test/test_networking.py
similarity index 100%
rename from test/test_http.py
rename to test/test_networking.py
diff --git a/test/test_utils.py b/test/test_utils.py
index bdbd2d879..862c7d0f7 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -258,15 +258,6 @@ def test_sanitize_url(self):
         self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar')
         self.assertEqual(sanitize_url('foo bar'), 'foo bar')
 
-    def test_extract_basic_auth(self):
-        auth_header = lambda url: sanitized_Request(url).get_header('Authorization')
-        self.assertFalse(auth_header('http://foo.bar'))
-        self.assertFalse(auth_header('http://:foo.bar'))
-        self.assertEqual(auth_header('http://@foo.bar'), 'Basic Og==')
-        self.assertEqual(auth_header('http://:pass@foo.bar'), 'Basic OnBhc3M=')
-        self.assertEqual(auth_header('http://user:@foo.bar'), 'Basic dXNlcjo=')
-        self.assertEqual(auth_header('http://user:pass@foo.bar'), 'Basic dXNlcjpwYXNz')
-
     def test_expand_path(self):
         def env(var):
             return f'%{var}%' if sys.platform == 'win32' else f'${var}'
@@ -2324,6 +2315,15 @@ def test_traverse_obj(self):
         self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 'group')), ['0123', '3'],
                          msg='function on a `re.Match` should give group name as well')
 
+    def test_extract_basic_auth(self):
+        auth_header = lambda url: sanitized_Request(url).get_header('Authorization')
+        self.assertFalse(auth_header('http://foo.bar'))
+        self.assertFalse(auth_header('http://:foo.bar'))
+        self.assertEqual(auth_header('http://@foo.bar'), 'Basic Og==')
+        self.assertEqual(auth_header('http://:pass@foo.bar'), 'Basic OnBhc3M=')
+        self.assertEqual(auth_header('http://user:@foo.bar'), 'Basic dXNlcjo=')
+        self.assertEqual(auth_header('http://user:pass@foo.bar'), 'Basic dXNlcjpwYXNz')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 7f5571666..138646ebf 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -151,6 +151,7 @@
     write_json_file,
     write_string,
 )
+from .utils.networking import clean_headers
 from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
 
 if compat_os_name == 'nt':
@@ -672,6 +673,7 @@ def process_color_policy(stream):
                     raise
 
         self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
+        self.params['http_headers'] = merge_headers(std_headers, self.params.get('http_headers', {}))
         if auto_init and auto_init != 'no_verbose_header':
             self.print_debug_header()
 
@@ -745,9 +747,6 @@ def check_deprecated(param, option, suggestion):
             else self.params['format'] if callable(self.params['format'])
             else self.build_format_selector(self.params['format']))
 
-        # Set http_headers defaults according to std_headers
-        self.params['http_headers'] = merge_headers(std_headers, self.params.get('http_headers', {}))
-
         hooks = {
             'post_hooks': self.add_post_hook,
             'progress_hooks': self.add_progress_hook,
@@ -941,12 +940,14 @@ def __enter__(self):
         self.save_console_title()
         return self
 
-    def __exit__(self, *args):
-        self.restore_console_title()
-
+    def save_cookies(self):
         if self.params.get('cookiefile') is not None:
             self.cookiejar.save(ignore_discard=True, ignore_expires=True)
 
+    def __exit__(self, *args):
+        self.restore_console_title()
+        self.save_cookies()
+
     def trouble(self, message=None, tb=None, is_error=True):
         """Determine action to take when a download problem appears.
 
@@ -2468,9 +2469,7 @@ def restore_last_token(self):
 
     def _calc_headers(self, info_dict):
         res = merge_headers(self.params['http_headers'], info_dict.get('http_headers') or {})
-        if 'Youtubedl-No-Compression' in res:  # deprecated
-            res.pop('Youtubedl-No-Compression', None)
-            res['Accept-Encoding'] = 'identity'
+        clean_headers(res)
         cookies = self.cookiejar.get_cookies_for_url(info_dict['url'])
         if cookies:
             encoder = LenientSimpleCookie()
@@ -3856,12 +3855,6 @@ def list_thumbnails(self, info_dict):
     def list_subtitles(self, video_id, subtitles, name='subtitles'):
         self.__list_table(video_id, name, self.render_subtitles_table, video_id, subtitles)
 
-    def urlopen(self, req):
-        """ Start an HTTP download """
-        if isinstance(req, str):
-            req = sanitized_Request(req)
-        return self._opener.open(req, timeout=self._socket_timeout)
-
     def print_debug_header(self):
         if not self.params.get('verbose'):
             return
@@ -3989,13 +3982,8 @@ def _setup_opener(self):
             return
         timeout_val = self.params.get('socket_timeout')
         self._socket_timeout = 20 if timeout_val is None else float(timeout_val)
-
-        opts_cookiesfrombrowser = self.params.get('cookiesfrombrowser')
-        opts_cookiefile = self.params.get('cookiefile')
         opts_proxy = self.params.get('proxy')
 
-        self.cookiejar = load_cookies(opts_cookiefile, opts_cookiesfrombrowser, self)
-
         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
         if opts_proxy is not None:
             if opts_proxy == '':
@@ -4037,6 +4025,18 @@ def file_open(*args, **kwargs):
         opener.addheaders = []
         self._opener = opener
 
+    @functools.cached_property
+    def cookiejar(self):
+        """Global cookiejar instance"""
+        return load_cookies(
+            self.params.get('cookiefile'), self.params.get('cookiesfrombrowser'), self)
+
+    def urlopen(self, req):
+        """ Start an HTTP download """
+        if isinstance(req, str):
+            req = sanitized_Request(req)
+        return self._opener.open(req, timeout=self._socket_timeout)
+
     def encode(self, s):
         if isinstance(s, bytes):
             return s  # Already encoded
diff --git a/yt_dlp/networking/__init__.py b/yt_dlp/networking/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/yt_dlp/networking/_helper.py b/yt_dlp/networking/_helper.py
new file mode 100644
index 000000000..367f3f444
--- /dev/null
+++ b/yt_dlp/networking/_helper.py
@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import contextlib
+import ssl
+import sys
+import urllib.parse
+
+from ..dependencies import certifi
+from ..socks import ProxyType
+from ..utils import YoutubeDLError
+
+
+def ssl_load_certs(context: ssl.SSLContext, use_certifi=True):
+    if certifi and use_certifi:
+        context.load_verify_locations(cafile=certifi.where())
+    else:
+        try:
+            context.load_default_certs()
+        # Work around the issue in load_default_certs when there are bad certificates. See:
+        # https://github.com/yt-dlp/yt-dlp/issues/1060,
+        # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
+        except ssl.SSLError:
+            # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
+            if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
+                for storename in ('CA', 'ROOT'):
+                    _ssl_load_windows_store_certs(context, storename)
+            context.set_default_verify_paths()
+
+
+def _ssl_load_windows_store_certs(ssl_context, storename):
+    # Code adapted from _load_windows_store_certs in https://github.com/python/cpython/blob/main/Lib/ssl.py
+    try:
+        certs = [cert for cert, encoding, trust in ssl.enum_certificates(storename)
+                 if encoding == 'x509_asn' and (
+                     trust is True or ssl.Purpose.SERVER_AUTH.oid in trust)]
+    except PermissionError:
+        return
+    for cert in certs:
+        with contextlib.suppress(ssl.SSLError):
+            ssl_context.load_verify_locations(cadata=cert)
+
+
+def make_socks_proxy_opts(socks_proxy):
+    url_components = urllib.parse.urlparse(socks_proxy)
+    if url_components.scheme.lower() == 'socks5':
+        socks_type = ProxyType.SOCKS5
+    elif url_components.scheme.lower() in ('socks', 'socks4'):
+        socks_type = ProxyType.SOCKS4
+    elif url_components.scheme.lower() == 'socks4a':
+        socks_type = ProxyType.SOCKS4A
+
+    def unquote_if_non_empty(s):
+        if not s:
+            return s
+        return urllib.parse.unquote_plus(s)
+    return {
+        'proxytype': socks_type,
+        'addr': url_components.hostname,
+        'port': url_components.port or 1080,
+        'rdns': True,
+        'username': unquote_if_non_empty(url_components.username),
+        'password': unquote_if_non_empty(url_components.password),
+    }
+
+
+def get_redirect_method(method, status):
+    """Unified redirect method handling"""
+
+    # A 303 must either use GET or HEAD for subsequent request
+    # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.4
+    if status == 303 and method != 'HEAD':
+        method = 'GET'
+    # 301 and 302 redirects are commonly turned into a GET from a POST
+    # for subsequent requests by browsers, so we'll do the same.
+    # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.2
+    # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.3
+    if status in (301, 302) and method == 'POST':
+        method = 'GET'
+    return method
+
+
+def make_ssl_context(
+    verify=True,
+    client_certificate=None,
+    client_certificate_key=None,
+    client_certificate_password=None,
+    legacy_support=False,
+    use_certifi=True,
+):
+    context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    context.check_hostname = verify
+    context.verify_mode = ssl.CERT_REQUIRED if verify else ssl.CERT_NONE
+
+    # Some servers may reject requests if ALPN extension is not sent. See:
+    # https://github.com/python/cpython/issues/85140
+    # https://github.com/yt-dlp/yt-dlp/issues/3878
+    with contextlib.suppress(NotImplementedError):
+        context.set_alpn_protocols(['http/1.1'])
+    if verify:
+        ssl_load_certs(context, use_certifi)
+
+    if legacy_support:
+        context.options |= 4  # SSL_OP_LEGACY_SERVER_CONNECT
+        context.set_ciphers('DEFAULT')  # compat
+
+    elif ssl.OPENSSL_VERSION_INFO >= (1, 1, 1) and not ssl.OPENSSL_VERSION.startswith('LibreSSL'):
+        # Use the default SSL ciphers and minimum TLS version settings from Python 3.10 [1].
+        # This is to ensure consistent behavior across Python versions and libraries, and help avoid fingerprinting
+        # in some situations [2][3].
+        # Python 3.10 only supports OpenSSL 1.1.1+ [4]. Because this change is likely
+        # untested on older versions, we only apply this to OpenSSL 1.1.1+ to be safe.
+        # LibreSSL is excluded until further investigation due to cipher support issues [5][6].
+        # 1. https://github.com/python/cpython/commit/e983252b516edb15d4338b0a47631b59ef1e2536
+        # 2. https://github.com/yt-dlp/yt-dlp/issues/4627
+        # 3. https://github.com/yt-dlp/yt-dlp/pull/5294
+        # 4. https://peps.python.org/pep-0644/
+        # 5. https://peps.python.org/pep-0644/#libressl-support
+        # 6. https://github.com/yt-dlp/yt-dlp/commit/5b9f253fa0aee996cf1ed30185d4b502e00609c4#commitcomment-89054368
+        context.set_ciphers(
+            '@SECLEVEL=2:ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES:DHE+AES:!aNULL:!eNULL:!aDSS:!SHA1:!AESCCM')
+        context.minimum_version = ssl.TLSVersion.TLSv1_2
+
+    if client_certificate:
+        try:
+            context.load_cert_chain(
+                client_certificate, keyfile=client_certificate_key,
+                password=client_certificate_password)
+        except ssl.SSLError:
+            raise YoutubeDLError('Unable to load client certificate')
+
+    return context
+
+
+def add_accept_encoding_header(headers, supported_encodings):
+    if supported_encodings and 'Accept-Encoding' not in headers:
+        headers['Accept-Encoding'] = ', '.join(supported_encodings)
+
+    elif 'Accept-Encoding' not in headers:
+        headers['Accept-Encoding'] = 'identity'
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
new file mode 100644
index 000000000..1f5871ae6
--- /dev/null
+++ b/yt_dlp/networking/_urllib.py
@@ -0,0 +1,315 @@
+import functools
+import gzip
+import http.client
+import io
+import socket
+import ssl
+import urllib.error
+import urllib.parse
+import urllib.request
+import urllib.response
+import zlib
+
+from ._helper import (
+    add_accept_encoding_header,
+    get_redirect_method,
+    make_socks_proxy_opts,
+)
+from ..dependencies import brotli
+from ..socks import sockssocket
+from ..utils import escape_url, update_url_query
+from ..utils.networking import clean_headers, std_headers
+
+SUPPORTED_ENCODINGS = ['gzip', 'deflate']
+
+if brotli:
+    SUPPORTED_ENCODINGS.append('br')
+
+
+def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
+    hc = http_class(*args, **kwargs)
+    source_address = ydl_handler._params.get('source_address')
+
+    if source_address is not None:
+        # This is to workaround _create_connection() from socket where it will try all
+        # address data from getaddrinfo() including IPv6. This filters the result from
+        # getaddrinfo() based on the source_address value.
+        # This is based on the cpython socket.create_connection() function.
+        # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
+        def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            host, port = address
+            err = None
+            addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
+            af = socket.AF_INET if '.' in source_address[0] else socket.AF_INET6
+            ip_addrs = [addr for addr in addrs if addr[0] == af]
+            if addrs and not ip_addrs:
+                ip_version = 'v4' if af == socket.AF_INET else 'v6'
+                raise OSError(
+                    "No remote IP%s addresses available for connect, can't use '%s' as source address"
+                    % (ip_version, source_address[0]))
+            for res in ip_addrs:
+                af, socktype, proto, canonname, sa = res
+                sock = None
+                try:
+                    sock = socket.socket(af, socktype, proto)
+                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+                        sock.settimeout(timeout)
+                    sock.bind(source_address)
+                    sock.connect(sa)
+                    err = None  # Explicitly break reference cycle
+                    return sock
+                except OSError as _:
+                    err = _
+                    if sock is not None:
+                        sock.close()
+            if err is not None:
+                raise err
+            else:
+                raise OSError('getaddrinfo returns an empty list')
+        if hasattr(hc, '_create_connection'):
+            hc._create_connection = _create_connection
+        hc.source_address = (source_address, 0)
+
+    return hc
+
+
+class HTTPHandler(urllib.request.HTTPHandler):
+    """Handler for HTTP requests and responses.
+
+    This class, when installed with an OpenerDirector, automatically adds
+    the standard headers to every HTTP request and handles gzipped, deflated and
+    brotli responses from web servers.
+
+    Part of this code was copied from:
+
+    http://techknack.net/python-urllib2-handlers/
+
+    Andrew Rowls, the author of that code, agreed to release it to the
+    public domain.
+    """
+
+    def __init__(self, params, *args, **kwargs):
+        urllib.request.HTTPHandler.__init__(self, *args, **kwargs)
+        self._params = params
+
+    def http_open(self, req):
+        conn_class = http.client.HTTPConnection
+
+        socks_proxy = req.headers.get('Ytdl-socks-proxy')
+        if socks_proxy:
+            conn_class = make_socks_conn_class(conn_class, socks_proxy)
+            del req.headers['Ytdl-socks-proxy']
+
+        return self.do_open(functools.partial(
+            _create_http_connection, self, conn_class, False),
+            req)
+
+    @staticmethod
+    def deflate(data):
+        if not data:
+            return data
+        try:
+            return zlib.decompress(data, -zlib.MAX_WBITS)
+        except zlib.error:
+            return zlib.decompress(data)
+
+    @staticmethod
+    def brotli(data):
+        if not data:
+            return data
+        return brotli.decompress(data)
+
+    @staticmethod
+    def gz(data):
+        gz = gzip.GzipFile(fileobj=io.BytesIO(data), mode='rb')
+        try:
+            return gz.read()
+        except OSError as original_oserror:
+            # There may be junk add the end of the file
+            # See http://stackoverflow.com/q/4928560/35070 for details
+            for i in range(1, 1024):
+                try:
+                    gz = gzip.GzipFile(fileobj=io.BytesIO(data[:-i]), mode='rb')
+                    return gz.read()
+                except OSError:
+                    continue
+            else:
+                raise original_oserror
+
+    def http_request(self, req):
+        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
+        # always respected by websites, some tend to give out URLs with non percent-encoded
+        # non-ASCII characters (see telemb.py, ard.py [#3412])
+        # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
+        # To work around aforementioned issue we will replace request's original URL with
+        # percent-encoded one
+        # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
+        # the code of this workaround has been moved here from YoutubeDL.urlopen()
+        url = req.get_full_url()
+        url_escaped = escape_url(url)
+
+        # Substitute URL if any change after escaping
+        if url != url_escaped:
+            req = update_Request(req, url=url_escaped)
+
+        for h, v in self._params.get('http_headers', std_headers).items():
+            # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
+            # The dict keys are capitalized because of this bug by urllib
+            if h.capitalize() not in req.headers:
+                req.add_header(h, v)
+
+        clean_headers(req.headers)
+        add_accept_encoding_header(req.headers, SUPPORTED_ENCODINGS)
+        return super().do_request_(req)
+
+    def http_response(self, req, resp):
+        old_resp = resp
+
+        # Content-Encoding header lists the encodings in order that they were applied [1].
+        # To decompress, we simply do the reverse.
+        # [1]: https://datatracker.ietf.org/doc/html/rfc9110#name-content-encoding
+        decoded_response = None
+        for encoding in (e.strip() for e in reversed(resp.headers.get('Content-encoding', '').split(','))):
+            if encoding == 'gzip':
+                decoded_response = self.gz(decoded_response or resp.read())
+            elif encoding == 'deflate':
+                decoded_response = self.deflate(decoded_response or resp.read())
+            elif encoding == 'br' and brotli:
+                decoded_response = self.brotli(decoded_response or resp.read())
+
+        if decoded_response is not None:
+            resp = urllib.request.addinfourl(io.BytesIO(decoded_response), old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
+        # https://github.com/ytdl-org/youtube-dl/issues/6457).
+        if 300 <= resp.code < 400:
+            location = resp.headers.get('Location')
+            if location:
+                # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
+                location = location.encode('iso-8859-1').decode()
+                location_escaped = escape_url(location)
+                if location != location_escaped:
+                    del resp.headers['Location']
+                    resp.headers['Location'] = location_escaped
+        return resp
+
+    https_request = http_request
+    https_response = http_response
+
+
+def make_socks_conn_class(base_class, socks_proxy):
+    assert issubclass(base_class, (
+        http.client.HTTPConnection, http.client.HTTPSConnection))
+
+    proxy_args = make_socks_proxy_opts(socks_proxy)
+
+    class SocksConnection(base_class):
+        def connect(self):
+            self.sock = sockssocket()
+            self.sock.setproxy(**proxy_args)
+            if isinstance(self.timeout, (int, float)):
+                self.sock.settimeout(self.timeout)
+            self.sock.connect((self.host, self.port))
+
+            if isinstance(self, http.client.HTTPSConnection):
+                if hasattr(self, '_context'):  # Python > 2.6
+                    self.sock = self._context.wrap_socket(
+                        self.sock, server_hostname=self.host)
+                else:
+                    self.sock = ssl.wrap_socket(self.sock)
+
+    return SocksConnection
+
+
+class RedirectHandler(urllib.request.HTTPRedirectHandler):
+    """YoutubeDL redirect handler
+
+    The code is based on HTTPRedirectHandler implementation from CPython [1].
+
+    This redirect handler fixes and improves the logic to better align with RFC7261
+     and what browsers tend to do [2][3]
+
+    1. https://github.com/python/cpython/blob/master/Lib/urllib/request.py
+    2. https://datatracker.ietf.org/doc/html/rfc7231
+    3. https://github.com/python/cpython/issues/91306
+    """
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        if code not in (301, 302, 303, 307, 308):
+            raise urllib.error.HTTPError(req.full_url, code, msg, headers, fp)
+
+        new_data = req.data
+
+        # Technically the Cookie header should be in unredirected_hdrs,
+        # however in practice some may set it in normal headers anyway.
+        # We will remove it here to prevent any leaks.
+        remove_headers = ['Cookie']
+
+        new_method = get_redirect_method(req.get_method(), code)
+        # only remove payload if method changed (e.g. POST to GET)
+        if new_method != req.get_method():
+            new_data = None
+            remove_headers.extend(['Content-Length', 'Content-Type'])
+
+        new_headers = {k: v for k, v in req.headers.items() if k.title() not in remove_headers}
+
+        return urllib.request.Request(
+            newurl, headers=new_headers, origin_req_host=req.origin_req_host,
+            unverifiable=True, method=new_method, data=new_data)
+
+
+class ProxyHandler(urllib.request.ProxyHandler):
+    def __init__(self, proxies=None):
+        # Set default handlers
+        for type in ('http', 'https'):
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
+                        meth(r, proxy, type))
+        urllib.request.ProxyHandler.__init__(self, proxies)
+
+    def proxy_open(self, req, proxy, type):
+        req_proxy = req.headers.get('Ytdl-request-proxy')
+        if req_proxy is not None:
+            proxy = req_proxy
+            del req.headers['Ytdl-request-proxy']
+
+        if proxy == '__noproxy__':
+            return None  # No Proxy
+        if urllib.parse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
+            req.add_header('Ytdl-socks-proxy', proxy)
+            # yt-dlp's http/https handlers do wrapping the socket with socks
+            return None
+        return urllib.request.ProxyHandler.proxy_open(
+            self, req, proxy, type)
+
+
+class PUTRequest(urllib.request.Request):
+    def get_method(self):
+        return 'PUT'
+
+
+class HEADRequest(urllib.request.Request):
+    def get_method(self):
+        return 'HEAD'
+
+
+def update_Request(req, url=None, data=None, headers=None, query=None):
+    req_headers = req.headers.copy()
+    req_headers.update(headers or {})
+    req_data = data or req.data
+    req_url = update_url_query(url or req.get_full_url(), query)
+    req_get_method = req.get_method()
+    if req_get_method == 'HEAD':
+        req_type = HEADRequest
+    elif req_get_method == 'PUT':
+        req_type = PUTRequest
+    else:
+        req_type = urllib.request.Request
+    new_req = req_type(
+        req_url, data=req_data, headers=req_headers,
+        origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
+    if hasattr(req, 'timeout'):
+        new_req.timeout = req.timeout
+    return new_req
diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py
new file mode 100644
index 000000000..89b484a22
--- /dev/null
+++ b/yt_dlp/networking/exceptions.py
@@ -0,0 +1,9 @@
+import http.client
+import socket
+import ssl
+import urllib.error
+
+network_exceptions = [urllib.error.URLError, http.client.HTTPException, socket.error]
+if hasattr(ssl, 'CertificateError'):
+    network_exceptions.append(ssl.CertificateError)
+network_exceptions = tuple(network_exceptions)
diff --git a/yt_dlp/utils/__init__.py b/yt_dlp/utils/__init__.py
index 2dd20ada2..0b00adddb 100644
--- a/yt_dlp/utils/__init__.py
+++ b/yt_dlp/utils/__init__.py
@@ -3,13 +3,10 @@
 
 from ..compat.compat_utils import passthrough_module
 
-# XXX: Implement this the same way as other DeprecationWarnings without circular import
-passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
-    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=5))
+passthrough_module(__name__, '._deprecated')
 del passthrough_module
 
 # isort: off
 from .traversal import *
 from ._utils import *
 from ._utils import _configuration_args, _get_exe_version_output
-from ._deprecated import *
diff --git a/yt_dlp/utils/_deprecated.py b/yt_dlp/utils/_deprecated.py
index 4454d84a7..ca0fb1614 100644
--- a/yt_dlp/utils/_deprecated.py
+++ b/yt_dlp/utils/_deprecated.py
@@ -1,7 +1,26 @@
 """Deprecated - New code should avoid these"""
+import warnings
+
+from ..compat.compat_utils import passthrough_module
+
+# XXX: Implement this the same way as other DeprecationWarnings without circular import
+passthrough_module(__name__, '.._legacy', callback=lambda attr: warnings.warn(
+    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=6))
+del passthrough_module
+
 
 from ._utils import preferredencoding
 
+# isort: split
+from ..networking._urllib import PUTRequest  # noqa: F401
+from ..networking._urllib import SUPPORTED_ENCODINGS, HEADRequest  # noqa: F401
+from ..networking._urllib import HTTPHandler as YoutubeDLHandler  # noqa: F401
+from ..networking._urllib import ProxyHandler as PerRequestProxyHandler  # noqa: F401
+from ..networking._urllib import RedirectHandler as YoutubeDLRedirectHandler  # noqa: F401
+from ..networking._urllib import make_socks_conn_class, update_Request  # noqa: F401
+from ..networking.exceptions import network_exceptions  # noqa: F401
+from .networking import random_user_agent, std_headers  # noqa: F401
+
 
 def encodeFilename(s, for_subprocess=False):
     assert isinstance(s, str)
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 4af955743..d5704cadc 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -11,7 +11,6 @@
 import email.header
 import email.utils
 import errno
-import gzip
 import hashlib
 import hmac
 import html.entities
@@ -46,7 +45,6 @@
 import urllib.parse
 import urllib.request
 import xml.etree.ElementTree
-import zlib
 
 from . import traversal
 
@@ -58,8 +56,7 @@
     compat_os_name,
     compat_shlex_quote,
 )
-from ..dependencies import brotli, certifi, websockets, xattr
-from ..socks import ProxyType, sockssocket
+from ..dependencies import websockets, xattr
 
 __name__ = __name__.rsplit('.', 1)[0]  # Pretend to be the parent module
 
@@ -67,65 +64,6 @@
 compiled_regex_type = type(re.compile(''))
 
 
-def random_user_agent():
-    _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
-    _CHROME_VERSIONS = (
-        '90.0.4430.212',
-        '90.0.4430.24',
-        '90.0.4430.70',
-        '90.0.4430.72',
-        '90.0.4430.85',
-        '90.0.4430.93',
-        '91.0.4472.101',
-        '91.0.4472.106',
-        '91.0.4472.114',
-        '91.0.4472.124',
-        '91.0.4472.164',
-        '91.0.4472.19',
-        '91.0.4472.77',
-        '92.0.4515.107',
-        '92.0.4515.115',
-        '92.0.4515.131',
-        '92.0.4515.159',
-        '92.0.4515.43',
-        '93.0.4556.0',
-        '93.0.4577.15',
-        '93.0.4577.63',
-        '93.0.4577.82',
-        '94.0.4606.41',
-        '94.0.4606.54',
-        '94.0.4606.61',
-        '94.0.4606.71',
-        '94.0.4606.81',
-        '94.0.4606.85',
-        '95.0.4638.17',
-        '95.0.4638.50',
-        '95.0.4638.54',
-        '95.0.4638.69',
-        '95.0.4638.74',
-        '96.0.4664.18',
-        '96.0.4664.45',
-        '96.0.4664.55',
-        '96.0.4664.93',
-        '97.0.4692.20',
-    )
-    return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)
-
-
-SUPPORTED_ENCODINGS = [
-    'gzip', 'deflate'
-]
-if brotli:
-    SUPPORTED_ENCODINGS.append('br')
-
-std_headers = {
-    'User-Agent': random_user_agent(),
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    'Accept-Language': 'en-us,en;q=0.5',
-    'Sec-Fetch-Mode': 'navigate',
-}
-
-
 USER_AGENTS = {
     'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
 }
@@ -958,80 +896,16 @@ def formatSeconds(secs, delim=':', msec=False):
     return '%s.%03d' % (ret, time.milliseconds) if msec else ret
 
 
-def _ssl_load_windows_store_certs(ssl_context, storename):
-    # Code adapted from _load_windows_store_certs in https://github.com/python/cpython/blob/main/Lib/ssl.py
-    try:
-        certs = [cert for cert, encoding, trust in ssl.enum_certificates(storename)
-                 if encoding == 'x509_asn' and (
-                     trust is True or ssl.Purpose.SERVER_AUTH.oid in trust)]
-    except PermissionError:
-        return
-    for cert in certs:
-        with contextlib.suppress(ssl.SSLError):
-            ssl_context.load_verify_locations(cadata=cert)
-
-
 def make_HTTPS_handler(params, **kwargs):
-    opts_check_certificate = not params.get('nocheckcertificate')
-    context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-    context.check_hostname = opts_check_certificate
-    if params.get('legacyserverconnect'):
-        context.options |= 4  # SSL_OP_LEGACY_SERVER_CONNECT
-        # Allow use of weaker ciphers in Python 3.10+. See https://bugs.python.org/issue43998
-        context.set_ciphers('DEFAULT')
-    elif (
-        sys.version_info < (3, 10)
-        and ssl.OPENSSL_VERSION_INFO >= (1, 1, 1)
-        and not ssl.OPENSSL_VERSION.startswith('LibreSSL')
-    ):
-        # Backport the default SSL ciphers and minimum TLS version settings from Python 3.10 [1].
-        # This is to ensure consistent behavior across Python versions, and help avoid fingerprinting
-        # in some situations [2][3].
-        # Python 3.10 only supports OpenSSL 1.1.1+ [4]. Because this change is likely
-        # untested on older versions, we only apply this to OpenSSL 1.1.1+ to be safe.
-        # LibreSSL is excluded until further investigation due to cipher support issues [5][6].
-        # 1. https://github.com/python/cpython/commit/e983252b516edb15d4338b0a47631b59ef1e2536
-        # 2. https://github.com/yt-dlp/yt-dlp/issues/4627
-        # 3. https://github.com/yt-dlp/yt-dlp/pull/5294
-        # 4. https://peps.python.org/pep-0644/
-        # 5. https://peps.python.org/pep-0644/#libressl-support
-        # 6. https://github.com/yt-dlp/yt-dlp/commit/5b9f253fa0aee996cf1ed30185d4b502e00609c4#commitcomment-89054368
-        context.set_ciphers('@SECLEVEL=2:ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES:DHE+AES:!aNULL:!eNULL:!aDSS:!SHA1:!AESCCM')
-        context.minimum_version = ssl.TLSVersion.TLSv1_2
-
-    context.verify_mode = ssl.CERT_REQUIRED if opts_check_certificate else ssl.CERT_NONE
-    if opts_check_certificate:
-        if certifi and 'no-certifi' not in params.get('compat_opts', []):
-            context.load_verify_locations(cafile=certifi.where())
-        else:
-            try:
-                context.load_default_certs()
-                # Work around the issue in load_default_certs when there are bad certificates. See:
-                # https://github.com/yt-dlp/yt-dlp/issues/1060,
-                # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
-            except ssl.SSLError:
-                # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
-                if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
-                    for storename in ('CA', 'ROOT'):
-                        _ssl_load_windows_store_certs(context, storename)
-                context.set_default_verify_paths()
-
-    client_certfile = params.get('client_certificate')
-    if client_certfile:
-        try:
-            context.load_cert_chain(
-                client_certfile, keyfile=params.get('client_certificate_key'),
-                password=params.get('client_certificate_password'))
-        except ssl.SSLError:
-            raise YoutubeDLError('Unable to load client certificate')
-
-    # Some servers may reject requests if ALPN extension is not sent. See:
-    # https://github.com/python/cpython/issues/85140
-    # https://github.com/yt-dlp/yt-dlp/issues/3878
-    with contextlib.suppress(NotImplementedError):
-        context.set_alpn_protocols(['http/1.1'])
-
-    return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
+    from ..networking._helper import make_ssl_context
+    return YoutubeDLHTTPSHandler(params, context=make_ssl_context(
+        verify=not params.get('nocheckcertificate'),
+        client_certificate=params.get('client_certificate'),
+        client_certificate_key=params.get('client_certificate_key'),
+        client_certificate_password=params.get('client_certificate_password'),
+        legacy_support=params.get('legacyserverconnect'),
+        use_certifi='no-certifi' not in params.get('compat_opts', []),
+    ), **kwargs)
 
 
 def bug_reports_message(before=';'):
@@ -1059,12 +933,6 @@ def __init__(self, msg=None):
         super().__init__(self.msg)
 
 
-network_exceptions = [urllib.error.URLError, http.client.HTTPException, socket.error]
-if hasattr(ssl, 'CertificateError'):
-    network_exceptions.append(ssl.CertificateError)
-network_exceptions = tuple(network_exceptions)
-
-
 class ExtractorError(YoutubeDLError):
     """Error during info extraction."""
 
@@ -1072,6 +940,7 @@ def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None, ie=N
         """ tb, if given, is the original traceback (so that it can be printed out).
         If expected is set, this is a normal error message and most likely not a bug in yt-dlp.
         """
+        from ..networking.exceptions import network_exceptions
         if sys.exc_info()[0] in network_exceptions:
             expected = True
 
@@ -1271,225 +1140,6 @@ class XAttrUnavailableError(YoutubeDLError):
     pass
 
 
-def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
-    hc = http_class(*args, **kwargs)
-    source_address = ydl_handler._params.get('source_address')
-
-    if source_address is not None:
-        # This is to workaround _create_connection() from socket where it will try all
-        # address data from getaddrinfo() including IPv6. This filters the result from
-        # getaddrinfo() based on the source_address value.
-        # This is based on the cpython socket.create_connection() function.
-        # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
-        def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
-            host, port = address
-            err = None
-            addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
-            af = socket.AF_INET if '.' in source_address[0] else socket.AF_INET6
-            ip_addrs = [addr for addr in addrs if addr[0] == af]
-            if addrs and not ip_addrs:
-                ip_version = 'v4' if af == socket.AF_INET else 'v6'
-                raise OSError(
-                    "No remote IP%s addresses available for connect, can't use '%s' as source address"
-                    % (ip_version, source_address[0]))
-            for res in ip_addrs:
-                af, socktype, proto, canonname, sa = res
-                sock = None
-                try:
-                    sock = socket.socket(af, socktype, proto)
-                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
-                        sock.settimeout(timeout)
-                    sock.bind(source_address)
-                    sock.connect(sa)
-                    err = None  # Explicitly break reference cycle
-                    return sock
-                except OSError as _:
-                    err = _
-                    if sock is not None:
-                        sock.close()
-            if err is not None:
-                raise err
-            else:
-                raise OSError('getaddrinfo returns an empty list')
-        if hasattr(hc, '_create_connection'):
-            hc._create_connection = _create_connection
-        hc.source_address = (source_address, 0)
-
-    return hc
-
-
-class YoutubeDLHandler(urllib.request.HTTPHandler):
-    """Handler for HTTP requests and responses.
-
-    This class, when installed with an OpenerDirector, automatically adds
-    the standard headers to every HTTP request and handles gzipped, deflated and
-    brotli responses from web servers.
-
-    Part of this code was copied from:
-
-    http://techknack.net/python-urllib2-handlers/
-
-    Andrew Rowls, the author of that code, agreed to release it to the
-    public domain.
-    """
-
-    def __init__(self, params, *args, **kwargs):
-        urllib.request.HTTPHandler.__init__(self, *args, **kwargs)
-        self._params = params
-
-    def http_open(self, req):
-        conn_class = http.client.HTTPConnection
-
-        socks_proxy = req.headers.get('Ytdl-socks-proxy')
-        if socks_proxy:
-            conn_class = make_socks_conn_class(conn_class, socks_proxy)
-            del req.headers['Ytdl-socks-proxy']
-
-        return self.do_open(functools.partial(
-            _create_http_connection, self, conn_class, False),
-            req)
-
-    @staticmethod
-    def deflate(data):
-        if not data:
-            return data
-        try:
-            return zlib.decompress(data, -zlib.MAX_WBITS)
-        except zlib.error:
-            return zlib.decompress(data)
-
-    @staticmethod
-    def brotli(data):
-        if not data:
-            return data
-        return brotli.decompress(data)
-
-    @staticmethod
-    def gz(data):
-        gz = gzip.GzipFile(fileobj=io.BytesIO(data), mode='rb')
-        try:
-            return gz.read()
-        except OSError as original_oserror:
-            # There may be junk add the end of the file
-            # See http://stackoverflow.com/q/4928560/35070 for details
-            for i in range(1, 1024):
-                try:
-                    gz = gzip.GzipFile(fileobj=io.BytesIO(data[:-i]), mode='rb')
-                    return gz.read()
-                except OSError:
-                    continue
-            else:
-                raise original_oserror
-
-    def http_request(self, req):
-        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
-        # always respected by websites, some tend to give out URLs with non percent-encoded
-        # non-ASCII characters (see telemb.py, ard.py [#3412])
-        # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
-        # To work around aforementioned issue we will replace request's original URL with
-        # percent-encoded one
-        # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
-        # the code of this workaround has been moved here from YoutubeDL.urlopen()
-        url = req.get_full_url()
-        url_escaped = escape_url(url)
-
-        # Substitute URL if any change after escaping
-        if url != url_escaped:
-            req = update_Request(req, url=url_escaped)
-
-        for h, v in self._params.get('http_headers', std_headers).items():
-            # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
-            # The dict keys are capitalized because of this bug by urllib
-            if h.capitalize() not in req.headers:
-                req.add_header(h, v)
-
-        if 'Youtubedl-no-compression' in req.headers:  # deprecated
-            req.headers.pop('Youtubedl-no-compression', None)
-            req.add_header('Accept-encoding', 'identity')
-
-        if 'Accept-encoding' not in req.headers:
-            req.add_header('Accept-encoding', ', '.join(SUPPORTED_ENCODINGS))
-
-        return super().do_request_(req)
-
-    def http_response(self, req, resp):
-        old_resp = resp
-
-        # Content-Encoding header lists the encodings in order that they were applied [1].
-        # To decompress, we simply do the reverse.
-        # [1]: https://datatracker.ietf.org/doc/html/rfc9110#name-content-encoding
-        decoded_response = None
-        for encoding in (e.strip() for e in reversed(resp.headers.get('Content-encoding', '').split(','))):
-            if encoding == 'gzip':
-                decoded_response = self.gz(decoded_response or resp.read())
-            elif encoding == 'deflate':
-                decoded_response = self.deflate(decoded_response or resp.read())
-            elif encoding == 'br' and brotli:
-                decoded_response = self.brotli(decoded_response or resp.read())
-
-        if decoded_response is not None:
-            resp = urllib.request.addinfourl(io.BytesIO(decoded_response), old_resp.headers, old_resp.url, old_resp.code)
-            resp.msg = old_resp.msg
-        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
-        # https://github.com/ytdl-org/youtube-dl/issues/6457).
-        if 300 <= resp.code < 400:
-            location = resp.headers.get('Location')
-            if location:
-                # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
-                location = location.encode('iso-8859-1').decode()
-                location_escaped = escape_url(location)
-                if location != location_escaped:
-                    del resp.headers['Location']
-                    resp.headers['Location'] = location_escaped
-        return resp
-
-    https_request = http_request
-    https_response = http_response
-
-
-def make_socks_conn_class(base_class, socks_proxy):
-    assert issubclass(base_class, (
-        http.client.HTTPConnection, http.client.HTTPSConnection))
-
-    url_components = urllib.parse.urlparse(socks_proxy)
-    if url_components.scheme.lower() == 'socks5':
-        socks_type = ProxyType.SOCKS5
-    elif url_components.scheme.lower() in ('socks', 'socks4'):
-        socks_type = ProxyType.SOCKS4
-    elif url_components.scheme.lower() == 'socks4a':
-        socks_type = ProxyType.SOCKS4A
-
-    def unquote_if_non_empty(s):
-        if not s:
-            return s
-        return urllib.parse.unquote_plus(s)
-
-    proxy_args = (
-        socks_type,
-        url_components.hostname, url_components.port or 1080,
-        True,  # Remote DNS
-        unquote_if_non_empty(url_components.username),
-        unquote_if_non_empty(url_components.password),
-    )
-
-    class SocksConnection(base_class):
-        def connect(self):
-            self.sock = sockssocket()
-            self.sock.setproxy(*proxy_args)
-            if isinstance(self.timeout, (int, float)):
-                self.sock.settimeout(self.timeout)
-            self.sock.connect((self.host, self.port))
-
-            if isinstance(self, http.client.HTTPSConnection):
-                if hasattr(self, '_context'):  # Python > 2.6
-                    self.sock = self._context.wrap_socket(
-                        self.sock, server_hostname=self.host)
-                else:
-                    self.sock = ssl.wrap_socket(self.sock)
-
-    return SocksConnection
-
-
 class YoutubeDLHTTPSHandler(urllib.request.HTTPSHandler):
     def __init__(self, params, https_conn_class=None, *args, **kwargs):
         urllib.request.HTTPSHandler.__init__(self, *args, **kwargs)
@@ -1507,9 +1157,11 @@ def https_open(self, req):
 
         socks_proxy = req.headers.get('Ytdl-socks-proxy')
         if socks_proxy:
+            from ..networking._urllib import make_socks_conn_class
             conn_class = make_socks_conn_class(conn_class, socks_proxy)
             del req.headers['Ytdl-socks-proxy']
 
+        from ..networking._urllib import _create_http_connection
         try:
             return self.do_open(
                 functools.partial(_create_http_connection, self, conn_class, True), req, **kwargs)
@@ -1535,56 +1187,6 @@ def http_response(self, request, response):
     https_response = http_response
 
 
-class YoutubeDLRedirectHandler(urllib.request.HTTPRedirectHandler):
-    """YoutubeDL redirect handler
-
-    The code is based on HTTPRedirectHandler implementation from CPython [1].
-
-    This redirect handler fixes and improves the logic to better align with RFC7261
-     and what browsers tend to do [2][3]
-
-    1. https://github.com/python/cpython/blob/master/Lib/urllib/request.py
-    2. https://datatracker.ietf.org/doc/html/rfc7231
-    3. https://github.com/python/cpython/issues/91306
-    """
-
-    http_error_301 = http_error_303 = http_error_307 = http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
-
-    def redirect_request(self, req, fp, code, msg, headers, newurl):
-        if code not in (301, 302, 303, 307, 308):
-            raise urllib.error.HTTPError(req.full_url, code, msg, headers, fp)
-
-        new_method = req.get_method()
-        new_data = req.data
-
-        # Technically the Cookie header should be in unredirected_hdrs,
-        # however in practice some may set it in normal headers anyway.
-        # We will remove it here to prevent any leaks.
-        remove_headers = ['Cookie']
-
-        # A 303 must either use GET or HEAD for subsequent request
-        # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.4
-        if code == 303 and req.get_method() != 'HEAD':
-            new_method = 'GET'
-        # 301 and 302 redirects are commonly turned into a GET from a POST
-        # for subsequent requests by browsers, so we'll do the same.
-        # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.2
-        # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.3
-        elif code in (301, 302) and req.get_method() == 'POST':
-            new_method = 'GET'
-
-        # only remove payload if method changed (e.g. POST to GET)
-        if new_method != req.get_method():
-            new_data = None
-            remove_headers.extend(['Content-Length', 'Content-Type'])
-
-        new_headers = {k: v for k, v in req.headers.items() if k.title() not in remove_headers}
-
-        return urllib.request.Request(
-            newurl, headers=new_headers, origin_req_host=req.origin_req_host,
-            unverifiable=True, method=new_method, data=new_data)
-
-
 def extract_timezone(date_str):
     m = re.search(
         r'''(?x)
@@ -2390,16 +1992,6 @@ def urljoin(base, path):
     return urllib.parse.urljoin(base, path)
 
 
-class HEADRequest(urllib.request.Request):
-    def get_method(self):
-        return 'HEAD'
-
-
-class PUTRequest(urllib.request.Request):
-    def get_method(self):
-        return 'PUT'
-
-
 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
     if get_attr and v is not None:
         v = getattr(v, get_attr, None)
@@ -3016,26 +2608,6 @@ def update_url_query(url, query):
     return update_url(url, query_update=query)
 
 
-def update_Request(req, url=None, data=None, headers=None, query=None):
-    req_headers = req.headers.copy()
-    req_headers.update(headers or {})
-    req_data = data or req.data
-    req_url = update_url_query(url or req.get_full_url(), query)
-    req_get_method = req.get_method()
-    if req_get_method == 'HEAD':
-        req_type = HEADRequest
-    elif req_get_method == 'PUT':
-        req_type = PUTRequest
-    else:
-        req_type = urllib.request.Request
-    new_req = req_type(
-        req_url, data=req_data, headers=req_headers,
-        origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
-    if hasattr(req, 'timeout'):
-        new_req.timeout = req.timeout
-    return new_req
-
-
 def _multipart_encode_impl(data, boundary):
     content_type = 'multipart/form-data; boundary=%s' % boundary
 
@@ -4769,31 +4341,6 @@ def random_ipv4(cls, code_or_block):
             struct.pack('!L', random.randint(addr_min, addr_max))))
 
 
-class PerRequestProxyHandler(urllib.request.ProxyHandler):
-    def __init__(self, proxies=None):
-        # Set default handlers
-        for type in ('http', 'https'):
-            setattr(self, '%s_open' % type,
-                    lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
-                        meth(r, proxy, type))
-        urllib.request.ProxyHandler.__init__(self, proxies)
-
-    def proxy_open(self, req, proxy, type):
-        req_proxy = req.headers.get('Ytdl-request-proxy')
-        if req_proxy is not None:
-            proxy = req_proxy
-            del req.headers['Ytdl-request-proxy']
-
-        if proxy == '__noproxy__':
-            return None  # No Proxy
-        if urllib.parse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
-            req.add_header('Ytdl-socks-proxy', proxy)
-            # yt-dlp's http/https handlers do wrapping the socket with socks
-            return None
-        return urllib.request.ProxyHandler.proxy_open(
-            self, req, proxy, type)
-
-
 # Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
 # released into Public Domain
 # https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py
new file mode 100644
index 000000000..95b54fabe
--- /dev/null
+++ b/yt_dlp/utils/networking.py
@@ -0,0 +1,60 @@
+import random
+
+
+def random_user_agent():
+    _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
+    _CHROME_VERSIONS = (
+        '90.0.4430.212',
+        '90.0.4430.24',
+        '90.0.4430.70',
+        '90.0.4430.72',
+        '90.0.4430.85',
+        '90.0.4430.93',
+        '91.0.4472.101',
+        '91.0.4472.106',
+        '91.0.4472.114',
+        '91.0.4472.124',
+        '91.0.4472.164',
+        '91.0.4472.19',
+        '91.0.4472.77',
+        '92.0.4515.107',
+        '92.0.4515.115',
+        '92.0.4515.131',
+        '92.0.4515.159',
+        '92.0.4515.43',
+        '93.0.4556.0',
+        '93.0.4577.15',
+        '93.0.4577.63',
+        '93.0.4577.82',
+        '94.0.4606.41',
+        '94.0.4606.54',
+        '94.0.4606.61',
+        '94.0.4606.71',
+        '94.0.4606.81',
+        '94.0.4606.85',
+        '95.0.4638.17',
+        '95.0.4638.50',
+        '95.0.4638.54',
+        '95.0.4638.69',
+        '95.0.4638.74',
+        '96.0.4664.18',
+        '96.0.4664.45',
+        '96.0.4664.55',
+        '96.0.4664.93',
+        '97.0.4692.20',
+    )
+    return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)
+
+
+std_headers = {
+    'User-Agent': random_user_agent(),
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-us,en;q=0.5',
+    'Sec-Fetch-Mode': 'navigate',
+}
+
+
+def clean_headers(headers):
+    if 'Youtubedl-no-compression' in headers:  # compat
+        del headers['Youtubedl-no-compression']
+        headers['Accept-Encoding'] = 'identity'

From 227bf1a33be7b89cd7d44ad046844c4ccba104f4 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 15 Jul 2023 15:55:23 +0530
Subject: [PATCH 447/871] [networking] Rewrite architecture (#2861)

New networking interface consists of a `RequestDirector` that directs
each `Request` to appropriate `RequestHandler` and returns the
`Response` or raises `RequestError`. The handlers define adapters to
transform its internal Request/Response/Errors to our interfaces.

User-facing changes:
- Fix issues with per request proxies on redirects for urllib
- Support for `ALL_PROXY` environment variable for proxy setting
- Support for `socks5h` proxy
   - Closes https://github.com/yt-dlp/yt-dlp/issues/6325, https://github.com/ytdl-org/youtube-dl/issues/22618, https://github.com/ytdl-org/youtube-dl/pull/28093
- Raise error when using `https` proxy instead of silently converting it to `http`

Authored by: coletdjnz
---
 test/test_download.py           |    9 +-
 test/test_networking.py         | 1351 +++++++++++++++++++++++++------
 test/test_networking_utils.py   |  239 ++++++
 test/test_utils.py              |   46 +-
 yt_dlp/YoutubeDL.py             |  175 ++--
 yt_dlp/compat/__init__.py       |   10 +
 yt_dlp/downloader/http.py       |   24 +-
 yt_dlp/extractor/common.py      |   32 +-
 yt_dlp/networking/__init__.py   |   13 +
 yt_dlp/networking/_helper.py    |   91 ++-
 yt_dlp/networking/_urllib.py    |  231 +++++-
 yt_dlp/networking/common.py     |  522 ++++++++++++
 yt_dlp/networking/exceptions.py |  202 ++++-
 yt_dlp/utils/_deprecated.py     |   13 +-
 yt_dlp/utils/_utils.py          |   35 +-
 yt_dlp/utils/networking.py      |   67 +-
 16 files changed, 2586 insertions(+), 474 deletions(-)
 create mode 100644 test/test_networking_utils.py
 create mode 100644 yt_dlp/networking/common.py

diff --git a/test/test_download.py b/test/test_download.py
index 43b39c36b..fd7752cdd 100755
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -10,10 +10,7 @@
 
 import collections
 import hashlib
-import http.client
 import json
-import socket
-import urllib.error
 
 from test.helper import (
     assertGreaterEqual,
@@ -29,6 +26,7 @@
 
 import yt_dlp.YoutubeDL  # isort: split
 from yt_dlp.extractor import get_info_extractor
+from yt_dlp.networking.exceptions import HTTPError, TransportError
 from yt_dlp.utils import (
     DownloadError,
     ExtractorError,
@@ -162,8 +160,7 @@ def try_rm_tcs_files(tcs=None):
                         force_generic_extractor=params.get('force_generic_extractor', False))
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
-                    if (err.exc_info[0] not in (urllib.error.URLError, socket.timeout, UnavailableVideoError, http.client.BadStatusLine)
-                            or (err.exc_info[0] == urllib.error.HTTPError and err.exc_info[1].code == 503)):
+                    if not isinstance(err.exc_info[1], (TransportError, UnavailableVideoError)) or (isinstance(err.exc_info[1], HTTPError) and err.exc_info[1].code == 503):
                         err.msg = f'{getattr(err, "msg", err)} ({tname})'
                         raise
 
@@ -249,7 +246,7 @@ def try_rm_tcs_files(tcs=None):
                 # extractor returns full results even with extract_flat
                 res_tcs = [{'info_dict': e} for e in res_dict['entries']]
                 try_rm_tcs_files(res_tcs)
-
+            ydl.close()
     return test_template
 
 
diff --git a/test/test_networking.py b/test/test_networking.py
index e4e66dce1..147a4ff49 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -3,32 +3,74 @@
 # Allow direct execution
 import os
 import sys
-import unittest
+
+import pytest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import functools
 import gzip
+import http.client
 import http.cookiejar
 import http.server
+import inspect
 import io
 import pathlib
+import random
 import ssl
 import tempfile
 import threading
+import time
 import urllib.error
 import urllib.request
+import warnings
 import zlib
+from email.message import Message
+from http.cookiejar import CookieJar
 
-from test.helper import http_server_port
-from yt_dlp import YoutubeDL
+from test.helper import FakeYDL, http_server_port
 from yt_dlp.dependencies import brotli
-from yt_dlp.utils import sanitized_Request, urlencode_postdata
-
-from .helper import FakeYDL
+from yt_dlp.networking import (
+    HEADRequest,
+    PUTRequest,
+    Request,
+    RequestDirector,
+    RequestHandler,
+    Response,
+)
+from yt_dlp.networking._urllib import UrllibRH
+from yt_dlp.networking.common import _REQUEST_HANDLERS
+from yt_dlp.networking.exceptions import (
+    CertificateVerifyError,
+    HTTPError,
+    IncompleteRead,
+    NoSupportingHandlers,
+    RequestError,
+    SSLError,
+    TransportError,
+    UnsupportedRequest,
+)
+from yt_dlp.utils._utils import _YDLLogger as FakeLogger
+from yt_dlp.utils.networking import HTTPHeaderDict
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
+def _build_proxy_handler(name):
+    class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
+        proxy_name = name
+
+        def log_message(self, format, *args):
+            pass
+
+        def do_GET(self):
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain; charset=utf-8')
+            self.end_headers()
+            self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode())
+    return HTTPTestRequestHandler
+
+
 class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
     protocol_version = 'HTTP/1.1'
 
@@ -36,7 +78,7 @@ def log_message(self, format, *args):
         pass
 
     def _headers(self):
-        payload = str(self.headers).encode('utf-8')
+        payload = str(self.headers).encode()
         self.send_response(200)
         self.send_header('Content-Type', 'application/json')
         self.send_header('Content-Length', str(len(payload)))
@@ -70,7 +112,7 @@ def _read_data(self):
             return self.rfile.read(int(self.headers['Content-Length']))
 
     def do_POST(self):
-        data = self._read_data()
+        data = self._read_data() + str(self.headers).encode()
         if self.path.startswith('/redirect_'):
             self._redirect()
         elif self.path.startswith('/method'):
@@ -89,7 +131,7 @@ def do_HEAD(self):
             self._status(404)
 
     def do_PUT(self):
-        data = self._read_data()
+        data = self._read_data() + str(self.headers).encode()
         if self.path.startswith('/redirect_'):
             self._redirect()
         elif self.path.startswith('/method'):
@@ -102,7 +144,7 @@ def do_GET(self):
             payload = b'<html><video src="/vid.mp4" /></html>'
             self.send_response(200)
             self.send_header('Content-Type', 'text/html; charset=utf-8')
-            self.send_header('Content-Length', str(len(payload)))  # required for persistent connections
+            self.send_header('Content-Length', str(len(payload)))
             self.end_headers()
             self.wfile.write(payload)
         elif self.path == '/vid.mp4':
@@ -126,10 +168,15 @@ def do_GET(self):
             self.send_header('Content-Length', str(len(payload)))
             self.end_headers()
             self.wfile.write(payload)
+        elif self.path.startswith('/redirect_loop'):
+            self.send_response(301)
+            self.send_header('Location', self.path)
+            self.send_header('Content-Length', '0')
+            self.end_headers()
         elif self.path.startswith('/redirect_'):
             self._redirect()
         elif self.path.startswith('/method'):
-            self._method('GET')
+            self._method('GET', str(self.headers).encode())
         elif self.path.startswith('/headers'):
             self._headers()
         elif self.path.startswith('/308-to-headers'):
@@ -179,7 +226,32 @@ def do_GET(self):
             self.send_header('Content-Length', str(len(payload)))
             self.end_headers()
             self.wfile.write(payload)
-
+        elif self.path.startswith('/gen_'):
+            payload = b'<html></html>'
+            self.send_response(int(self.path[len('/gen_'):]))
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.send_header('Content-Length', str(len(payload)))
+            self.end_headers()
+            self.wfile.write(payload)
+        elif self.path.startswith('/incompleteread'):
+            payload = b'<html></html>'
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.send_header('Content-Length', '234234')
+            self.end_headers()
+            self.wfile.write(payload)
+            self.finish()
+        elif self.path.startswith('/timeout_'):
+            time.sleep(int(self.path[len('/timeout_'):]))
+            self._headers()
+        elif self.path == '/source_address':
+            payload = str(self.client_address[0]).encode()
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.send_header('Content-Length', str(len(payload)))
+            self.end_headers()
+            self.wfile.write(payload)
+            self.finish()
         else:
             self._status(404)
 
@@ -198,334 +270,1099 @@ def send_header(self, keyword, value):
         self._headers_buffer.append(f'{keyword}: {value}\r\n'.encode())
 
 
-class FakeLogger:
-    def debug(self, msg):
-        pass
-
-    def warning(self, msg):
-        pass
-
-    def error(self, msg):
-        pass
+def validate_and_send(rh, req):
+    rh.validate(req)
+    return rh.send(req)
 
 
-class TestHTTP(unittest.TestCase):
-    def setUp(self):
-        # HTTP server
-        self.http_httpd = http.server.ThreadingHTTPServer(
+class TestRequestHandlerBase:
+    @classmethod
+    def setup_class(cls):
+        cls.http_httpd = http.server.ThreadingHTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
-        self.http_port = http_server_port(self.http_httpd)
-        self.http_server_thread = threading.Thread(target=self.http_httpd.serve_forever)
+        cls.http_port = http_server_port(cls.http_httpd)
+        cls.http_server_thread = threading.Thread(target=cls.http_httpd.serve_forever)
         # FIXME: we should probably stop the http server thread after each test
         # See: https://github.com/yt-dlp/yt-dlp/pull/7094#discussion_r1199746041
-        self.http_server_thread.daemon = True
-        self.http_server_thread.start()
+        cls.http_server_thread.daemon = True
+        cls.http_server_thread.start()
 
         # HTTPS server
         certfn = os.path.join(TEST_DIR, 'testcert.pem')
-        self.https_httpd = http.server.ThreadingHTTPServer(
+        cls.https_httpd = http.server.ThreadingHTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
         sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         sslctx.load_cert_chain(certfn, None)
-        self.https_httpd.socket = sslctx.wrap_socket(self.https_httpd.socket, server_side=True)
-        self.https_port = http_server_port(self.https_httpd)
-        self.https_server_thread = threading.Thread(target=self.https_httpd.serve_forever)
-        self.https_server_thread.daemon = True
-        self.https_server_thread.start()
+        cls.https_httpd.socket = sslctx.wrap_socket(cls.https_httpd.socket, server_side=True)
+        cls.https_port = http_server_port(cls.https_httpd)
+        cls.https_server_thread = threading.Thread(target=cls.https_httpd.serve_forever)
+        cls.https_server_thread.daemon = True
+        cls.https_server_thread.start()
 
-    def test_nocheckcertificate(self):
-        with FakeYDL({'logger': FakeLogger()}) as ydl:
-            with self.assertRaises(urllib.error.URLError):
-                ydl.urlopen(sanitized_Request(f'https://127.0.0.1:{self.https_port}/headers'))
 
-        with FakeYDL({'logger': FakeLogger(), 'nocheckcertificate': True}) as ydl:
-            r = ydl.urlopen(sanitized_Request(f'https://127.0.0.1:{self.https_port}/headers'))
-            self.assertEqual(r.status, 200)
+@pytest.fixture
+def handler(request):
+    RH_KEY = request.param
+    if inspect.isclass(RH_KEY) and issubclass(RH_KEY, RequestHandler):
+        handler = RH_KEY
+    elif RH_KEY in _REQUEST_HANDLERS:
+        handler = _REQUEST_HANDLERS[RH_KEY]
+    else:
+        pytest.skip(f'{RH_KEY} request handler is not available')
+
+    return functools.partial(handler, logger=FakeLogger)
+
+
+class TestHTTPRequestHandler(TestRequestHandlerBase):
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_verify_cert(self, handler):
+        with handler() as rh:
+            with pytest.raises(CertificateVerifyError):
+                validate_and_send(rh, Request(f'https://127.0.0.1:{self.https_port}/headers'))
+
+        with handler(verify=False) as rh:
+            r = validate_and_send(rh, Request(f'https://127.0.0.1:{self.https_port}/headers'))
+            assert r.status == 200
             r.close()
 
-    def test_percent_encode(self):
-        with FakeYDL() as ydl:
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_ssl_error(self, handler):
+        # HTTPS server with too old TLS version
+        # XXX: is there a better way to test this than to create a new server?
+        https_httpd = http.server.ThreadingHTTPServer(
+            ('127.0.0.1', 0), HTTPTestRequestHandler)
+        sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        https_httpd.socket = sslctx.wrap_socket(https_httpd.socket, server_side=True)
+        https_port = http_server_port(https_httpd)
+        https_server_thread = threading.Thread(target=https_httpd.serve_forever)
+        https_server_thread.daemon = True
+        https_server_thread.start()
+
+        with handler(verify=False) as rh:
+            with pytest.raises(SSLError, match='sslv3 alert handshake failure') as exc_info:
+                validate_and_send(rh, Request(f'https://127.0.0.1:{https_port}/headers'))
+            assert not issubclass(exc_info.type, CertificateVerifyError)
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_percent_encode(self, handler):
+        with handler() as rh:
             # Unicode characters should be encoded with uppercase percent-encoding
-            res = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/中文.html'))
-            self.assertEqual(res.status, 200)
+            res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/中文.html'))
+            assert res.status == 200
             res.close()
             # don't normalize existing percent encodings
-            res = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/%c7%9f'))
-            self.assertEqual(res.status, 200)
+            res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/%c7%9f'))
+            assert res.status == 200
             res.close()
 
-    def test_unicode_path_redirection(self):
-        with FakeYDL() as ydl:
-            r = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/302-non-ascii-redirect'))
-            self.assertEqual(r.url, f'http://127.0.0.1:{self.http_port}/%E4%B8%AD%E6%96%87.html')
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_unicode_path_redirection(self, handler):
+        with handler() as rh:
+            r = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/302-non-ascii-redirect'))
+            assert r.url == f'http://127.0.0.1:{self.http_port}/%E4%B8%AD%E6%96%87.html'
             r.close()
 
-    def test_redirect(self):
-        with FakeYDL() as ydl:
-            def do_req(redirect_status, method):
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_raise_http_error(self, handler):
+        with handler() as rh:
+            for bad_status in (400, 500, 599, 302):
+                with pytest.raises(HTTPError):
+                    validate_and_send(rh, Request('http://127.0.0.1:%d/gen_%d' % (self.http_port, bad_status)))
+
+            # Should not raise an error
+            validate_and_send(rh, Request('http://127.0.0.1:%d/gen_200' % self.http_port)).close()
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_response_url(self, handler):
+        with handler() as rh:
+            # Response url should be that of the last url in redirect chain
+            res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/redirect_301'))
+            assert res.url == f'http://127.0.0.1:{self.http_port}/method'
+            res.close()
+            res2 = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/gen_200'))
+            assert res2.url == f'http://127.0.0.1:{self.http_port}/gen_200'
+            res2.close()
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_redirect(self, handler):
+        with handler() as rh:
+            def do_req(redirect_status, method, assert_no_content=False):
                 data = b'testdata' if method in ('POST', 'PUT') else None
-                res = ydl.urlopen(sanitized_Request(
-                    f'http://127.0.0.1:{self.http_port}/redirect_{redirect_status}', method=method, data=data))
-                return res.read().decode('utf-8'), res.headers.get('method', '')
+                res = validate_and_send(
+                    rh, Request(f'http://127.0.0.1:{self.http_port}/redirect_{redirect_status}', method=method, data=data))
+
+                headers = b''
+                data_sent = b''
+                if data is not None:
+                    data_sent += res.read(len(data))
+                    if data_sent != data:
+                        headers += data_sent
+                        data_sent = b''
+
+                headers += res.read()
+
+                if assert_no_content or data is None:
+                    assert b'Content-Type' not in headers
+                    assert b'Content-Length' not in headers
+                else:
+                    assert b'Content-Type' in headers
+                    assert b'Content-Length' in headers
+
+                return data_sent.decode(), res.headers.get('method', '')
 
             # A 303 must either use GET or HEAD for subsequent request
-            self.assertEqual(do_req(303, 'POST'), ('', 'GET'))
-            self.assertEqual(do_req(303, 'HEAD'), ('', 'HEAD'))
+            assert do_req(303, 'POST', True) == ('', 'GET')
+            assert do_req(303, 'HEAD') == ('', 'HEAD')
 
-            self.assertEqual(do_req(303, 'PUT'), ('', 'GET'))
+            assert do_req(303, 'PUT', True) == ('', 'GET')
 
             # 301 and 302 turn POST only into a GET
-            # XXX: we should also test if the Content-Type and Content-Length headers are removed
-            self.assertEqual(do_req(301, 'POST'), ('', 'GET'))
-            self.assertEqual(do_req(301, 'HEAD'), ('', 'HEAD'))
-            self.assertEqual(do_req(302, 'POST'), ('', 'GET'))
-            self.assertEqual(do_req(302, 'HEAD'), ('', 'HEAD'))
+            assert do_req(301, 'POST', True) == ('', 'GET')
+            assert do_req(301, 'HEAD') == ('', 'HEAD')
+            assert do_req(302, 'POST', True) == ('', 'GET')
+            assert do_req(302, 'HEAD') == ('', 'HEAD')
 
-            self.assertEqual(do_req(301, 'PUT'), ('testdata', 'PUT'))
-            self.assertEqual(do_req(302, 'PUT'), ('testdata', 'PUT'))
+            assert do_req(301, 'PUT') == ('testdata', 'PUT')
+            assert do_req(302, 'PUT') == ('testdata', 'PUT')
 
             # 307 and 308 should not change method
             for m in ('POST', 'PUT'):
-                self.assertEqual(do_req(307, m), ('testdata', m))
-                self.assertEqual(do_req(308, m), ('testdata', m))
+                assert do_req(307, m) == ('testdata', m)
+                assert do_req(308, m) == ('testdata', m)
 
-            self.assertEqual(do_req(307, 'HEAD'), ('', 'HEAD'))
-            self.assertEqual(do_req(308, 'HEAD'), ('', 'HEAD'))
+            assert do_req(307, 'HEAD') == ('', 'HEAD')
+            assert do_req(308, 'HEAD') == ('', 'HEAD')
 
             # These should not redirect and instead raise an HTTPError
             for code in (300, 304, 305, 306):
-                with self.assertRaises(urllib.error.HTTPError):
+                with pytest.raises(HTTPError):
                     do_req(code, 'GET')
 
-    def test_content_type(self):
-        # https://github.com/yt-dlp/yt-dlp/commit/379a4f161d4ad3e40932dcf5aca6e6fb9715ab28
-        with FakeYDL({'nocheckcertificate': True}) as ydl:
-            # method should be auto-detected as POST
-            r = sanitized_Request(f'https://localhost:{self.https_port}/headers', data=urlencode_postdata({'test': 'test'}))
-
-            headers = ydl.urlopen(r).read().decode('utf-8')
-            self.assertIn('Content-Type: application/x-www-form-urlencoded', headers)
-
-            # test http
-            r = sanitized_Request(f'http://localhost:{self.http_port}/headers', data=urlencode_postdata({'test': 'test'}))
-            headers = ydl.urlopen(r).read().decode('utf-8')
-            self.assertIn('Content-Type: application/x-www-form-urlencoded', headers)
-
-    def test_cookiejar(self):
-        with FakeYDL() as ydl:
-            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(
-                0, 'test', 'ytdlp', None, False, '127.0.0.1', True,
-                False, '/headers', True, False, None, False, None, None, {}))
-            data = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/headers')).read()
-            self.assertIn(b'Cookie: test=ytdlp', data)
-
-    def test_passed_cookie_header(self):
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_request_cookie_header(self, handler):
         # We should accept a Cookie header being passed as in normal headers and handle it appropriately.
-        with FakeYDL() as ydl:
+        with handler() as rh:
             # Specified Cookie header should be used
-            res = ydl.urlopen(
-                sanitized_Request(f'http://127.0.0.1:{self.http_port}/headers',
-                                  headers={'Cookie': 'test=test'})).read().decode('utf-8')
-            self.assertIn('Cookie: test=test', res)
+            res = validate_and_send(
+                rh, Request(
+                    f'http://127.0.0.1:{self.http_port}/headers',
+                    headers={'Cookie': 'test=test'})).read().decode()
+            assert 'Cookie: test=test' in res
 
             # Specified Cookie header should be removed on any redirect
-            res = ydl.urlopen(
-                sanitized_Request(f'http://127.0.0.1:{self.http_port}/308-to-headers', headers={'Cookie': 'test=test'})).read().decode('utf-8')
-            self.assertNotIn('Cookie: test=test', res)
+            res = validate_and_send(
+                rh, Request(
+                    f'http://127.0.0.1:{self.http_port}/308-to-headers',
+                    headers={'Cookie': 'test=test'})).read().decode()
+            assert 'Cookie: test=test' not in res
 
-            # Specified Cookie header should override global cookiejar for that request
-            ydl.cookiejar.set_cookie(http.cookiejar.Cookie(
-                version=0, name='test', value='ytdlp', port=None, port_specified=False,
-                domain='127.0.0.1', domain_specified=True, domain_initial_dot=False, path='/',
-                path_specified=True, secure=False, expires=None, discard=False, comment=None,
-                comment_url=None, rest={}))
+        # Specified Cookie header should override global cookiejar for that request
+        cookiejar = http.cookiejar.CookieJar()
+        cookiejar.set_cookie(http.cookiejar.Cookie(
+            version=0, name='test', value='ytdlp', port=None, port_specified=False,
+            domain='127.0.0.1', domain_specified=True, domain_initial_dot=False, path='/',
+            path_specified=True, secure=False, expires=None, discard=False, comment=None,
+            comment_url=None, rest={}))
 
-            data = ydl.urlopen(sanitized_Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'Cookie': 'test=test'})).read()
-            self.assertNotIn(b'Cookie: test=ytdlp', data)
-            self.assertIn(b'Cookie: test=test', data)
+        with handler(cookiejar=cookiejar) as rh:
+            data = validate_and_send(
+                rh, Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'cookie': 'test=test'})).read()
+            assert b'Cookie: test=ytdlp' not in data
+            assert b'Cookie: test=test' in data
 
-    def test_no_compression_compat_header(self):
-        with FakeYDL() as ydl:
-            data = ydl.urlopen(
-                sanitized_Request(
-                    f'http://127.0.0.1:{self.http_port}/headers',
-                    headers={'Youtubedl-no-compression': True})).read()
-            self.assertIn(b'Accept-Encoding: identity', data)
-            self.assertNotIn(b'youtubedl-no-compression', data.lower())
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_redirect_loop(self, handler):
+        with handler() as rh:
+            with pytest.raises(HTTPError, match='redirect loop'):
+                validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/redirect_loop'))
 
-    def test_gzip_trailing_garbage(self):
-        # https://github.com/ytdl-org/youtube-dl/commit/aa3e950764337ef9800c936f4de89b31c00dfcf5
-        # https://github.com/ytdl-org/youtube-dl/commit/6f2ec15cee79d35dba065677cad9da7491ec6e6f
-        with FakeYDL() as ydl:
-            data = ydl.urlopen(sanitized_Request(f'http://localhost:{self.http_port}/trailing_garbage')).read().decode('utf-8')
-            self.assertEqual(data, '<html><video src="/vid.mp4" /></html>')
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_incompleteread(self, handler):
+        with handler(timeout=2) as rh:
+            with pytest.raises(IncompleteRead):
+                validate_and_send(rh, Request('http://127.0.0.1:%d/incompleteread' % self.http_port)).read()
 
-    @unittest.skipUnless(brotli, 'brotli support is not installed')
-    def test_brotli(self):
-        with FakeYDL() as ydl:
-            res = ydl.urlopen(
-                sanitized_Request(
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_cookies(self, handler):
+        cookiejar = http.cookiejar.CookieJar()
+        cookiejar.set_cookie(http.cookiejar.Cookie(
+            0, 'test', 'ytdlp', None, False, '127.0.0.1', True,
+            False, '/headers', True, False, None, False, None, None, {}))
+
+        with handler(cookiejar=cookiejar) as rh:
+            data = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/headers')).read()
+            assert b'Cookie: test=ytdlp' in data
+
+        # Per request
+        with handler() as rh:
+            data = validate_and_send(
+                rh, Request(f'http://127.0.0.1:{self.http_port}/headers', extensions={'cookiejar': cookiejar})).read()
+            assert b'Cookie: test=ytdlp' in data
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_headers(self, handler):
+
+        with handler(headers=HTTPHeaderDict({'test1': 'test', 'test2': 'test2'})) as rh:
+            # Global Headers
+            data = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/headers')).read()
+            assert b'Test1: test' in data
+
+            # Per request headers, merged with global
+            data = validate_and_send(rh, Request(
+                f'http://127.0.0.1:{self.http_port}/headers', headers={'test2': 'changed', 'test3': 'test3'})).read()
+            assert b'Test1: test' in data
+            assert b'Test2: changed' in data
+            assert b'Test2: test2' not in data
+            assert b'Test3: test3' in data
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_timeout(self, handler):
+        with handler() as rh:
+            # Default timeout is 20 seconds, so this should go through
+            validate_and_send(
+                rh, Request(f'http://127.0.0.1:{self.http_port}/timeout_3'))
+
+        with handler(timeout=0.5) as rh:
+            with pytest.raises(TransportError):
+                validate_and_send(
+                    rh, Request(f'http://127.0.0.1:{self.http_port}/timeout_1'))
+
+            # Per request timeout, should override handler timeout
+            validate_and_send(
+                rh, Request(f'http://127.0.0.1:{self.http_port}/timeout_1', extensions={'timeout': 4}))
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_source_address(self, handler):
+        source_address = f'127.0.0.{random.randint(5, 255)}'
+        with handler(source_address=source_address) as rh:
+            data = validate_and_send(
+                rh, Request(f'http://127.0.0.1:{self.http_port}/source_address')).read().decode()
+            assert source_address == data
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_gzip_trailing_garbage(self, handler):
+        with handler() as rh:
+            data = validate_and_send(rh, Request(f'http://localhost:{self.http_port}/trailing_garbage')).read().decode()
+            assert data == '<html><video src="/vid.mp4" /></html>'
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.skipif(not brotli, reason='brotli support is not installed')
+    def test_brotli(self, handler):
+        with handler() as rh:
+            res = validate_and_send(
+                rh, Request(
                     f'http://127.0.0.1:{self.http_port}/content-encoding',
                     headers={'ytdl-encoding': 'br'}))
-            self.assertEqual(res.headers.get('Content-Encoding'), 'br')
-            self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+            assert res.headers.get('Content-Encoding') == 'br'
+            assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    def test_deflate(self):
-        with FakeYDL() as ydl:
-            res = ydl.urlopen(
-                sanitized_Request(
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_deflate(self, handler):
+        with handler() as rh:
+            res = validate_and_send(
+                rh, Request(
                     f'http://127.0.0.1:{self.http_port}/content-encoding',
                     headers={'ytdl-encoding': 'deflate'}))
-            self.assertEqual(res.headers.get('Content-Encoding'), 'deflate')
-            self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+            assert res.headers.get('Content-Encoding') == 'deflate'
+            assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    def test_gzip(self):
-        with FakeYDL() as ydl:
-            res = ydl.urlopen(
-                sanitized_Request(
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_gzip(self, handler):
+        with handler() as rh:
+            res = validate_and_send(
+                rh, Request(
                     f'http://127.0.0.1:{self.http_port}/content-encoding',
                     headers={'ytdl-encoding': 'gzip'}))
-            self.assertEqual(res.headers.get('Content-Encoding'), 'gzip')
-            self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+            assert res.headers.get('Content-Encoding') == 'gzip'
+            assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    def test_multiple_encodings(self):
-        # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.4
-        with FakeYDL() as ydl:
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_multiple_encodings(self, handler):
+        with handler() as rh:
             for pair in ('gzip,deflate', 'deflate, gzip', 'gzip, gzip', 'deflate, deflate'):
-                res = ydl.urlopen(
-                    sanitized_Request(
+                res = validate_and_send(
+                    rh, Request(
                         f'http://127.0.0.1:{self.http_port}/content-encoding',
                         headers={'ytdl-encoding': pair}))
-                self.assertEqual(res.headers.get('Content-Encoding'), pair)
-                self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+                assert res.headers.get('Content-Encoding') == pair
+                assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    def test_unsupported_encoding(self):
-        # it should return the raw content
-        with FakeYDL() as ydl:
-            res = ydl.urlopen(
-                sanitized_Request(
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_unsupported_encoding(self, handler):
+        with handler() as rh:
+            res = validate_and_send(
+                rh, Request(
                     f'http://127.0.0.1:{self.http_port}/content-encoding',
                     headers={'ytdl-encoding': 'unsupported'}))
-            self.assertEqual(res.headers.get('Content-Encoding'), 'unsupported')
-            self.assertEqual(res.read(), b'raw')
+            assert res.headers.get('Content-Encoding') == 'unsupported'
+            assert res.read() == b'raw'
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_read(self, handler):
+        with handler() as rh:
+            res = validate_and_send(
+                rh, Request(f'http://127.0.0.1:{self.http_port}/headers'))
+            assert res.readable()
+            assert res.read(1) == b'H'
+            assert res.read(3) == b'ost'
 
 
-class TestClientCert(unittest.TestCase):
-    def setUp(self):
+class TestHTTPProxy(TestRequestHandlerBase):
+    @classmethod
+    def setup_class(cls):
+        super().setup_class()
+        # HTTP Proxy server
+        cls.proxy = http.server.ThreadingHTTPServer(
+            ('127.0.0.1', 0), _build_proxy_handler('normal'))
+        cls.proxy_port = http_server_port(cls.proxy)
+        cls.proxy_thread = threading.Thread(target=cls.proxy.serve_forever)
+        cls.proxy_thread.daemon = True
+        cls.proxy_thread.start()
+
+        # Geo proxy server
+        cls.geo_proxy = http.server.ThreadingHTTPServer(
+            ('127.0.0.1', 0), _build_proxy_handler('geo'))
+        cls.geo_port = http_server_port(cls.geo_proxy)
+        cls.geo_proxy_thread = threading.Thread(target=cls.geo_proxy.serve_forever)
+        cls.geo_proxy_thread.daemon = True
+        cls.geo_proxy_thread.start()
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_http_proxy(self, handler):
+        http_proxy = f'http://127.0.0.1:{self.proxy_port}'
+        geo_proxy = f'http://127.0.0.1:{self.geo_port}'
+
+        # Test global http proxy
+        # Test per request http proxy
+        # Test per request http proxy disables proxy
+        url = 'http://foo.com/bar'
+
+        # Global HTTP proxy
+        with handler(proxies={'http': http_proxy}) as rh:
+            res = validate_and_send(rh, Request(url)).read().decode()
+            assert res == f'normal: {url}'
+
+            # Per request proxy overrides global
+            res = validate_and_send(rh, Request(url, proxies={'http': geo_proxy})).read().decode()
+            assert res == f'geo: {url}'
+
+            # and setting to None disables all proxies for that request
+            real_url = f'http://127.0.0.1:{self.http_port}/headers'
+            res = validate_and_send(
+                rh, Request(real_url, proxies={'http': None})).read().decode()
+            assert res != f'normal: {real_url}'
+            assert 'Accept' in res
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_noproxy(self, handler):
+        with handler(proxies={'proxy': f'http://127.0.0.1:{self.proxy_port}'}) as rh:
+            # NO_PROXY
+            for no_proxy in (f'127.0.0.1:{self.http_port}', '127.0.0.1', 'localhost'):
+                nop_response = validate_and_send(
+                    rh, Request(f'http://127.0.0.1:{self.http_port}/headers', proxies={'no': no_proxy})).read().decode(
+                    'utf-8')
+                assert 'Accept' in nop_response
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_allproxy(self, handler):
+        url = 'http://foo.com/bar'
+        with handler() as rh:
+            response = validate_and_send(rh, Request(url, proxies={'all': f'http://127.0.0.1:{self.proxy_port}'})).read().decode(
+                'utf-8')
+            assert response == f'normal: {url}'
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_http_proxy_with_idn(self, handler):
+        with handler(proxies={
+            'http': f'http://127.0.0.1:{self.proxy_port}',
+        }) as rh:
+            url = 'http://中文.tw/'
+            response = rh.send(Request(url)).read().decode()
+            # b'xn--fiq228c' is '中文'.encode('idna')
+            assert response == 'normal: http://xn--fiq228c.tw/'
+
+
+class TestClientCertificate:
+
+    @classmethod
+    def setup_class(cls):
         certfn = os.path.join(TEST_DIR, 'testcert.pem')
-        self.certdir = os.path.join(TEST_DIR, 'testdata', 'certificate')
-        cacertfn = os.path.join(self.certdir, 'ca.crt')
-        self.httpd = http.server.HTTPServer(('127.0.0.1', 0), HTTPTestRequestHandler)
+        cls.certdir = os.path.join(TEST_DIR, 'testdata', 'certificate')
+        cacertfn = os.path.join(cls.certdir, 'ca.crt')
+        cls.httpd = http.server.ThreadingHTTPServer(('127.0.0.1', 0), HTTPTestRequestHandler)
         sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         sslctx.verify_mode = ssl.CERT_REQUIRED
         sslctx.load_verify_locations(cafile=cacertfn)
         sslctx.load_cert_chain(certfn, None)
-        self.httpd.socket = sslctx.wrap_socket(self.httpd.socket, server_side=True)
-        self.port = http_server_port(self.httpd)
-        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
-        self.server_thread.daemon = True
-        self.server_thread.start()
+        cls.httpd.socket = sslctx.wrap_socket(cls.httpd.socket, server_side=True)
+        cls.port = http_server_port(cls.httpd)
+        cls.server_thread = threading.Thread(target=cls.httpd.serve_forever)
+        cls.server_thread.daemon = True
+        cls.server_thread.start()
 
-    def _run_test(self, **params):
-        ydl = YoutubeDL({
-            'logger': FakeLogger(),
+    def _run_test(self, handler, **handler_kwargs):
+        with handler(
             # Disable client-side validation of unacceptable self-signed testcert.pem
             # The test is of a check on the server side, so unaffected
-            'nocheckcertificate': True,
-            **params,
+            verify=False,
+            **handler_kwargs,
+        ) as rh:
+            validate_and_send(rh, Request(f'https://127.0.0.1:{self.port}/video.html')).read().decode()
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_certificate_combined_nopass(self, handler):
+        self._run_test(handler, client_cert={
+            'client_certificate': os.path.join(self.certdir, 'clientwithkey.crt'),
         })
-        r = ydl.extract_info(f'https://127.0.0.1:{self.port}/video.html')
-        self.assertEqual(r['url'], f'https://127.0.0.1:{self.port}/vid.mp4')
 
-    def test_certificate_combined_nopass(self):
-        self._run_test(client_certificate=os.path.join(self.certdir, 'clientwithkey.crt'))
-
-    def test_certificate_nocombined_nopass(self):
-        self._run_test(client_certificate=os.path.join(self.certdir, 'client.crt'),
-                       client_certificate_key=os.path.join(self.certdir, 'client.key'))
-
-    def test_certificate_combined_pass(self):
-        self._run_test(client_certificate=os.path.join(self.certdir, 'clientwithencryptedkey.crt'),
-                       client_certificate_password='foobar')
-
-    def test_certificate_nocombined_pass(self):
-        self._run_test(client_certificate=os.path.join(self.certdir, 'client.crt'),
-                       client_certificate_key=os.path.join(self.certdir, 'clientencrypted.key'),
-                       client_certificate_password='foobar')
-
-
-def _build_proxy_handler(name):
-    class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
-        proxy_name = name
-
-        def log_message(self, format, *args):
-            pass
-
-        def do_GET(self):
-            self.send_response(200)
-            self.send_header('Content-Type', 'text/plain; charset=utf-8')
-            self.end_headers()
-            self.wfile.write(f'{self.proxy_name}: {self.path}'.encode())
-    return HTTPTestRequestHandler
-
-
-class TestProxy(unittest.TestCase):
-    def setUp(self):
-        self.proxy = http.server.HTTPServer(
-            ('127.0.0.1', 0), _build_proxy_handler('normal'))
-        self.port = http_server_port(self.proxy)
-        self.proxy_thread = threading.Thread(target=self.proxy.serve_forever)
-        self.proxy_thread.daemon = True
-        self.proxy_thread.start()
-
-        self.geo_proxy = http.server.HTTPServer(
-            ('127.0.0.1', 0), _build_proxy_handler('geo'))
-        self.geo_port = http_server_port(self.geo_proxy)
-        self.geo_proxy_thread = threading.Thread(target=self.geo_proxy.serve_forever)
-        self.geo_proxy_thread.daemon = True
-        self.geo_proxy_thread.start()
-
-    def test_proxy(self):
-        geo_proxy = f'127.0.0.1:{self.geo_port}'
-        ydl = YoutubeDL({
-            'proxy': f'127.0.0.1:{self.port}',
-            'geo_verification_proxy': geo_proxy,
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_certificate_nocombined_nopass(self, handler):
+        self._run_test(handler, client_cert={
+            'client_certificate': os.path.join(self.certdir, 'client.crt'),
+            'client_certificate_key': os.path.join(self.certdir, 'client.key'),
         })
-        url = 'http://foo.com/bar'
-        response = ydl.urlopen(url).read().decode()
-        self.assertEqual(response, f'normal: {url}')
 
-        req = urllib.request.Request(url)
-        req.add_header('Ytdl-request-proxy', geo_proxy)
-        response = ydl.urlopen(req).read().decode()
-        self.assertEqual(response, f'geo: {url}')
-
-    def test_proxy_with_idn(self):
-        ydl = YoutubeDL({
-            'proxy': f'127.0.0.1:{self.port}',
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_certificate_combined_pass(self, handler):
+        self._run_test(handler, client_cert={
+            'client_certificate': os.path.join(self.certdir, 'clientwithencryptedkey.crt'),
+            'client_certificate_password': 'foobar',
+        })
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_certificate_nocombined_pass(self, handler):
+        self._run_test(handler, client_cert={
+            'client_certificate': os.path.join(self.certdir, 'client.crt'),
+            'client_certificate_key': os.path.join(self.certdir, 'clientencrypted.key'),
+            'client_certificate_password': 'foobar',
         })
-        url = 'http://中文.tw/'
-        response = ydl.urlopen(url).read().decode()
-        # b'xn--fiq228c' is '中文'.encode('idna')
-        self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')
 
 
-class TestFileURL(unittest.TestCase):
-    # See https://github.com/ytdl-org/youtube-dl/issues/8227
-    def test_file_urls(self):
+class TestUrllibRequestHandler(TestRequestHandlerBase):
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_file_urls(self, handler):
+        # See https://github.com/ytdl-org/youtube-dl/issues/8227
         tf = tempfile.NamedTemporaryFile(delete=False)
         tf.write(b'foobar')
         tf.close()
-        url = pathlib.Path(tf.name).as_uri()
-        with FakeYDL() as ydl:
-            self.assertRaisesRegex(
-                urllib.error.URLError, 'file:// URLs are explicitly disabled in yt-dlp for security reasons', ydl.urlopen, url)
-        with FakeYDL({'enable_file_urls': True}) as ydl:
-            res = ydl.urlopen(url)
-            self.assertEqual(res.read(), b'foobar')
+        req = Request(pathlib.Path(tf.name).as_uri())
+        with handler() as rh:
+            with pytest.raises(UnsupportedRequest):
+                rh.validate(req)
+
+            # Test that urllib never loaded FileHandler
+            with pytest.raises(TransportError):
+                rh.send(req)
+
+        with handler(enable_file_urls=True) as rh:
+            res = validate_and_send(rh, req)
+            assert res.read() == b'foobar'
             res.close()
+
         os.unlink(tf.name)
 
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_http_error_returns_content(self, handler):
+        # urllib HTTPError will try close the underlying response if reference to the HTTPError object is lost
+        def get_response():
+            with handler() as rh:
+                # headers url
+                try:
+                    validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/gen_404'))
+                except HTTPError as e:
+                    return e.response
 
-if __name__ == '__main__':
-    unittest.main()
+        assert get_response().read() == b'<html></html>'
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_verify_cert_error_text(self, handler):
+        # Check the output of the error message
+        with handler() as rh:
+            with pytest.raises(
+                CertificateVerifyError,
+                match=r'\[SSL: CERTIFICATE_VERIFY_FAILED\] certificate verify failed: self.signed certificate'
+            ):
+                validate_and_send(rh, Request(f'https://127.0.0.1:{self.https_port}/headers'))
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_httplib_validation_errors(self, handler):
+        with handler() as rh:
+
+            # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1256
+            with pytest.raises(RequestError, match='method can\'t contain control characters') as exc_info:
+                validate_and_send(rh, Request('http://127.0.0.1', method='GET\n'))
+            assert not isinstance(exc_info.value, TransportError)
+
+            # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1265
+            with pytest.raises(RequestError, match='URL can\'t contain control characters') as exc_info:
+                validate_and_send(rh, Request('http://127.0.0. 1', method='GET\n'))
+            assert not isinstance(exc_info.value, TransportError)
+
+            # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1288C31-L1288C50
+            with pytest.raises(RequestError, match='Invalid header name') as exc_info:
+                validate_and_send(rh, Request('http://127.0.0.1', headers={'foo\n': 'bar'}))
+            assert not isinstance(exc_info.value, TransportError)
+
+
+def run_validation(handler, fail, req, **handler_kwargs):
+    with handler(**handler_kwargs) as rh:
+        if fail:
+            with pytest.raises(UnsupportedRequest):
+                rh.validate(req)
+        else:
+            rh.validate(req)
+
+
+class TestRequestHandlerValidation:
+
+    class ValidationRH(RequestHandler):
+        def _send(self, request):
+            raise RequestError('test')
+
+    class NoCheckRH(ValidationRH):
+        _SUPPORTED_FEATURES = None
+        _SUPPORTED_PROXY_SCHEMES = None
+        _SUPPORTED_URL_SCHEMES = None
+
+    class HTTPSupportedRH(ValidationRH):
+        _SUPPORTED_URL_SCHEMES = ('http',)
+
+    URL_SCHEME_TESTS = [
+        # scheme, expected to fail, handler kwargs
+        ('Urllib', [
+            ('http', False, {}),
+            ('https', False, {}),
+            ('data', False, {}),
+            ('ftp', False, {}),
+            ('file', True, {}),
+            ('file', False, {'enable_file_urls': True}),
+        ]),
+        (NoCheckRH, [('http', False, {})]),
+        (ValidationRH, [('http', True, {})])
+    ]
+
+    PROXY_SCHEME_TESTS = [
+        # scheme, expected to fail
+        ('Urllib', [
+            ('http', False),
+            ('https', True),
+            ('socks4', False),
+            ('socks4a', False),
+            ('socks5', False),
+            ('socks5h', False),
+            ('socks', True),
+        ]),
+        (NoCheckRH, [('http', False)]),
+        (HTTPSupportedRH, [('http', True)]),
+    ]
+
+    PROXY_KEY_TESTS = [
+        # key, expected to fail
+        ('Urllib', [
+            ('all', False),
+            ('unrelated', False),
+        ]),
+        (NoCheckRH, [('all', False)]),
+        (HTTPSupportedRH, [('all', True)]),
+        (HTTPSupportedRH, [('no', True)]),
+    ]
+
+    @pytest.mark.parametrize('handler,scheme,fail,handler_kwargs', [
+        (handler_tests[0], scheme, fail, handler_kwargs)
+        for handler_tests in URL_SCHEME_TESTS
+        for scheme, fail, handler_kwargs in handler_tests[1]
+
+    ], indirect=['handler'])
+    def test_url_scheme(self, handler, scheme, fail, handler_kwargs):
+        run_validation(handler, fail, Request(f'{scheme}://'), **(handler_kwargs or {}))
+
+    @pytest.mark.parametrize('handler,fail', [('Urllib', False)], indirect=['handler'])
+    def test_no_proxy(self, handler, fail):
+        run_validation(handler, fail, Request('http://', proxies={'no': '127.0.0.1,github.com'}))
+        run_validation(handler, fail, Request('http://'), proxies={'no': '127.0.0.1,github.com'})
+
+    @pytest.mark.parametrize('handler,proxy_key,fail', [
+        (handler_tests[0], proxy_key, fail)
+        for handler_tests in PROXY_KEY_TESTS
+        for proxy_key, fail in handler_tests[1]
+    ], indirect=['handler'])
+    def test_proxy_key(self, handler, proxy_key, fail):
+        run_validation(handler, fail, Request('http://', proxies={proxy_key: 'http://example.com'}))
+        run_validation(handler, fail, Request('http://'), proxies={proxy_key: 'http://example.com'})
+
+    @pytest.mark.parametrize('handler,scheme,fail', [
+        (handler_tests[0], scheme, fail)
+        for handler_tests in PROXY_SCHEME_TESTS
+        for scheme, fail in handler_tests[1]
+    ], indirect=['handler'])
+    def test_proxy_scheme(self, handler, scheme, fail):
+        run_validation(handler, fail, Request('http://', proxies={'http': f'{scheme}://example.com'}))
+        run_validation(handler, fail, Request('http://'), proxies={'http': f'{scheme}://example.com'})
+
+    @pytest.mark.parametrize('handler', ['Urllib', HTTPSupportedRH], indirect=True)
+    def test_empty_proxy(self, handler):
+        run_validation(handler, False, Request('http://', proxies={'http': None}))
+        run_validation(handler, False, Request('http://'), proxies={'http': None})
+
+    @pytest.mark.parametrize('proxy_url', ['//example.com', 'example.com', '127.0.0.1'])
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_missing_proxy_scheme(self, handler, proxy_url):
+        run_validation(handler, True, Request('http://', proxies={'http': 'example.com'}))
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_cookiejar_extension(self, handler):
+        run_validation(handler, True, Request('http://', extensions={'cookiejar': 'notacookiejar'}))
+
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_timeout_extension(self, handler):
+        run_validation(handler, True, Request('http://', extensions={'timeout': 'notavalidtimeout'}))
+
+    def test_invalid_request_type(self):
+        rh = self.ValidationRH(logger=FakeLogger())
+        for method in (rh.validate, rh.send):
+            with pytest.raises(TypeError, match='Expected an instance of Request'):
+                method('not a request')
+
+
+class FakeResponse(Response):
+    def __init__(self, request):
+        # XXX: we could make request part of standard response interface
+        self.request = request
+        super().__init__(fp=io.BytesIO(b''), headers={}, url=request.url)
+
+
+class FakeRH(RequestHandler):
+
+    def _validate(self, request):
+        return
+
+    def _send(self, request: Request):
+        if request.url.startswith('ssl://'):
+            raise SSLError(request.url[len('ssl://'):])
+        return FakeResponse(request)
+
+
+class FakeRHYDL(FakeYDL):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._request_director = self.build_request_director([FakeRH])
+
+
+class TestRequestDirector:
+
+    def test_handler_operations(self):
+        director = RequestDirector(logger=FakeLogger())
+        handler = FakeRH(logger=FakeLogger())
+        director.add_handler(handler)
+        assert director.handlers.get(FakeRH.RH_KEY) is handler
+
+        # Handler should overwrite
+        handler2 = FakeRH(logger=FakeLogger())
+        director.add_handler(handler2)
+        assert director.handlers.get(FakeRH.RH_KEY) is not handler
+        assert director.handlers.get(FakeRH.RH_KEY) is handler2
+        assert len(director.handlers) == 1
+
+        class AnotherFakeRH(FakeRH):
+            pass
+        director.add_handler(AnotherFakeRH(logger=FakeLogger()))
+        assert len(director.handlers) == 2
+        assert director.handlers.get(AnotherFakeRH.RH_KEY).RH_KEY == AnotherFakeRH.RH_KEY
+
+        director.handlers.pop(FakeRH.RH_KEY, None)
+        assert director.handlers.get(FakeRH.RH_KEY) is None
+        assert len(director.handlers) == 1
+
+        # RequestErrors should passthrough
+        with pytest.raises(SSLError):
+            director.send(Request('ssl://something'))
+
+    def test_send(self):
+        director = RequestDirector(logger=FakeLogger())
+        with pytest.raises(RequestError):
+            director.send(Request('any://'))
+        director.add_handler(FakeRH(logger=FakeLogger()))
+        assert isinstance(director.send(Request('http://')), FakeResponse)
+
+    def test_unsupported_handlers(self):
+        director = RequestDirector(logger=FakeLogger())
+        director.add_handler(FakeRH(logger=FakeLogger()))
+
+        class SupportedRH(RequestHandler):
+            _SUPPORTED_URL_SCHEMES = ['http']
+
+            def _send(self, request: Request):
+                return Response(fp=io.BytesIO(b'supported'), headers={}, url=request.url)
+
+        # This handler should by default take preference over FakeRH
+        director.add_handler(SupportedRH(logger=FakeLogger()))
+        assert director.send(Request('http://')).read() == b'supported'
+        assert director.send(Request('any://')).read() == b''
+
+        director.handlers.pop(FakeRH.RH_KEY)
+        with pytest.raises(NoSupportingHandlers):
+            director.send(Request('any://'))
+
+    def test_unexpected_error(self):
+        director = RequestDirector(logger=FakeLogger())
+
+        class UnexpectedRH(FakeRH):
+            def _send(self, request: Request):
+                raise TypeError('something')
+
+        director.add_handler(UnexpectedRH(logger=FakeLogger))
+        with pytest.raises(NoSupportingHandlers, match=r'1 unexpected error'):
+            director.send(Request('any://'))
+
+        director.handlers.clear()
+        assert len(director.handlers) == 0
+
+        # Should not be fatal
+        director.add_handler(FakeRH(logger=FakeLogger()))
+        director.add_handler(UnexpectedRH(logger=FakeLogger))
+        assert director.send(Request('any://'))
+
+
+# XXX: do we want to move this to test_YoutubeDL.py?
+class TestYoutubeDLNetworking:
+
+    @staticmethod
+    def build_handler(ydl, handler: RequestHandler = FakeRH):
+        return ydl.build_request_director([handler]).handlers.get(handler.RH_KEY)
+
+    def test_compat_opener(self):
+        with FakeYDL() as ydl:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', category=DeprecationWarning)
+                assert isinstance(ydl._opener, urllib.request.OpenerDirector)
+
+    @pytest.mark.parametrize('proxy,expected', [
+        ('http://127.0.0.1:8080', {'all': 'http://127.0.0.1:8080'}),
+        ('', {'all': '__noproxy__'}),
+        (None, {'http': 'http://127.0.0.1:8081', 'https': 'http://127.0.0.1:8081'})  # env, set https
+    ])
+    def test_proxy(self, proxy, expected):
+        old_http_proxy = os.environ.get('HTTP_PROXY')
+        try:
+            os.environ['HTTP_PROXY'] = 'http://127.0.0.1:8081'  # ensure that provided proxies override env
+            with FakeYDL({'proxy': proxy}) as ydl:
+                assert ydl.proxies == expected
+        finally:
+            if old_http_proxy:
+                os.environ['HTTP_PROXY'] = old_http_proxy
+
+    def test_compat_request(self):
+        with FakeRHYDL() as ydl:
+            assert ydl.urlopen('test://')
+            urllib_req = urllib.request.Request('http://foo.bar', data=b'test', method='PUT', headers={'X-Test': '1'})
+            urllib_req.add_unredirected_header('Cookie', 'bob=bob')
+            urllib_req.timeout = 2
+
+            req = ydl.urlopen(urllib_req).request
+            assert req.url == urllib_req.get_full_url()
+            assert req.data == urllib_req.data
+            assert req.method == urllib_req.get_method()
+            assert 'X-Test' in req.headers
+            assert 'Cookie' in req.headers
+            assert req.extensions.get('timeout') == 2
+
+            with pytest.raises(AssertionError):
+                ydl.urlopen(None)
+
+    def test_extract_basic_auth(self):
+        with FakeRHYDL() as ydl:
+            res = ydl.urlopen(Request('http://user:pass@foo.bar'))
+            assert res.request.headers['Authorization'] == 'Basic dXNlcjpwYXNz'
+
+    def test_sanitize_url(self):
+        with FakeRHYDL() as ydl:
+            res = ydl.urlopen(Request('httpss://foo.bar'))
+            assert res.request.url == 'https://foo.bar'
+
+    def test_file_urls_error(self):
+        # use urllib handler
+        with FakeYDL() as ydl:
+            with pytest.raises(RequestError, match=r'file:// URLs are disabled by default'):
+                ydl.urlopen('file://')
+
+    def test_legacy_server_connect_error(self):
+        with FakeRHYDL() as ydl:
+            for error in ('UNSAFE_LEGACY_RENEGOTIATION_DISABLED', 'SSLV3_ALERT_HANDSHAKE_FAILURE'):
+                with pytest.raises(RequestError, match=r'Try using --legacy-server-connect'):
+                    ydl.urlopen(f'ssl://{error}')
+
+            with pytest.raises(SSLError, match='testerror'):
+                ydl.urlopen('ssl://testerror')
+
+    @pytest.mark.parametrize('proxy_key,proxy_url,expected', [
+        ('http', '__noproxy__', None),
+        ('no', '127.0.0.1,foo.bar', '127.0.0.1,foo.bar'),
+        ('https', 'example.com', 'http://example.com'),
+        ('https', 'socks5://example.com', 'socks5h://example.com'),
+        ('http', 'socks://example.com', 'socks4://example.com'),
+        ('http', 'socks4://example.com', 'socks4://example.com'),
+    ])
+    def test_clean_proxy(self, proxy_key, proxy_url, expected):
+        # proxies should be cleaned in urlopen()
+        with FakeRHYDL() as ydl:
+            req = ydl.urlopen(Request('test://', proxies={proxy_key: proxy_url})).request
+            assert req.proxies[proxy_key] == expected
+
+        # and should also be cleaned when building the handler
+        env_key = f'{proxy_key.upper()}_PROXY'
+        old_env_proxy = os.environ.get(env_key)
+        try:
+            os.environ[env_key] = proxy_url  # ensure that provided proxies override env
+            with FakeYDL() as ydl:
+                rh = self.build_handler(ydl)
+                assert rh.proxies[proxy_key] == expected
+        finally:
+            if old_env_proxy:
+                os.environ[env_key] = old_env_proxy
+
+    def test_clean_proxy_header(self):
+        with FakeRHYDL() as ydl:
+            req = ydl.urlopen(Request('test://', headers={'ytdl-request-proxy': '//foo.bar'})).request
+            assert 'ytdl-request-proxy' not in req.headers
+            assert req.proxies == {'all': 'http://foo.bar'}
+
+        with FakeYDL({'http_headers': {'ytdl-request-proxy': '//foo.bar'}}) as ydl:
+            rh = self.build_handler(ydl)
+            assert 'ytdl-request-proxy' not in rh.headers
+            assert rh.proxies == {'all': 'http://foo.bar'}
+
+    def test_clean_header(self):
+        with FakeRHYDL() as ydl:
+            res = ydl.urlopen(Request('test://', headers={'Youtubedl-no-compression': True}))
+            assert 'Youtubedl-no-compression' not in res.request.headers
+            assert res.request.headers.get('Accept-Encoding') == 'identity'
+
+        with FakeYDL({'http_headers': {'Youtubedl-no-compression': True}}) as ydl:
+            rh = self.build_handler(ydl)
+            assert 'Youtubedl-no-compression' not in rh.headers
+            assert rh.headers.get('Accept-Encoding') == 'identity'
+
+    def test_build_handler_params(self):
+        with FakeYDL({
+            'http_headers': {'test': 'testtest'},
+            'socket_timeout': 2,
+            'proxy': 'http://127.0.0.1:8080',
+            'source_address': '127.0.0.45',
+            'debug_printtraffic': True,
+            'compat_opts': ['no-certifi'],
+            'nocheckcertificate': True,
+            'legacy_server_connect': True,
+        }) as ydl:
+            rh = self.build_handler(ydl)
+            assert rh.headers.get('test') == 'testtest'
+            assert 'Accept' in rh.headers  # ensure std_headers are still there
+            assert rh.timeout == 2
+            assert rh.proxies.get('all') == 'http://127.0.0.1:8080'
+            assert rh.source_address == '127.0.0.45'
+            assert rh.verbose is True
+            assert rh.prefer_system_certs is True
+            assert rh.verify is False
+            assert rh.legacy_ssl_support is True
+
+    @pytest.mark.parametrize('ydl_params', [
+        {'client_certificate': 'fakecert.crt'},
+        {'client_certificate': 'fakecert.crt', 'client_certificate_key': 'fakekey.key'},
+        {'client_certificate': 'fakecert.crt', 'client_certificate_key': 'fakekey.key', 'client_certificate_password': 'foobar'},
+        {'client_certificate_key': 'fakekey.key', 'client_certificate_password': 'foobar'},
+    ])
+    def test_client_certificate(self, ydl_params):
+        with FakeYDL(ydl_params) as ydl:
+            rh = self.build_handler(ydl)
+            assert rh._client_cert == ydl_params  # XXX: Too bound to implementation
+
+    def test_urllib_file_urls(self):
+        with FakeYDL({'enable_file_urls': False}) as ydl:
+            rh = self.build_handler(ydl, UrllibRH)
+            assert rh.enable_file_urls is False
+
+        with FakeYDL({'enable_file_urls': True}) as ydl:
+            rh = self.build_handler(ydl, UrllibRH)
+            assert rh.enable_file_urls is True
+
+
+class TestRequest:
+
+    def test_query(self):
+        req = Request('http://example.com?q=something', query={'v': 'xyz'})
+        assert req.url == 'http://example.com?q=something&v=xyz'
+
+        req.update(query={'v': '123'})
+        assert req.url == 'http://example.com?q=something&v=123'
+        req.update(url='http://example.com', query={'v': 'xyz'})
+        assert req.url == 'http://example.com?v=xyz'
+
+    def test_method(self):
+        req = Request('http://example.com')
+        assert req.method == 'GET'
+        req.data = b'test'
+        assert req.method == 'POST'
+        req.data = None
+        assert req.method == 'GET'
+        req.data = b'test2'
+        req.method = 'PUT'
+        assert req.method == 'PUT'
+        req.data = None
+        assert req.method == 'PUT'
+        with pytest.raises(TypeError):
+            req.method = 1
+
+    def test_request_helpers(self):
+        assert HEADRequest('http://example.com').method == 'HEAD'
+        assert PUTRequest('http://example.com').method == 'PUT'
+
+    def test_headers(self):
+        req = Request('http://example.com', headers={'tesT': 'test'})
+        assert req.headers == HTTPHeaderDict({'test': 'test'})
+        req.update(headers={'teSt2': 'test2'})
+        assert req.headers == HTTPHeaderDict({'test': 'test', 'test2': 'test2'})
+
+        req.headers = new_headers = HTTPHeaderDict({'test': 'test'})
+        assert req.headers == HTTPHeaderDict({'test': 'test'})
+        assert req.headers is new_headers
+
+        # test converts dict to case insensitive dict
+        req.headers = new_headers = {'test2': 'test2'}
+        assert isinstance(req.headers, HTTPHeaderDict)
+        assert req.headers is not new_headers
+
+        with pytest.raises(TypeError):
+            req.headers = None
+
+    def test_data_type(self):
+        req = Request('http://example.com')
+        assert req.data is None
+        # test bytes is allowed
+        req.data = b'test'
+        assert req.data == b'test'
+        # test iterable of bytes is allowed
+        i = [b'test', b'test2']
+        req.data = i
+        assert req.data == i
+
+        # test file-like object is allowed
+        f = io.BytesIO(b'test')
+        req.data = f
+        assert req.data == f
+
+        # common mistake: test str not allowed
+        with pytest.raises(TypeError):
+            req.data = 'test'
+        assert req.data != 'test'
+
+        # common mistake: test dict is not allowed
+        with pytest.raises(TypeError):
+            req.data = {'test': 'test'}
+        assert req.data != {'test': 'test'}
+
+    def test_content_length_header(self):
+        req = Request('http://example.com', headers={'Content-Length': '0'}, data=b'')
+        assert req.headers.get('Content-Length') == '0'
+
+        req.data = b'test'
+        assert 'Content-Length' not in req.headers
+
+        req = Request('http://example.com', headers={'Content-Length': '10'})
+        assert 'Content-Length' not in req.headers
+
+    def test_content_type_header(self):
+        req = Request('http://example.com', headers={'Content-Type': 'test'}, data=b'test')
+        assert req.headers.get('Content-Type') == 'test'
+        req.data = b'test2'
+        assert req.headers.get('Content-Type') == 'test'
+        req.data = None
+        assert 'Content-Type' not in req.headers
+        req.data = b'test3'
+        assert req.headers.get('Content-Type') == 'application/x-www-form-urlencoded'
+
+    def test_proxies(self):
+        req = Request(url='http://example.com', proxies={'http': 'http://127.0.0.1:8080'})
+        assert req.proxies == {'http': 'http://127.0.0.1:8080'}
+
+    def test_extensions(self):
+        req = Request(url='http://example.com', extensions={'timeout': 2})
+        assert req.extensions == {'timeout': 2}
+
+    def test_copy(self):
+        req = Request(
+            url='http://example.com',
+            extensions={'cookiejar': CookieJar()},
+            headers={'Accept-Encoding': 'br'},
+            proxies={'http': 'http://127.0.0.1'},
+            data=[b'123']
+        )
+        req_copy = req.copy()
+        assert req_copy is not req
+        assert req_copy.url == req.url
+        assert req_copy.headers == req.headers
+        assert req_copy.headers is not req.headers
+        assert req_copy.proxies == req.proxies
+        assert req_copy.proxies is not req.proxies
+
+        # Data is not able to be copied
+        assert req_copy.data == req.data
+        assert req_copy.data is req.data
+
+        # Shallow copy extensions
+        assert req_copy.extensions is not req.extensions
+        assert req_copy.extensions['cookiejar'] == req.extensions['cookiejar']
+
+        # Subclasses are copied by default
+        class AnotherRequest(Request):
+            pass
+
+        req = AnotherRequest(url='http://127.0.0.1')
+        assert isinstance(req.copy(), AnotherRequest)
+
+    def test_url(self):
+        req = Request(url='https://фtest.example.com/ some spaceв?ä=c',)
+        assert req.url == 'https://xn--test-z6d.example.com/%20some%20space%D0%B2?%C3%A4=c'
+
+        assert Request(url='//example.com').url == 'http://example.com'
+
+        with pytest.raises(TypeError):
+            Request(url='https://').url = None
+
+
+class TestResponse:
+
+    @pytest.mark.parametrize('reason,status,expected', [
+        ('custom', 200, 'custom'),
+        (None, 404, 'Not Found'),  # fallback status
+        ('', 403, 'Forbidden'),
+        (None, 999, None)
+    ])
+    def test_reason(self, reason, status, expected):
+        res = Response(io.BytesIO(b''), url='test://', headers={}, status=status, reason=reason)
+        assert res.reason == expected
+
+    def test_headers(self):
+        headers = Message()
+        headers.add_header('Test', 'test')
+        headers.add_header('Test', 'test2')
+        headers.add_header('content-encoding', 'br')
+        res = Response(io.BytesIO(b''), headers=headers, url='test://')
+        assert res.headers.get_all('test') == ['test', 'test2']
+        assert 'Content-Encoding' in res.headers
+
+    def test_get_header(self):
+        headers = Message()
+        headers.add_header('Set-Cookie', 'cookie1')
+        headers.add_header('Set-cookie', 'cookie2')
+        headers.add_header('Test', 'test')
+        headers.add_header('Test', 'test2')
+        res = Response(io.BytesIO(b''), headers=headers, url='test://')
+        assert res.get_header('test') == 'test, test2'
+        assert res.get_header('set-Cookie') == 'cookie1'
+        assert res.get_header('notexist', 'default') == 'default'
+
+    def test_compat(self):
+        res = Response(io.BytesIO(b''), url='test://', status=404, headers={'test': 'test'})
+        assert res.code == res.getcode() == res.status
+        assert res.geturl() == res.url
+        assert res.info() is res.headers
+        assert res.getheader('test') == res.get_header('test')
diff --git a/test/test_networking_utils.py b/test/test_networking_utils.py
new file mode 100644
index 000000000..f9f876af3
--- /dev/null
+++ b/test/test_networking_utils.py
@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+
+# Allow direct execution
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import io
+import platform
+import random
+import ssl
+import urllib.error
+
+from yt_dlp.cookies import YoutubeDLCookieJar
+from yt_dlp.dependencies import certifi
+from yt_dlp.networking import Response
+from yt_dlp.networking._helper import (
+    InstanceStoreMixin,
+    add_accept_encoding_header,
+    get_redirect_method,
+    make_socks_proxy_opts,
+    select_proxy,
+    ssl_load_certs,
+)
+from yt_dlp.networking.exceptions import (
+    HTTPError,
+    IncompleteRead,
+    _CompatHTTPError,
+)
+from yt_dlp.socks import ProxyType
+from yt_dlp.utils.networking import HTTPHeaderDict
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class TestNetworkingUtils:
+
+    def test_select_proxy(self):
+        proxies = {
+            'all': 'socks5://example.com',
+            'http': 'http://example.com:1080',
+            'no': 'bypass.example.com,yt-dl.org'
+        }
+
+        assert select_proxy('https://example.com', proxies) == proxies['all']
+        assert select_proxy('http://example.com', proxies) == proxies['http']
+        assert select_proxy('http://bypass.example.com', proxies) is None
+        assert select_proxy('https://yt-dl.org', proxies) is None
+
+    @pytest.mark.parametrize('socks_proxy,expected', [
+        ('socks5h://example.com', {
+            'proxytype': ProxyType.SOCKS5,
+            'addr': 'example.com',
+            'port': 1080,
+            'rdns': True,
+            'username': None,
+            'password': None
+        }),
+        ('socks5://user:@example.com:5555', {
+            'proxytype': ProxyType.SOCKS5,
+            'addr': 'example.com',
+            'port': 5555,
+            'rdns': False,
+            'username': 'user',
+            'password': ''
+        }),
+        ('socks4://u%40ser:pa%20ss@127.0.0.1:1080', {
+            'proxytype': ProxyType.SOCKS4,
+            'addr': '127.0.0.1',
+            'port': 1080,
+            'rdns': False,
+            'username': 'u@ser',
+            'password': 'pa ss'
+        }),
+        ('socks4a://:pa%20ss@127.0.0.1', {
+            'proxytype': ProxyType.SOCKS4A,
+            'addr': '127.0.0.1',
+            'port': 1080,
+            'rdns': True,
+            'username': '',
+            'password': 'pa ss'
+        })
+    ])
+    def test_make_socks_proxy_opts(self, socks_proxy, expected):
+        assert make_socks_proxy_opts(socks_proxy) == expected
+
+    def test_make_socks_proxy_unknown(self):
+        with pytest.raises(ValueError, match='Unknown SOCKS proxy version: socks'):
+            make_socks_proxy_opts('socks://127.0.0.1')
+
+    @pytest.mark.skipif(not certifi, reason='certifi is not installed')
+    def test_load_certifi(self):
+        context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        context2 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        ssl_load_certs(context, use_certifi=True)
+        context2.load_verify_locations(cafile=certifi.where())
+        assert context.get_ca_certs() == context2.get_ca_certs()
+
+        # Test load normal certs
+        # XXX: could there be a case where system certs are the same as certifi?
+        context3 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        ssl_load_certs(context3, use_certifi=False)
+        assert context3.get_ca_certs() != context.get_ca_certs()
+
+    @pytest.mark.parametrize('method,status,expected', [
+        ('GET', 303, 'GET'),
+        ('HEAD', 303, 'HEAD'),
+        ('PUT', 303, 'GET'),
+        ('POST', 301, 'GET'),
+        ('HEAD', 301, 'HEAD'),
+        ('POST', 302, 'GET'),
+        ('HEAD', 302, 'HEAD'),
+        ('PUT', 302, 'PUT'),
+        ('POST', 308, 'POST'),
+        ('POST', 307, 'POST'),
+        ('HEAD', 308, 'HEAD'),
+        ('HEAD', 307, 'HEAD'),
+    ])
+    def test_get_redirect_method(self, method, status, expected):
+        assert get_redirect_method(method, status) == expected
+
+    @pytest.mark.parametrize('headers,supported_encodings,expected', [
+        ({'Accept-Encoding': 'br'}, ['gzip', 'br'], {'Accept-Encoding': 'br'}),
+        ({}, ['gzip', 'br'], {'Accept-Encoding': 'gzip, br'}),
+        ({'Content-type': 'application/json'}, [], {'Content-type': 'application/json', 'Accept-Encoding': 'identity'}),
+    ])
+    def test_add_accept_encoding_header(self, headers, supported_encodings, expected):
+        headers = HTTPHeaderDict(headers)
+        add_accept_encoding_header(headers, supported_encodings)
+        assert headers == HTTPHeaderDict(expected)
+
+
+class TestInstanceStoreMixin:
+
+    class FakeInstanceStoreMixin(InstanceStoreMixin):
+        def _create_instance(self, **kwargs):
+            return random.randint(0, 1000000)
+
+        def _close_instance(self, instance):
+            pass
+
+    def test_mixin(self):
+        mixin = self.FakeInstanceStoreMixin()
+        assert mixin._get_instance(d={'a': 1, 'b': 2, 'c': {'d', 4}}) == mixin._get_instance(d={'a': 1, 'b': 2, 'c': {'d', 4}})
+
+        assert mixin._get_instance(d={'a': 1, 'b': 2, 'c': {'e', 4}}) != mixin._get_instance(d={'a': 1, 'b': 2, 'c': {'d', 4}})
+
+        assert mixin._get_instance(d={'a': 1, 'b': 2, 'c': {'d', 4}} != mixin._get_instance(d={'a': 1, 'b': 2, 'g': {'d', 4}}))
+
+        assert mixin._get_instance(d={'a': 1}, e=[1, 2, 3]) == mixin._get_instance(d={'a': 1}, e=[1, 2, 3])
+
+        assert mixin._get_instance(d={'a': 1}, e=[1, 2, 3]) != mixin._get_instance(d={'a': 1}, e=[1, 2, 3, 4])
+
+        cookiejar = YoutubeDLCookieJar()
+        assert mixin._get_instance(b=[1, 2], c=cookiejar) == mixin._get_instance(b=[1, 2], c=cookiejar)
+
+        assert mixin._get_instance(b=[1, 2], c=cookiejar) != mixin._get_instance(b=[1, 2], c=YoutubeDLCookieJar())
+
+        # Different order
+        assert mixin._get_instance(c=cookiejar, b=[1, 2]) == mixin._get_instance(b=[1, 2], c=cookiejar)
+
+        m = mixin._get_instance(t=1234)
+        assert mixin._get_instance(t=1234) == m
+        mixin._clear_instances()
+        assert mixin._get_instance(t=1234) != m
+
+
+class TestNetworkingExceptions:
+
+    @staticmethod
+    def create_response(status):
+        return Response(fp=io.BytesIO(b'test'), url='http://example.com', headers={'tesT': 'test'}, status=status)
+
+    @pytest.mark.parametrize('http_error_class', [HTTPError, lambda r: _CompatHTTPError(HTTPError(r))])
+    def test_http_error(self, http_error_class):
+
+        response = self.create_response(403)
+        error = http_error_class(response)
+
+        assert error.status == 403
+        assert str(error) == error.msg == 'HTTP Error 403: Forbidden'
+        assert error.reason == response.reason
+        assert error.response is response
+
+        data = error.response.read()
+        assert data == b'test'
+        assert repr(error) == '<HTTPError 403: Forbidden>'
+
+    @pytest.mark.parametrize('http_error_class', [HTTPError, lambda *args, **kwargs: _CompatHTTPError(HTTPError(*args, **kwargs))])
+    def test_redirect_http_error(self, http_error_class):
+        response = self.create_response(301)
+        error = http_error_class(response, redirect_loop=True)
+        assert str(error) == error.msg == 'HTTP Error 301: Moved Permanently (redirect loop detected)'
+        assert error.reason == 'Moved Permanently'
+
+    def test_compat_http_error(self):
+        response = self.create_response(403)
+        error = _CompatHTTPError(HTTPError(response))
+        assert isinstance(error, HTTPError)
+        assert isinstance(error, urllib.error.HTTPError)
+
+        assert error.code == 403
+        assert error.getcode() == 403
+        assert error.hdrs is error.response.headers
+        assert error.info() is error.response.headers
+        assert error.headers is error.response.headers
+        assert error.filename == error.response.url
+        assert error.url == error.response.url
+        assert error.geturl() == error.response.url
+
+        # Passthrough file operations
+        assert error.read() == b'test'
+        assert not error.closed
+        # Technically Response operations are also passed through, which should not be used.
+        assert error.get_header('test') == 'test'
+
+    @pytest.mark.skipif(
+        platform.python_implementation() == 'PyPy', reason='garbage collector works differently in pypy')
+    def test_compat_http_error_autoclose(self):
+        # Compat HTTPError should not autoclose response
+        response = self.create_response(403)
+        _CompatHTTPError(HTTPError(response))
+        assert not response.closed
+
+    def test_incomplete_read_error(self):
+        error = IncompleteRead(b'test', 3, cause='test')
+        assert isinstance(error, IncompleteRead)
+        assert repr(error) == '<IncompleteRead: 4 bytes read, 3 more expected>'
+        assert str(error) == error.msg == '4 bytes read, 3 more expected'
+        assert error.partial == b'test'
+        assert error.expected == 3
+        assert error.cause == 'test'
+
+        error = IncompleteRead(b'aaa')
+        assert repr(error) == '<IncompleteRead: 3 bytes read>'
+        assert str(error) == '3 bytes read'
diff --git a/test/test_utils.py b/test/test_utils.py
index 862c7d0f7..768edfd0c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -51,6 +51,7 @@
     escape_url,
     expand_path,
     extract_attributes,
+    extract_basic_auth,
     find_xpath_attr,
     fix_xml_ampersands,
     float_or_none,
@@ -103,7 +104,6 @@
     sanitize_filename,
     sanitize_path,
     sanitize_url,
-    sanitized_Request,
     shell_quote,
     smuggle_url,
     str_or_none,
@@ -132,6 +132,7 @@
     xpath_text,
     xpath_with_ns,
 )
+from yt_dlp.utils.networking import HTTPHeaderDict
 
 
 class TestUtil(unittest.TestCase):
@@ -2315,14 +2316,43 @@ def test_traverse_obj(self):
         self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 'group')), ['0123', '3'],
                          msg='function on a `re.Match` should give group name as well')
 
+    def test_http_header_dict(self):
+        headers = HTTPHeaderDict()
+        headers['ytdl-test'] = 1
+        self.assertEqual(list(headers.items()), [('Ytdl-Test', '1')])
+        headers['Ytdl-test'] = '2'
+        self.assertEqual(list(headers.items()), [('Ytdl-Test', '2')])
+        self.assertTrue('ytDl-Test' in headers)
+        self.assertEqual(str(headers), str(dict(headers)))
+        self.assertEqual(repr(headers), str(dict(headers)))
+
+        headers.update({'X-dlp': 'data'})
+        self.assertEqual(set(headers.items()), {('Ytdl-Test', '2'), ('X-Dlp', 'data')})
+        self.assertEqual(dict(headers), {'Ytdl-Test': '2', 'X-Dlp': 'data'})
+        self.assertEqual(len(headers), 2)
+        self.assertEqual(headers.copy(), headers)
+        headers2 = HTTPHeaderDict({'X-dlp': 'data3'}, **headers, **{'X-dlp': 'data2'})
+        self.assertEqual(set(headers2.items()), {('Ytdl-Test', '2'), ('X-Dlp', 'data2')})
+        self.assertEqual(len(headers2), 2)
+        headers2.clear()
+        self.assertEqual(len(headers2), 0)
+
+        # ensure we prefer latter headers
+        headers3 = HTTPHeaderDict({'Ytdl-TeSt': 1}, {'Ytdl-test': 2})
+        self.assertEqual(set(headers3.items()), {('Ytdl-Test', '2')})
+        del headers3['ytdl-tesT']
+        self.assertEqual(dict(headers3), {})
+
+        headers4 = HTTPHeaderDict({'ytdl-test': 'data;'})
+        self.assertEqual(set(headers4.items()), {('Ytdl-Test', 'data;')})
+
     def test_extract_basic_auth(self):
-        auth_header = lambda url: sanitized_Request(url).get_header('Authorization')
-        self.assertFalse(auth_header('http://foo.bar'))
-        self.assertFalse(auth_header('http://:foo.bar'))
-        self.assertEqual(auth_header('http://@foo.bar'), 'Basic Og==')
-        self.assertEqual(auth_header('http://:pass@foo.bar'), 'Basic OnBhc3M=')
-        self.assertEqual(auth_header('http://user:@foo.bar'), 'Basic dXNlcjo=')
-        self.assertEqual(auth_header('http://user:pass@foo.bar'), 'Basic dXNlcjpwYXNz')
+        assert extract_basic_auth('http://:foo.bar') == ('http://:foo.bar', None)
+        assert extract_basic_auth('http://foo.bar') == ('http://foo.bar', None)
+        assert extract_basic_auth('http://@foo.bar') == ('http://foo.bar', 'Basic Og==')
+        assert extract_basic_auth('http://:pass@foo.bar') == ('http://foo.bar', 'Basic OnBhc3M=')
+        assert extract_basic_auth('http://user:@foo.bar') == ('http://foo.bar', 'Basic dXNlcjo=')
+        assert extract_basic_auth('http://user:pass@foo.bar') == ('http://foo.bar', 'Basic dXNlcjpwYXNz')
 
 
 if __name__ == '__main__':
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 138646ebf..29a18aef0 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -4,7 +4,6 @@
 import datetime
 import errno
 import fileinput
-import functools
 import http.cookiejar
 import io
 import itertools
@@ -25,8 +24,8 @@
 import unicodedata
 
 from .cache import Cache
-from .compat import urllib  # isort: split
-from .compat import compat_os_name, compat_shlex_quote
+from .compat import functools, urllib  # isort: split
+from .compat import compat_os_name, compat_shlex_quote, urllib_req_to_req
 from .cookies import LenientSimpleCookie, load_cookies
 from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name
 from .downloader.rtmp import rtmpdump_version
@@ -34,6 +33,15 @@
 from .extractor.common import UnsupportedURLIE
 from .extractor.openload import PhantomJSwrapper
 from .minicurses import format_text
+from .networking import Request, RequestDirector
+from .networking.common import _REQUEST_HANDLERS
+from .networking.exceptions import (
+    HTTPError,
+    NoSupportingHandlers,
+    RequestError,
+    SSLError,
+    _CompatHTTPError,
+)
 from .plugins import directories as plugin_directories
 from .postprocessor import _PLUGIN_CLASSES as plugin_pps
 from .postprocessor import (
@@ -78,7 +86,6 @@
     MaxDownloadsReached,
     Namespace,
     PagedList,
-    PerRequestProxyHandler,
     PlaylistEntries,
     Popen,
     PostProcessingError,
@@ -87,9 +94,6 @@
     SameFileError,
     UnavailableVideoError,
     UserNotLive,
-    YoutubeDLCookieProcessor,
-    YoutubeDLHandler,
-    YoutubeDLRedirectHandler,
     age_restricted,
     args_to_str,
     bug_reports_message,
@@ -102,6 +106,7 @@
     error_to_compat_str,
     escapeHTML,
     expand_path,
+    extract_basic_auth,
     filter_dict,
     float_or_none,
     format_bytes,
@@ -117,8 +122,6 @@
     locked_file,
     make_archive_id,
     make_dir,
-    make_HTTPS_handler,
-    merge_headers,
     network_exceptions,
     number_of_digits,
     orderedSet,
@@ -132,7 +135,6 @@
     sanitize_filename,
     sanitize_path,
     sanitize_url,
-    sanitized_Request,
     std_headers,
     str_or_none,
     strftime_or_none,
@@ -151,7 +153,12 @@
     write_json_file,
     write_string,
 )
-from .utils.networking import clean_headers
+from .utils._utils import _YDLLogger
+from .utils.networking import (
+    HTTPHeaderDict,
+    clean_headers,
+    clean_proxies,
+)
 from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
 
 if compat_os_name == 'nt':
@@ -673,7 +680,9 @@ def process_color_policy(stream):
                     raise
 
         self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
-        self.params['http_headers'] = merge_headers(std_headers, self.params.get('http_headers', {}))
+        self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers'))
+        self._request_director = self.build_request_director(
+            sorted(_REQUEST_HANDLERS.values(), key=lambda rh: rh.RH_NAME.lower()))
         if auto_init and auto_init != 'no_verbose_header':
             self.print_debug_header()
 
@@ -763,8 +772,6 @@ def check_deprecated(param, option, suggestion):
                 get_postprocessor(pp_def.pop('key'))(self, **pp_def),
                 when=when)
 
-        self._setup_opener()
-
         def preload_download_archive(fn):
             """Preload the archive, if any is specified"""
             archive = set()
@@ -946,7 +953,11 @@ def save_cookies(self):
 
     def __exit__(self, *args):
         self.restore_console_title()
+        self.close()
+
+    def close(self):
         self.save_cookies()
+        self._request_director.close()
 
     def trouble(self, message=None, tb=None, is_error=True):
         """Determine action to take when a download problem appears.
@@ -2468,7 +2479,7 @@ def restore_last_token(self):
         return _build_selector_function(parsed_selector)
 
     def _calc_headers(self, info_dict):
-        res = merge_headers(self.params['http_headers'], info_dict.get('http_headers') or {})
+        res = HTTPHeaderDict(self.params['http_headers'], info_dict.get('http_headers'))
         clean_headers(res)
         cookies = self.cookiejar.get_cookies_for_url(info_dict['url'])
         if cookies:
@@ -3943,13 +3954,8 @@ def get_encoding(stream):
             join_nonempty(*get_package_info(m)) for m in available_dependencies.values()
         })) or 'none'))
 
-        self._setup_opener()
-        proxy_map = {}
-        for handler in self._opener.handlers:
-            if hasattr(handler, 'proxies'):
-                proxy_map.update(handler.proxies)
-        write_debug(f'Proxy map: {proxy_map}')
-
+        write_debug(f'Proxy map: {self.proxies}')
+        # write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers)}')
         for plugin_type, plugins in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}.items():
             display_list = ['%s%s' % (
                 klass.__name__, '' if klass.__name__ == name else f' as {name}')
@@ -3977,53 +3983,21 @@ def get_encoding(stream):
                     'See https://yt-dl.org/update if you need help updating.' %
                     latest_version)
 
-    def _setup_opener(self):
-        if hasattr(self, '_opener'):
-            return
-        timeout_val = self.params.get('socket_timeout')
-        self._socket_timeout = 20 if timeout_val is None else float(timeout_val)
+    @functools.cached_property
+    def proxies(self):
+        """Global proxy configuration"""
         opts_proxy = self.params.get('proxy')
-
-        cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
         if opts_proxy is not None:
             if opts_proxy == '':
-                proxies = {}
-            else:
-                proxies = {'http': opts_proxy, 'https': opts_proxy}
+                opts_proxy = '__noproxy__'
+            proxies = {'all': opts_proxy}
         else:
             proxies = urllib.request.getproxies()
-            # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
+            # compat. Set HTTPS_PROXY to __noproxy__ to revert
             if 'http' in proxies and 'https' not in proxies:
                 proxies['https'] = proxies['http']
-        proxy_handler = PerRequestProxyHandler(proxies)
 
-        debuglevel = 1 if self.params.get('debug_printtraffic') else 0
-        https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
-        ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
-        redirect_handler = YoutubeDLRedirectHandler()
-        data_handler = urllib.request.DataHandler()
-
-        # When passing our own FileHandler instance, build_opener won't add the
-        # default FileHandler and allows us to disable the file protocol, which
-        # can be used for malicious purposes (see
-        # https://github.com/ytdl-org/youtube-dl/issues/8227)
-        file_handler = urllib.request.FileHandler()
-
-        if not self.params.get('enable_file_urls'):
-            def file_open(*args, **kwargs):
-                raise urllib.error.URLError(
-                    'file:// URLs are explicitly disabled in yt-dlp for security reasons. '
-                    'Use --enable-file-urls to enable at your own risk.')
-            file_handler.file_open = file_open
-
-        opener = urllib.request.build_opener(
-            proxy_handler, https_handler, cookie_processor, ydlh, redirect_handler, data_handler, file_handler)
-
-        # Delete the default user-agent header, which would otherwise apply in
-        # cases where our custom HTTP handler doesn't come into play
-        # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
-        opener.addheaders = []
-        self._opener = opener
+        return proxies
 
     @functools.cached_property
     def cookiejar(self):
@@ -4031,11 +4005,84 @@ def cookiejar(self):
         return load_cookies(
             self.params.get('cookiefile'), self.params.get('cookiesfrombrowser'), self)
 
+    @property
+    def _opener(self):
+        """
+        Get a urllib OpenerDirector from the Urllib handler (deprecated).
+        """
+        self.deprecation_warning('YoutubeDL._opener() is deprecated, use YoutubeDL.urlopen()')
+        handler = self._request_director.handlers['Urllib']
+        return handler._get_instance(cookiejar=self.cookiejar, proxies=self.proxies)
+
     def urlopen(self, req):
         """ Start an HTTP download """
         if isinstance(req, str):
-            req = sanitized_Request(req)
-        return self._opener.open(req, timeout=self._socket_timeout)
+            req = Request(req)
+        elif isinstance(req, urllib.request.Request):
+            req = urllib_req_to_req(req)
+        assert isinstance(req, Request)
+
+        # compat: Assume user:pass url params are basic auth
+        url, basic_auth_header = extract_basic_auth(req.url)
+        if basic_auth_header:
+            req.headers['Authorization'] = basic_auth_header
+        req.url = sanitize_url(url)
+
+        clean_proxies(proxies=req.proxies, headers=req.headers)
+        clean_headers(req.headers)
+
+        try:
+            return self._request_director.send(req)
+        except NoSupportingHandlers as e:
+            for ue in e.unsupported_errors:
+                if not (ue.handler and ue.msg):
+                    continue
+                if ue.handler.RH_KEY == 'Urllib' and 'unsupported url scheme: "file"' in ue.msg.lower():
+                    raise RequestError(
+                        'file:// URLs are disabled by default in yt-dlp for security reasons. '
+                        'Use --enable-file-urls to enable at your own risk.', cause=ue) from ue
+            raise
+        except SSLError as e:
+            if 'UNSAFE_LEGACY_RENEGOTIATION_DISABLED' in str(e):
+                raise RequestError('UNSAFE_LEGACY_RENEGOTIATION_DISABLED: Try using --legacy-server-connect', cause=e) from e
+            elif 'SSLV3_ALERT_HANDSHAKE_FAILURE' in str(e):
+                raise RequestError(
+                    'SSLV3_ALERT_HANDSHAKE_FAILURE: The server may not support the current cipher list. '
+                    'Try using --legacy-server-connect', cause=e) from e
+            raise
+        except HTTPError as e:  # TODO: Remove in a future release
+            raise _CompatHTTPError(e) from e
+
+    def build_request_director(self, handlers):
+        logger = _YDLLogger(self)
+        headers = self.params.get('http_headers').copy()
+        proxies = self.proxies.copy()
+        clean_headers(headers)
+        clean_proxies(proxies, headers)
+
+        director = RequestDirector(logger=logger, verbose=self.params.get('debug_printtraffic'))
+        for handler in handlers:
+            director.add_handler(handler(
+                logger=logger,
+                headers=headers,
+                cookiejar=self.cookiejar,
+                proxies=proxies,
+                prefer_system_certs='no-certifi' in self.params['compat_opts'],
+                verify=not self.params.get('nocheckcertificate'),
+                **traverse_obj(self.params, {
+                    'verbose': 'debug_printtraffic',
+                    'source_address': 'source_address',
+                    'timeout': 'socket_timeout',
+                    'legacy_ssl_support': 'legacy_server_connect',
+                    'enable_file_urls': 'enable_file_urls',
+                    'client_cert': {
+                        'client_certificate': 'client_certificate',
+                        'client_certificate_key': 'client_certificate_key',
+                        'client_certificate_password': 'client_certificate_password',
+                    },
+                }),
+            ))
+        return director
 
     def encode(self, s):
         if isinstance(s, bytes):
@@ -4188,7 +4235,7 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
             else:
                 self.to_screen(f'[info] Downloading {thumb_display_id} ...')
                 try:
-                    uf = self.urlopen(sanitized_Request(t['url'], headers=t.get('http_headers', {})))
+                    uf = self.urlopen(Request(t['url'], headers=t.get('http_headers', {})))
                     self.to_screen(f'[info] Writing {thumb_display_id} to: {thumb_filename}')
                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
                         shutil.copyfileobj(uf, thumbf)
diff --git a/yt_dlp/compat/__init__.py b/yt_dlp/compat/__init__.py
index c6c02541c..a41a80ebb 100644
--- a/yt_dlp/compat/__init__.py
+++ b/yt_dlp/compat/__init__.py
@@ -70,3 +70,13 @@ def compat_expanduser(path):
         return userhome + path[i:]
 else:
     compat_expanduser = os.path.expanduser
+
+
+def urllib_req_to_req(urllib_request):
+    """Convert urllib Request to a networking Request"""
+    from ..networking import Request
+    from ..utils.networking import HTTPHeaderDict
+    return Request(
+        urllib_request.get_full_url(), data=urllib_request.data, method=urllib_request.get_method(),
+        headers=HTTPHeaderDict(urllib_request.headers, urllib_request.unredirected_hdrs),
+        extensions={'timeout': urllib_request.timeout} if hasattr(urllib_request, 'timeout') else None)
diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index 7c5daea85..45d094721 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -1,12 +1,10 @@
-import http.client
 import os
 import random
-import socket
-import ssl
 import time
 import urllib.error
 
 from .common import FileDownloader
+from ..networking.exceptions import CertificateVerifyError, TransportError
 from ..utils import (
     ContentTooShortError,
     RetryManager,
@@ -21,14 +19,6 @@
     write_xattr,
 )
 
-RESPONSE_READ_EXCEPTIONS = (
-    TimeoutError,
-    socket.timeout,  # compat: py < 3.10
-    ConnectionError,
-    ssl.SSLError,
-    http.client.HTTPException
-)
-
 
 class HttpFD(FileDownloader):
     def real_download(self, filename, info_dict):
@@ -196,13 +186,9 @@ def establish_connection():
                     # Unexpected HTTP error
                     raise
                 raise RetryDownload(err)
-            except urllib.error.URLError as err:
-                if isinstance(err.reason, ssl.CertificateError):
-                    raise
-                raise RetryDownload(err)
-            # In urllib.request.AbstractHTTPHandler, the response is partially read on request.
-            # Any errors that occur during this will not be wrapped by URLError
-            except RESPONSE_READ_EXCEPTIONS as err:
+            except CertificateVerifyError:
+                raise
+            except TransportError as err:
                 raise RetryDownload(err)
 
         def close_stream():
@@ -258,7 +244,7 @@ def retry(e):
                 try:
                     # Download and write
                     data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
-                except RESPONSE_READ_EXCEPTIONS as err:
+                except TransportError as err:
                     retry(err)
 
                 byte_counter += len(data_block)
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index fe08839aa..63156d3ac 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -17,16 +17,22 @@
 import sys
 import time
 import types
-import urllib.error
 import urllib.parse
 import urllib.request
 import xml.etree.ElementTree
 
 from ..compat import functools  # isort: split
-from ..compat import compat_etree_fromstring, compat_expanduser, compat_os_name
+from ..compat import (
+    compat_etree_fromstring,
+    compat_expanduser,
+    compat_os_name,
+    urllib_req_to_req,
+)
 from ..cookies import LenientSimpleCookie
 from ..downloader.f4m import get_base_url, remove_encrypted_media
 from ..downloader.hls import HlsFD
+from ..networking.common import HEADRequest, Request
+from ..networking.exceptions import network_exceptions
 from ..utils import (
     IDENTITY,
     JSON_LD_RE,
@@ -35,7 +41,6 @@
     FormatSorter,
     GeoRestrictedError,
     GeoUtils,
-    HEADRequest,
     LenientJSONDecoder,
     Popen,
     RegexNotFoundError,
@@ -61,7 +66,6 @@
     js_to_json,
     mimetype2ext,
     netrc_from_content,
-    network_exceptions,
     orderedSet,
     parse_bitrate,
     parse_codecs,
@@ -71,7 +75,6 @@
     parse_resolution,
     sanitize_filename,
     sanitize_url,
-    sanitized_Request,
     smuggle_url,
     str_or_none,
     str_to_int,
@@ -83,8 +86,6 @@
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
-    update_Request,
-    update_url_query,
     url_basename,
     url_or_none,
     urlhandle_detect_ext,
@@ -797,10 +798,12 @@ def __can_accept_status_code(err, expected_status):
 
     def _create_request(self, url_or_request, data=None, headers=None, query=None):
         if isinstance(url_or_request, urllib.request.Request):
-            return update_Request(url_or_request, data=data, headers=headers, query=query)
-        if query:
-            url_or_request = update_url_query(url_or_request, query)
-        return sanitized_Request(url_or_request, data, headers or {})
+            url_or_request = urllib_req_to_req(url_or_request)
+        elif not isinstance(url_or_request, Request):
+            url_or_request = Request(url_or_request)
+
+        url_or_request.update(data=data, headers=headers, query=query)
+        return url_or_request
 
     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers=None, query=None, expected_status=None):
         """
@@ -838,12 +841,7 @@ def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fa
         except network_exceptions as err:
             if isinstance(err, urllib.error.HTTPError):
                 if self.__can_accept_status_code(err, expected_status):
-                    # Retain reference to error to prevent file object from
-                    # being closed before it can be read. Works around the
-                    # effects of <https://bugs.python.org/issue15002>
-                    # introduced in Python 3.4.1.
-                    err.fp._error = err
-                    return err.fp
+                    return err.response
 
             if errnote is False:
                 return False
diff --git a/yt_dlp/networking/__init__.py b/yt_dlp/networking/__init__.py
index e69de29bb..5e8876484 100644
--- a/yt_dlp/networking/__init__.py
+++ b/yt_dlp/networking/__init__.py
@@ -0,0 +1,13 @@
+# flake8: noqa: 401
+from .common import (
+    HEADRequest,
+    PUTRequest,
+    Request,
+    RequestDirector,
+    RequestHandler,
+    Response,
+)
+
+# isort: split
+# TODO: all request handlers should be safely imported
+from . import _urllib
diff --git a/yt_dlp/networking/_helper.py b/yt_dlp/networking/_helper.py
index 367f3f444..a43c57bb4 100644
--- a/yt_dlp/networking/_helper.py
+++ b/yt_dlp/networking/_helper.py
@@ -1,13 +1,22 @@
 from __future__ import annotations
 
 import contextlib
+import functools
 import ssl
 import sys
+import typing
 import urllib.parse
+import urllib.request
 
+from .exceptions import RequestError, UnsupportedRequest
 from ..dependencies import certifi
 from ..socks import ProxyType
-from ..utils import YoutubeDLError
+from ..utils import format_field, traverse_obj
+
+if typing.TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from ..utils.networking import HTTPHeaderDict
 
 
 def ssl_load_certs(context: ssl.SSLContext, use_certifi=True):
@@ -23,11 +32,11 @@ def ssl_load_certs(context: ssl.SSLContext, use_certifi=True):
             # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
             if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
                 for storename in ('CA', 'ROOT'):
-                    _ssl_load_windows_store_certs(context, storename)
+                    ssl_load_windows_store_certs(context, storename)
             context.set_default_verify_paths()
 
 
-def _ssl_load_windows_store_certs(ssl_context, storename):
+def ssl_load_windows_store_certs(ssl_context, storename):
     # Code adapted from _load_windows_store_certs in https://github.com/python/cpython/blob/main/Lib/ssl.py
     try:
         certs = [cert for cert, encoding, trust in ssl.enum_certificates(storename)
@@ -44,10 +53,18 @@ def make_socks_proxy_opts(socks_proxy):
     url_components = urllib.parse.urlparse(socks_proxy)
     if url_components.scheme.lower() == 'socks5':
         socks_type = ProxyType.SOCKS5
-    elif url_components.scheme.lower() in ('socks', 'socks4'):
+        rdns = False
+    elif url_components.scheme.lower() == 'socks5h':
+        socks_type = ProxyType.SOCKS5
+        rdns = True
+    elif url_components.scheme.lower() == 'socks4':
         socks_type = ProxyType.SOCKS4
+        rdns = False
     elif url_components.scheme.lower() == 'socks4a':
         socks_type = ProxyType.SOCKS4A
+        rdns = True
+    else:
+        raise ValueError(f'Unknown SOCKS proxy version: {url_components.scheme.lower()}')
 
     def unquote_if_non_empty(s):
         if not s:
@@ -57,12 +74,25 @@ def unquote_if_non_empty(s):
         'proxytype': socks_type,
         'addr': url_components.hostname,
         'port': url_components.port or 1080,
-        'rdns': True,
+        'rdns': rdns,
         'username': unquote_if_non_empty(url_components.username),
         'password': unquote_if_non_empty(url_components.password),
     }
 
 
+def select_proxy(url, proxies):
+    """Unified proxy selector for all backends"""
+    url_components = urllib.parse.urlparse(url)
+    if 'no' in proxies:
+        hostport = url_components.hostname + format_field(url_components.port, None, ':%s')
+        if urllib.request.proxy_bypass_environment(hostport, {'no': proxies['no']}):
+            return
+        elif urllib.request.proxy_bypass(hostport):  # check system settings
+            return
+
+    return traverse_obj(proxies, url_components.scheme or 'http', 'all')
+
+
 def get_redirect_method(method, status):
     """Unified redirect method handling"""
 
@@ -126,14 +156,53 @@ def make_ssl_context(
                 client_certificate, keyfile=client_certificate_key,
                 password=client_certificate_password)
         except ssl.SSLError:
-            raise YoutubeDLError('Unable to load client certificate')
+            raise RequestError('Unable to load client certificate')
 
+        if getattr(context, 'post_handshake_auth', None) is not None:
+            context.post_handshake_auth = True
     return context
 
 
-def add_accept_encoding_header(headers, supported_encodings):
-    if supported_encodings and 'Accept-Encoding' not in headers:
-        headers['Accept-Encoding'] = ', '.join(supported_encodings)
+class InstanceStoreMixin:
+    def __init__(self, **kwargs):
+        self.__instances = []
+        super().__init__(**kwargs)  # So that both MRO works
 
-    elif 'Accept-Encoding' not in headers:
-        headers['Accept-Encoding'] = 'identity'
+    @staticmethod
+    def _create_instance(**kwargs):
+        raise NotImplementedError
+
+    def _get_instance(self, **kwargs):
+        for key, instance in self.__instances:
+            if key == kwargs:
+                return instance
+
+        instance = self._create_instance(**kwargs)
+        self.__instances.append((kwargs, instance))
+        return instance
+
+    def _close_instance(self, instance):
+        if callable(getattr(instance, 'close', None)):
+            instance.close()
+
+    def _clear_instances(self):
+        for _, instance in self.__instances:
+            self._close_instance(instance)
+        self.__instances.clear()
+
+
+def add_accept_encoding_header(headers: HTTPHeaderDict, supported_encodings: Iterable[str]):
+    if 'Accept-Encoding' not in headers:
+        headers['Accept-Encoding'] = ', '.join(supported_encodings) or 'identity'
+
+
+def wrap_request_errors(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        try:
+            return func(self, *args, **kwargs)
+        except UnsupportedRequest as e:
+            if e.handler is None:
+                e.handler = self
+            raise
+    return wrapper
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index 1f5871ae6..2c5f09872 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import functools
 import gzip
 import http.client
@@ -9,26 +11,48 @@
 import urllib.request
 import urllib.response
 import zlib
+from urllib.request import (
+    DataHandler,
+    FileHandler,
+    FTPHandler,
+    HTTPCookieProcessor,
+    HTTPDefaultErrorHandler,
+    HTTPErrorProcessor,
+    UnknownHandler,
+)
 
 from ._helper import (
+    InstanceStoreMixin,
     add_accept_encoding_header,
     get_redirect_method,
     make_socks_proxy_opts,
+    select_proxy,
+)
+from .common import Features, RequestHandler, Response, register
+from .exceptions import (
+    CertificateVerifyError,
+    HTTPError,
+    IncompleteRead,
+    ProxyError,
+    RequestError,
+    SSLError,
+    TransportError,
 )
 from ..dependencies import brotli
+from ..socks import ProxyError as SocksProxyError
 from ..socks import sockssocket
 from ..utils import escape_url, update_url_query
-from ..utils.networking import clean_headers, std_headers
 
 SUPPORTED_ENCODINGS = ['gzip', 'deflate']
+CONTENT_DECODE_ERRORS = [zlib.error, OSError]
 
 if brotli:
     SUPPORTED_ENCODINGS.append('br')
+    CONTENT_DECODE_ERRORS.append(brotli.error)
 
 
-def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
+def _create_http_connection(http_class, source_address, *args, **kwargs):
     hc = http_class(*args, **kwargs)
-    source_address = ydl_handler._params.get('source_address')
 
     if source_address is not None:
         # This is to workaround _create_connection() from socket where it will try all
@@ -73,7 +97,7 @@ def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_a
     return hc
 
 
-class HTTPHandler(urllib.request.HTTPHandler):
+class HTTPHandler(urllib.request.AbstractHTTPHandler):
     """Handler for HTTP requests and responses.
 
     This class, when installed with an OpenerDirector, automatically adds
@@ -88,21 +112,30 @@ class HTTPHandler(urllib.request.HTTPHandler):
     public domain.
     """
 
-    def __init__(self, params, *args, **kwargs):
-        urllib.request.HTTPHandler.__init__(self, *args, **kwargs)
-        self._params = params
+    def __init__(self, context=None, source_address=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._source_address = source_address
+        self._context = context
 
-    def http_open(self, req):
-        conn_class = http.client.HTTPConnection
-
-        socks_proxy = req.headers.get('Ytdl-socks-proxy')
+    @staticmethod
+    def _make_conn_class(base, req):
+        conn_class = base
+        socks_proxy = req.headers.pop('Ytdl-socks-proxy', None)
         if socks_proxy:
             conn_class = make_socks_conn_class(conn_class, socks_proxy)
-            del req.headers['Ytdl-socks-proxy']
+        return conn_class
 
+    def http_open(self, req):
+        conn_class = self._make_conn_class(http.client.HTTPConnection, req)
         return self.do_open(functools.partial(
-            _create_http_connection, self, conn_class, False),
-            req)
+            _create_http_connection, conn_class, self._source_address), req)
+
+    def https_open(self, req):
+        conn_class = self._make_conn_class(http.client.HTTPSConnection, req)
+        return self.do_open(
+            functools.partial(
+                _create_http_connection, conn_class, self._source_address),
+            req, context=self._context)
 
     @staticmethod
     def deflate(data):
@@ -152,14 +185,6 @@ def http_request(self, req):
         if url != url_escaped:
             req = update_Request(req, url=url_escaped)
 
-        for h, v in self._params.get('http_headers', std_headers).items():
-            # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
-            # The dict keys are capitalized because of this bug by urllib
-            if h.capitalize() not in req.headers:
-                req.add_header(h, v)
-
-        clean_headers(req.headers)
-        add_accept_encoding_header(req.headers, SUPPORTED_ENCODINGS)
         return super().do_request_(req)
 
     def http_response(self, req, resp):
@@ -207,16 +232,12 @@ class SocksConnection(base_class):
         def connect(self):
             self.sock = sockssocket()
             self.sock.setproxy(**proxy_args)
-            if isinstance(self.timeout, (int, float)):
+            if type(self.timeout) in (int, float):  # noqa: E721
                 self.sock.settimeout(self.timeout)
             self.sock.connect((self.host, self.port))
 
             if isinstance(self, http.client.HTTPSConnection):
-                if hasattr(self, '_context'):  # Python > 2.6
-                    self.sock = self._context.wrap_socket(
-                        self.sock, server_hostname=self.host)
-                else:
-                    self.sock = ssl.wrap_socket(self.sock)
+                self.sock = self._context.wrap_socket(self.sock, server_hostname=self.host)
 
     return SocksConnection
 
@@ -260,29 +281,25 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
             unverifiable=True, method=new_method, data=new_data)
 
 
-class ProxyHandler(urllib.request.ProxyHandler):
+class ProxyHandler(urllib.request.BaseHandler):
+    handler_order = 100
+
     def __init__(self, proxies=None):
+        self.proxies = proxies
         # Set default handlers
-        for type in ('http', 'https'):
-            setattr(self, '%s_open' % type,
-                    lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
-                        meth(r, proxy, type))
-        urllib.request.ProxyHandler.__init__(self, proxies)
+        for type in ('http', 'https', 'ftp'):
+            setattr(self, '%s_open' % type, lambda r, meth=self.proxy_open: meth(r))
 
-    def proxy_open(self, req, proxy, type):
-        req_proxy = req.headers.get('Ytdl-request-proxy')
-        if req_proxy is not None:
-            proxy = req_proxy
-            del req.headers['Ytdl-request-proxy']
-
-        if proxy == '__noproxy__':
-            return None  # No Proxy
-        if urllib.parse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
+    def proxy_open(self, req):
+        proxy = select_proxy(req.get_full_url(), self.proxies)
+        if proxy is None:
+            return
+        if urllib.parse.urlparse(proxy).scheme.lower() in ('socks4', 'socks4a', 'socks5', 'socks5h'):
             req.add_header('Ytdl-socks-proxy', proxy)
             # yt-dlp's http/https handlers do wrapping the socket with socks
             return None
         return urllib.request.ProxyHandler.proxy_open(
-            self, req, proxy, type)
+            self, req, proxy, None)
 
 
 class PUTRequest(urllib.request.Request):
@@ -313,3 +330,129 @@ def update_Request(req, url=None, data=None, headers=None, query=None):
     if hasattr(req, 'timeout'):
         new_req.timeout = req.timeout
     return new_req
+
+
+class UrllibResponseAdapter(Response):
+    """
+    HTTP Response adapter class for urllib addinfourl and http.client.HTTPResponse
+    """
+
+    def __init__(self, res: http.client.HTTPResponse | urllib.response.addinfourl):
+        # addinfourl: In Python 3.9+, .status was introduced and .getcode() was deprecated [1]
+        # HTTPResponse: .getcode() was deprecated, .status always existed [2]
+        # 1. https://docs.python.org/3/library/urllib.request.html#urllib.response.addinfourl.getcode
+        # 2. https://docs.python.org/3.10/library/http.client.html#http.client.HTTPResponse.status
+        super().__init__(
+            fp=res, headers=res.headers, url=res.url,
+            status=getattr(res, 'status', None) or res.getcode(), reason=getattr(res, 'reason', None))
+
+    def read(self, amt=None):
+        try:
+            return self.fp.read(amt)
+        except Exception as e:
+            handle_response_read_exceptions(e)
+            raise e
+
+
+def handle_sslerror(e: ssl.SSLError):
+    if not isinstance(e, ssl.SSLError):
+        return
+    if isinstance(e, ssl.SSLCertVerificationError):
+        raise CertificateVerifyError(cause=e) from e
+    raise SSLError(cause=e) from e
+
+
+def handle_response_read_exceptions(e):
+    if isinstance(e, http.client.IncompleteRead):
+        raise IncompleteRead(partial=e.partial, cause=e, expected=e.expected) from e
+    elif isinstance(e, ssl.SSLError):
+        handle_sslerror(e)
+    elif isinstance(e, (OSError, EOFError, http.client.HTTPException, *CONTENT_DECODE_ERRORS)):
+        # OSErrors raised here should mostly be network related
+        raise TransportError(cause=e) from e
+
+
+@register
+class UrllibRH(RequestHandler, InstanceStoreMixin):
+    _SUPPORTED_URL_SCHEMES = ('http', 'https', 'data', 'ftp')
+    _SUPPORTED_PROXY_SCHEMES = ('http', 'socks4', 'socks4a', 'socks5', 'socks5h')
+    _SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY)
+    RH_NAME = 'urllib'
+
+    def __init__(self, *, enable_file_urls: bool = False, **kwargs):
+        super().__init__(**kwargs)
+        self.enable_file_urls = enable_file_urls
+        if self.enable_file_urls:
+            self._SUPPORTED_URL_SCHEMES = (*self._SUPPORTED_URL_SCHEMES, 'file')
+
+    def _create_instance(self, proxies, cookiejar):
+        opener = urllib.request.OpenerDirector()
+        handlers = [
+            ProxyHandler(proxies),
+            HTTPHandler(
+                debuglevel=int(bool(self.verbose)),
+                context=self._make_sslcontext(),
+                source_address=self.source_address),
+            HTTPCookieProcessor(cookiejar),
+            DataHandler(),
+            UnknownHandler(),
+            HTTPDefaultErrorHandler(),
+            FTPHandler(),
+            HTTPErrorProcessor(),
+            RedirectHandler(),
+        ]
+
+        if self.enable_file_urls:
+            handlers.append(FileHandler())
+
+        for handler in handlers:
+            opener.add_handler(handler)
+
+        # Delete the default user-agent header, which would otherwise apply in
+        # cases where our custom HTTP handler doesn't come into play
+        # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
+        opener.addheaders = []
+        return opener
+
+    def _send(self, request):
+        headers = self._merge_headers(request.headers)
+        add_accept_encoding_header(headers, SUPPORTED_ENCODINGS)
+        urllib_req = urllib.request.Request(
+            url=request.url,
+            data=request.data,
+            headers=dict(headers),
+            method=request.method
+        )
+
+        opener = self._get_instance(
+            proxies=request.proxies or self.proxies,
+            cookiejar=request.extensions.get('cookiejar') or self.cookiejar
+        )
+        try:
+            res = opener.open(urllib_req, timeout=float(request.extensions.get('timeout') or self.timeout))
+        except urllib.error.HTTPError as e:
+            if isinstance(e.fp, (http.client.HTTPResponse, urllib.response.addinfourl)):
+                # Prevent file object from being closed when urllib.error.HTTPError is destroyed.
+                e._closer.file = None
+                raise HTTPError(UrllibResponseAdapter(e.fp), redirect_loop='redirect error' in str(e)) from e
+            raise  # unexpected
+        except urllib.error.URLError as e:
+            cause = e.reason  # NOTE: cause may be a string
+
+            # proxy errors
+            if 'tunnel connection failed' in str(cause).lower() or isinstance(cause, SocksProxyError):
+                raise ProxyError(cause=e) from e
+
+            handle_response_read_exceptions(cause)
+            raise TransportError(cause=e) from e
+        except (http.client.InvalidURL, ValueError) as e:
+            # Validation errors
+            # http.client.HTTPConnection raises ValueError in some validation cases
+            # such as if request method contains illegal control characters [1]
+            # 1. https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1256
+            raise RequestError(cause=e) from e
+        except Exception as e:
+            handle_response_read_exceptions(e)
+            raise  # unexpected
+
+        return UrllibResponseAdapter(res)
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
new file mode 100644
index 000000000..e4b362827
--- /dev/null
+++ b/yt_dlp/networking/common.py
@@ -0,0 +1,522 @@
+from __future__ import annotations
+
+import abc
+import copy
+import enum
+import functools
+import io
+import typing
+import urllib.parse
+import urllib.request
+import urllib.response
+from collections.abc import Iterable, Mapping
+from email.message import Message
+from http import HTTPStatus
+from http.cookiejar import CookieJar
+
+from ._helper import make_ssl_context, wrap_request_errors
+from .exceptions import (
+    NoSupportingHandlers,
+    RequestError,
+    TransportError,
+    UnsupportedRequest,
+)
+from ..utils import (
+    bug_reports_message,
+    classproperty,
+    error_to_str,
+    escape_url,
+    update_url_query,
+)
+from ..utils.networking import HTTPHeaderDict
+
+if typing.TYPE_CHECKING:
+    RequestData = bytes | Iterable[bytes] | typing.IO | None
+
+
+class RequestDirector:
+    """RequestDirector class
+
+    Helper class that, when given a request, forward it to a RequestHandler that supports it.
+
+    @param logger: Logger instance.
+    @param verbose: Print debug request information to stdout.
+    """
+
+    def __init__(self, logger, verbose=False):
+        self.handlers: dict[str, RequestHandler] = {}
+        self.logger = logger  # TODO(Grub4k): default logger
+        self.verbose = verbose
+
+    def close(self):
+        for handler in self.handlers.values():
+            handler.close()
+
+    def add_handler(self, handler: RequestHandler):
+        """Add a handler. If a handler of the same RH_KEY exists, it will overwrite it"""
+        assert isinstance(handler, RequestHandler), 'handler must be a RequestHandler'
+        self.handlers[handler.RH_KEY] = handler
+
+    def _print_verbose(self, msg):
+        if self.verbose:
+            self.logger.stdout(f'director: {msg}')
+
+    def send(self, request: Request) -> Response:
+        """
+        Passes a request onto a suitable RequestHandler
+        """
+        if not self.handlers:
+            raise RequestError('No request handlers configured')
+
+        assert isinstance(request, Request)
+
+        unexpected_errors = []
+        unsupported_errors = []
+        # TODO (future): add a per-request preference system
+        for handler in reversed(list(self.handlers.values())):
+            self._print_verbose(f'Checking if "{handler.RH_NAME}" supports this request.')
+            try:
+                handler.validate(request)
+            except UnsupportedRequest as e:
+                self._print_verbose(
+                    f'"{handler.RH_NAME}" cannot handle this request (reason: {error_to_str(e)})')
+                unsupported_errors.append(e)
+                continue
+
+            self._print_verbose(f'Sending request via "{handler.RH_NAME}"')
+            try:
+                response = handler.send(request)
+            except RequestError:
+                raise
+            except Exception as e:
+                self.logger.error(
+                    f'[{handler.RH_NAME}] Unexpected error: {error_to_str(e)}{bug_reports_message()}',
+                    is_error=False)
+                unexpected_errors.append(e)
+                continue
+
+            assert isinstance(response, Response)
+            return response
+
+        raise NoSupportingHandlers(unsupported_errors, unexpected_errors)
+
+
+_REQUEST_HANDLERS = {}
+
+
+def register(handler):
+    """Register a RequestHandler class"""
+    assert issubclass(handler, RequestHandler), f'{handler} must be a subclass of RequestHandler'
+    assert handler.RH_KEY not in _REQUEST_HANDLERS, f'RequestHandler {handler.RH_KEY} already registered'
+    _REQUEST_HANDLERS[handler.RH_KEY] = handler
+    return handler
+
+
+class Features(enum.Enum):
+    ALL_PROXY = enum.auto()
+    NO_PROXY = enum.auto()
+
+
+class RequestHandler(abc.ABC):
+
+    """Request Handler class
+
+    Request handlers are class that, given a Request,
+    process the request from start to finish and return a Response.
+
+    Concrete subclasses need to redefine the _send(request) method,
+    which handles the underlying request logic and returns a Response.
+
+    RH_NAME class variable may contain a display name for the RequestHandler.
+    By default, this is generated from the class name.
+
+    The concrete request handler MUST have "RH" as the suffix in the class name.
+
+    All exceptions raised by a RequestHandler should be an instance of RequestError.
+    Any other exception raised will be treated as a handler issue.
+
+    If a Request is not supported by the handler, an UnsupportedRequest
+    should be raised with a reason.
+
+    By default, some checks are done on the request in _validate() based on the following class variables:
+    - `_SUPPORTED_URL_SCHEMES`: a tuple of supported url schemes.
+        Any Request with an url scheme not in this list will raise an UnsupportedRequest.
+
+    - `_SUPPORTED_PROXY_SCHEMES`: a tuple of support proxy url schemes. Any Request that contains
+        a proxy url with an url scheme not in this list will raise an UnsupportedRequest.
+
+    - `_SUPPORTED_FEATURES`: a tuple of supported features, as defined in Features enum.
+    The above may be set to None to disable the checks.
+
+    Parameters:
+    @param logger: logger instance
+    @param headers: HTTP Headers to include when sending requests.
+    @param cookiejar: Cookiejar to use for requests.
+    @param timeout: Socket timeout to use when sending requests.
+    @param proxies: Proxies to use for sending requests.
+    @param source_address: Client-side IP address to bind to for requests.
+    @param verbose: Print debug request and traffic information to stdout.
+    @param prefer_system_certs: Whether to prefer system certificates over other means (e.g. certifi).
+    @param client_cert: SSL client certificate configuration.
+            dict with {client_certificate, client_certificate_key, client_certificate_password}
+    @param verify: Verify SSL certificates
+    @param legacy_ssl_support: Enable legacy SSL options such as legacy server connect and older cipher support.
+
+    Some configuration options may be available for individual Requests too. In this case,
+    either the Request configuration option takes precedence or they are merged.
+
+    Requests may have additional optional parameters defined as extensions.
+     RequestHandler subclasses may choose to support custom extensions.
+
+    The following extensions are defined for RequestHandler:
+    - `cookiejar`: Cookiejar to use for this request
+    - `timeout`: socket timeout to use for this request
+
+    Apart from the url protocol, proxies dict may contain the following keys:
+    - `all`: proxy to use for all protocols. Used as a fallback if no proxy is set for a specific protocol.
+    - `no`: comma seperated list of hostnames (optionally with port) to not use a proxy for.
+    Note: a RequestHandler may not support these, as defined in `_SUPPORTED_FEATURES`.
+
+    """
+
+    _SUPPORTED_URL_SCHEMES = ()
+    _SUPPORTED_PROXY_SCHEMES = ()
+    _SUPPORTED_FEATURES = ()
+
+    def __init__(
+        self, *,
+        logger,  # TODO(Grub4k): default logger
+        headers: HTTPHeaderDict = None,
+        cookiejar: CookieJar = None,
+        timeout: float | int | None = None,
+        proxies: dict = None,
+        source_address: str = None,
+        verbose: bool = False,
+        prefer_system_certs: bool = False,
+        client_cert: dict[str, str | None] = None,
+        verify: bool = True,
+        legacy_ssl_support: bool = False,
+        **_,
+    ):
+
+        self._logger = logger
+        self.headers = headers or {}
+        self.cookiejar = cookiejar if cookiejar is not None else CookieJar()
+        self.timeout = float(timeout or 20)
+        self.proxies = proxies or {}
+        self.source_address = source_address
+        self.verbose = verbose
+        self.prefer_system_certs = prefer_system_certs
+        self._client_cert = client_cert or {}
+        self.verify = verify
+        self.legacy_ssl_support = legacy_ssl_support
+        super().__init__()
+
+    def _make_sslcontext(self):
+        return make_ssl_context(
+            verify=self.verify,
+            legacy_support=self.legacy_ssl_support,
+            use_certifi=not self.prefer_system_certs,
+            **self._client_cert,
+        )
+
+    def _merge_headers(self, request_headers):
+        return HTTPHeaderDict(self.headers, request_headers)
+
+    def _check_url_scheme(self, request: Request):
+        scheme = urllib.parse.urlparse(request.url).scheme.lower()
+        if self._SUPPORTED_URL_SCHEMES is not None and scheme not in self._SUPPORTED_URL_SCHEMES:
+            raise UnsupportedRequest(f'Unsupported url scheme: "{scheme}"')
+        return scheme  # for further processing
+
+    def _check_proxies(self, proxies):
+        for proxy_key, proxy_url in proxies.items():
+            if proxy_url is None:
+                continue
+            if proxy_key == 'no':
+                if self._SUPPORTED_FEATURES is not None and Features.NO_PROXY not in self._SUPPORTED_FEATURES:
+                    raise UnsupportedRequest('"no" proxy is not supported')
+                continue
+            if (
+                proxy_key == 'all'
+                and self._SUPPORTED_FEATURES is not None
+                and Features.ALL_PROXY not in self._SUPPORTED_FEATURES
+            ):
+                raise UnsupportedRequest('"all" proxy is not supported')
+
+            # Unlikely this handler will use this proxy, so ignore.
+            # This is to allow a case where a proxy may be set for a protocol
+            # for one handler in which such protocol (and proxy) is not supported by another handler.
+            if self._SUPPORTED_URL_SCHEMES is not None and proxy_key not in (*self._SUPPORTED_URL_SCHEMES, 'all'):
+                continue
+
+            if self._SUPPORTED_PROXY_SCHEMES is None:
+                # Skip proxy scheme checks
+                continue
+
+            # Scheme-less proxies are not supported
+            if urllib.request._parse_proxy(proxy_url)[0] is None:
+                raise UnsupportedRequest(f'Proxy "{proxy_url}" missing scheme')
+
+            scheme = urllib.parse.urlparse(proxy_url).scheme.lower()
+            if scheme not in self._SUPPORTED_PROXY_SCHEMES:
+                raise UnsupportedRequest(f'Unsupported proxy type: "{scheme}"')
+
+    def _check_cookiejar_extension(self, extensions):
+        if not extensions.get('cookiejar'):
+            return
+        if not isinstance(extensions['cookiejar'], CookieJar):
+            raise UnsupportedRequest('cookiejar is not a CookieJar')
+
+    def _check_timeout_extension(self, extensions):
+        if extensions.get('timeout') is None:
+            return
+        if not isinstance(extensions['timeout'], (float, int)):
+            raise UnsupportedRequest('timeout is not a float or int')
+
+    def _check_extensions(self, extensions):
+        self._check_cookiejar_extension(extensions)
+        self._check_timeout_extension(extensions)
+
+    def _validate(self, request):
+        self._check_url_scheme(request)
+        self._check_proxies(request.proxies or self.proxies)
+        self._check_extensions(request.extensions)
+
+    @wrap_request_errors
+    def validate(self, request: Request):
+        if not isinstance(request, Request):
+            raise TypeError('Expected an instance of Request')
+        self._validate(request)
+
+    @wrap_request_errors
+    def send(self, request: Request) -> Response:
+        if not isinstance(request, Request):
+            raise TypeError('Expected an instance of Request')
+        return self._send(request)
+
+    @abc.abstractmethod
+    def _send(self, request: Request):
+        """Handle a request from start to finish. Redefine in subclasses."""
+
+    def close(self):
+        pass
+
+    @classproperty
+    def RH_NAME(cls):
+        return cls.__name__[:-2]
+
+    @classproperty
+    def RH_KEY(cls):
+        assert cls.__name__.endswith('RH'), 'RequestHandler class names must end with "RH"'
+        return cls.__name__[:-2]
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+
+class Request:
+    """
+    Represents a request to be made.
+    Partially backwards-compatible with urllib.request.Request.
+
+    @param url: url to send. Will be sanitized.
+    @param data: payload data to send. Must be bytes, iterable of bytes, a file-like object or None
+    @param headers: headers to send.
+    @param proxies: proxy dict mapping of proto:proxy to use for the request and any redirects.
+    @param query: URL query parameters to update the url with.
+    @param method: HTTP method to use. If no method specified, will use POST if payload data is present else GET
+    @param extensions: Dictionary of Request extensions to add, as supported by handlers.
+    """
+
+    def __init__(
+            self,
+            url: str,
+            data: RequestData = None,
+            headers: typing.Mapping = None,
+            proxies: dict = None,
+            query: dict = None,
+            method: str = None,
+            extensions: dict = None
+    ):
+
+        self._headers = HTTPHeaderDict()
+        self._data = None
+
+        if query:
+            url = update_url_query(url, query)
+
+        self.url = url
+        self.method = method
+        if headers:
+            self.headers = headers
+        self.data = data  # note: must be done after setting headers
+        self.proxies = proxies or {}
+        self.extensions = extensions or {}
+
+    @property
+    def url(self):
+        return self._url
+
+    @url.setter
+    def url(self, url):
+        if not isinstance(url, str):
+            raise TypeError('url must be a string')
+        elif url.startswith('//'):
+            url = 'http:' + url
+        self._url = escape_url(url)
+
+    @property
+    def method(self):
+        return self._method or ('POST' if self.data is not None else 'GET')
+
+    @method.setter
+    def method(self, method):
+        if method is None:
+            self._method = None
+        elif isinstance(method, str):
+            self._method = method.upper()
+        else:
+            raise TypeError('method must be a string')
+
+    @property
+    def data(self):
+        return self._data
+
+    @data.setter
+    def data(self, data: RequestData):
+        # Try catch some common mistakes
+        if data is not None and (
+            not isinstance(data, (bytes, io.IOBase, Iterable)) or isinstance(data, (str, Mapping))
+        ):
+            raise TypeError('data must be bytes, iterable of bytes, or a file-like object')
+
+        if data == self._data and self._data is None:
+            self.headers.pop('Content-Length', None)
+
+        # https://docs.python.org/3/library/urllib.request.html#urllib.request.Request.data
+        if data != self._data:
+            if self._data is not None:
+                self.headers.pop('Content-Length', None)
+            self._data = data
+
+        if self._data is None:
+            self.headers.pop('Content-Type', None)
+
+        if 'Content-Type' not in self.headers and self._data is not None:
+            self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+
+    @property
+    def headers(self) -> HTTPHeaderDict:
+        return self._headers
+
+    @headers.setter
+    def headers(self, new_headers: Mapping):
+        """Replaces headers of the request. If not a CaseInsensitiveDict, it will be converted to one."""
+        if isinstance(new_headers, HTTPHeaderDict):
+            self._headers = new_headers
+        elif isinstance(new_headers, Mapping):
+            self._headers = HTTPHeaderDict(new_headers)
+        else:
+            raise TypeError('headers must be a mapping')
+
+    def update(self, url=None, data=None, headers=None, query=None):
+        self.data = data or self.data
+        self.headers.update(headers or {})
+        self.url = update_url_query(url or self.url, query or {})
+
+    def copy(self):
+        return self.__class__(
+            url=self.url,
+            headers=copy.deepcopy(self.headers),
+            proxies=copy.deepcopy(self.proxies),
+            data=self._data,
+            extensions=copy.copy(self.extensions),
+            method=self._method,
+        )
+
+
+HEADRequest = functools.partial(Request, method='HEAD')
+PUTRequest = functools.partial(Request, method='PUT')
+
+
+class Response(io.IOBase):
+    """
+    Base class for HTTP response adapters.
+
+    By default, it provides a basic wrapper for a file-like response object.
+
+    Interface partially backwards-compatible with addinfourl and http.client.HTTPResponse.
+
+    @param fp: Original, file-like, response.
+    @param url: URL that this is a response of.
+    @param headers: response headers.
+    @param status: Response HTTP status code. Default is 200 OK.
+    @param reason: HTTP status reason. Will use built-in reasons based on status code if not provided.
+    """
+
+    def __init__(
+            self,
+            fp: typing.IO,
+            url: str,
+            headers: Mapping[str, str],
+            status: int = 200,
+            reason: str = None):
+
+        self.fp = fp
+        self.headers = Message()
+        for name, value in headers.items():
+            self.headers.add_header(name, value)
+        self.status = status
+        self.url = url
+        try:
+            self.reason = reason or HTTPStatus(status).phrase
+        except ValueError:
+            self.reason = None
+
+    def readable(self):
+        return self.fp.readable()
+
+    def read(self, amt: int = None) -> bytes:
+        # Expected errors raised here should be of type RequestError or subclasses.
+        # Subclasses should redefine this method with more precise error handling.
+        try:
+            return self.fp.read(amt)
+        except Exception as e:
+            raise TransportError(cause=e) from e
+
+    def close(self):
+        self.fp.close()
+        return super().close()
+
+    def get_header(self, name, default=None):
+        """Get header for name.
+        If there are multiple matching headers, return all seperated by comma."""
+        headers = self.headers.get_all(name)
+        if not headers:
+            return default
+        if name.title() == 'Set-Cookie':
+            # Special case, only get the first one
+            # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.3-4.1
+            return headers[0]
+        return ', '.join(headers)
+
+    # The following methods are for compatability reasons and are deprecated
+    @property
+    def code(self):
+        return self.status
+
+    def getcode(self):
+        return self.status
+
+    def geturl(self):
+        return self.url
+
+    def info(self):
+        return self.headers
+
+    def getheader(self, name, default=None):
+        return self.get_header(name, default)
diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py
index 89b484a22..6fe8afb92 100644
--- a/yt_dlp/networking/exceptions.py
+++ b/yt_dlp/networking/exceptions.py
@@ -1,9 +1,197 @@
-import http.client
-import socket
-import ssl
+from __future__ import annotations
+
+import typing
 import urllib.error
 
-network_exceptions = [urllib.error.URLError, http.client.HTTPException, socket.error]
-if hasattr(ssl, 'CertificateError'):
-    network_exceptions.append(ssl.CertificateError)
-network_exceptions = tuple(network_exceptions)
+from ..utils import YoutubeDLError
+
+if typing.TYPE_CHECKING:
+    from .common import RequestHandler, Response
+
+
+class RequestError(YoutubeDLError):
+    def __init__(
+        self,
+        msg: str | None = None,
+        cause: Exception | str | None = None,
+        handler: RequestHandler = None
+    ):
+        self.handler = handler
+        self.cause = cause
+        if not msg and cause:
+            msg = str(cause)
+        super().__init__(msg)
+
+
+class UnsupportedRequest(RequestError):
+    """raised when a handler cannot handle a request"""
+    pass
+
+
+class NoSupportingHandlers(RequestError):
+    """raised when no handlers can support a request for various reasons"""
+
+    def __init__(self, unsupported_errors: list[UnsupportedRequest], unexpected_errors: list[Exception]):
+        self.unsupported_errors = unsupported_errors or []
+        self.unexpected_errors = unexpected_errors or []
+
+        # Print a quick summary of the errors
+        err_handler_map = {}
+        for err in unsupported_errors:
+            err_handler_map.setdefault(err.msg, []).append(err.handler.RH_NAME)
+
+        reason_str = ', '.join([f'{msg} ({", ".join(handlers)})' for msg, handlers in err_handler_map.items()])
+        if unexpected_errors:
+            reason_str = ' + '.join(filter(None, [reason_str, f'{len(unexpected_errors)} unexpected error(s)']))
+
+        err_str = 'Unable to handle request'
+        if reason_str:
+            err_str += f': {reason_str}'
+
+        super().__init__(msg=err_str)
+
+
+class TransportError(RequestError):
+    """Network related errors"""
+
+
+class HTTPError(RequestError):
+    def __init__(self, response: Response, redirect_loop=False):
+        self.response = response
+        self.status = response.status
+        self.reason = response.reason
+        self.redirect_loop = redirect_loop
+        msg = f'HTTP Error {response.status}: {response.reason}'
+        if redirect_loop:
+            msg += ' (redirect loop detected)'
+
+        super().__init__(msg=msg)
+
+    def close(self):
+        self.response.close()
+
+    def __repr__(self):
+        return f'<HTTPError {self.status}: {self.reason}>'
+
+
+class IncompleteRead(TransportError):
+    def __init__(self, partial, expected=None, **kwargs):
+        self.partial = partial
+        self.expected = expected
+        msg = f'{len(partial)} bytes read'
+        if expected is not None:
+            msg += f', {expected} more expected'
+
+        super().__init__(msg=msg, **kwargs)
+
+    def __repr__(self):
+        return f'<IncompleteRead: {self.msg}>'
+
+
+class SSLError(TransportError):
+    pass
+
+
+class CertificateVerifyError(SSLError):
+    """Raised when certificate validated has failed"""
+    pass
+
+
+class ProxyError(TransportError):
+    pass
+
+
+class _CompatHTTPError(urllib.error.HTTPError, HTTPError):
+    """
+    Provides backwards compatibility with urllib.error.HTTPError.
+    Do not use this class directly, use HTTPError instead.
+    """
+
+    def __init__(self, http_error: HTTPError):
+        super().__init__(
+            url=http_error.response.url,
+            code=http_error.status,
+            msg=http_error.msg,
+            hdrs=http_error.response.headers,
+            fp=http_error.response
+        )
+        self._closer.file = None  # Disable auto close
+        self._http_error = http_error
+        HTTPError.__init__(self, http_error.response, redirect_loop=http_error.redirect_loop)
+
+    @property
+    def status(self):
+        return self._http_error.status
+
+    @status.setter
+    def status(self, value):
+        return
+
+    @property
+    def reason(self):
+        return self._http_error.reason
+
+    @reason.setter
+    def reason(self, value):
+        return
+
+    @property
+    def headers(self):
+        return self._http_error.response.headers
+
+    @headers.setter
+    def headers(self, value):
+        return
+
+    def info(self):
+        return self.response.headers
+
+    def getcode(self):
+        return self.status
+
+    def geturl(self):
+        return self.response.url
+
+    @property
+    def code(self):
+        return self.status
+
+    @code.setter
+    def code(self, value):
+        return
+
+    @property
+    def url(self):
+        return self.response.url
+
+    @url.setter
+    def url(self, value):
+        return
+
+    @property
+    def hdrs(self):
+        return self.response.headers
+
+    @hdrs.setter
+    def hdrs(self, value):
+        return
+
+    @property
+    def filename(self):
+        return self.response.url
+
+    @filename.setter
+    def filename(self, value):
+        return
+
+    def __getattr__(self, name):
+        return super().__getattr__(name)
+
+    def __str__(self):
+        return str(self._http_error)
+
+    def __repr__(self):
+        return repr(self._http_error)
+
+
+network_exceptions = (HTTPError, TransportError)
diff --git a/yt_dlp/utils/_deprecated.py b/yt_dlp/utils/_deprecated.py
index ca0fb1614..e55d42354 100644
--- a/yt_dlp/utils/_deprecated.py
+++ b/yt_dlp/utils/_deprecated.py
@@ -10,16 +10,16 @@
 
 
 from ._utils import preferredencoding
+from ..networking._urllib import HTTPHandler
 
 # isort: split
+from .networking import random_user_agent, std_headers  # noqa: F401
 from ..networking._urllib import PUTRequest  # noqa: F401
 from ..networking._urllib import SUPPORTED_ENCODINGS, HEADRequest  # noqa: F401
-from ..networking._urllib import HTTPHandler as YoutubeDLHandler  # noqa: F401
 from ..networking._urllib import ProxyHandler as PerRequestProxyHandler  # noqa: F401
 from ..networking._urllib import RedirectHandler as YoutubeDLRedirectHandler  # noqa: F401
 from ..networking._urllib import make_socks_conn_class, update_Request  # noqa: F401
 from ..networking.exceptions import network_exceptions  # noqa: F401
-from .networking import random_user_agent, std_headers  # noqa: F401
 
 
 def encodeFilename(s, for_subprocess=False):
@@ -47,3 +47,12 @@ def decodeOption(optval):
 
 def error_to_compat_str(err):
     return str(err)
+
+
+class YoutubeDLHandler(HTTPHandler):
+    def __init__(self, params, *args, **kwargs):
+        self._params = params
+        super().__init__(*args, **kwargs)
+
+
+YoutubeDLHTTPSHandler = YoutubeDLHandler
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index d5704cadc..d0e328716 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -15,8 +15,6 @@
 import hmac
 import html.entities
 import html.parser
-import http.client
-import http.cookiejar
 import inspect
 import io
 import itertools
@@ -897,6 +895,7 @@ def formatSeconds(secs, delim=':', msec=False):
 
 
 def make_HTTPS_handler(params, **kwargs):
+    from ._deprecated import YoutubeDLHTTPSHandler
     from ..networking._helper import make_ssl_context
     return YoutubeDLHTTPSHandler(params, context=make_ssl_context(
         verify=not params.get('nocheckcertificate'),
@@ -1140,38 +1139,6 @@ class XAttrUnavailableError(YoutubeDLError):
     pass
 
 
-class YoutubeDLHTTPSHandler(urllib.request.HTTPSHandler):
-    def __init__(self, params, https_conn_class=None, *args, **kwargs):
-        urllib.request.HTTPSHandler.__init__(self, *args, **kwargs)
-        self._https_conn_class = https_conn_class or http.client.HTTPSConnection
-        self._params = params
-
-    def https_open(self, req):
-        kwargs = {}
-        conn_class = self._https_conn_class
-
-        if hasattr(self, '_context'):  # python > 2.6
-            kwargs['context'] = self._context
-        if hasattr(self, '_check_hostname'):  # python 3.x
-            kwargs['check_hostname'] = self._check_hostname
-
-        socks_proxy = req.headers.get('Ytdl-socks-proxy')
-        if socks_proxy:
-            from ..networking._urllib import make_socks_conn_class
-            conn_class = make_socks_conn_class(conn_class, socks_proxy)
-            del req.headers['Ytdl-socks-proxy']
-
-        from ..networking._urllib import _create_http_connection
-        try:
-            return self.do_open(
-                functools.partial(_create_http_connection, self, conn_class, True), req, **kwargs)
-        except urllib.error.URLError as e:
-            if (isinstance(e.reason, ssl.SSLError)
-                    and getattr(e.reason, 'reason', None) == 'SSLV3_ALERT_HANDSHAKE_FAILURE'):
-                raise YoutubeDLError('SSLV3_ALERT_HANDSHAKE_FAILURE: Try using --legacy-server-connect')
-            raise
-
-
 def is_path_like(f):
     return isinstance(f, (str, bytes, os.PathLike))
 
diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py
index 95b54fabe..ac355ddc8 100644
--- a/yt_dlp/utils/networking.py
+++ b/yt_dlp/utils/networking.py
@@ -1,4 +1,9 @@
+import collections
 import random
+import urllib.parse
+import urllib.request
+
+from ._utils import remove_start
 
 
 def random_user_agent():
@@ -46,15 +51,67 @@ def random_user_agent():
     return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)
 
 
-std_headers = {
+class HTTPHeaderDict(collections.UserDict, dict):
+    """
+    Store and access keys case-insensitively.
+    The constructor can take multiple dicts, in which keys in the latter are prioritised.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__()
+        for dct in args:
+            if dct is not None:
+                self.update(dct)
+        self.update(kwargs)
+
+    def __setitem__(self, key, value):
+        super().__setitem__(key.title(), str(value))
+
+    def __getitem__(self, key):
+        return super().__getitem__(key.title())
+
+    def __delitem__(self, key):
+        super().__delitem__(key.title())
+
+    def __contains__(self, key):
+        return super().__contains__(key.title() if isinstance(key, str) else key)
+
+
+std_headers = HTTPHeaderDict({
     'User-Agent': random_user_agent(),
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Language': 'en-us,en;q=0.5',
     'Sec-Fetch-Mode': 'navigate',
-}
+})
 
 
-def clean_headers(headers):
-    if 'Youtubedl-no-compression' in headers:  # compat
-        del headers['Youtubedl-no-compression']
+def clean_proxies(proxies: dict, headers: HTTPHeaderDict):
+    req_proxy = headers.pop('Ytdl-Request-Proxy', None)
+    if req_proxy:
+        proxies.clear()  # XXX: compat: Ytdl-Request-Proxy takes preference over everything, including NO_PROXY
+        proxies['all'] = req_proxy
+    for proxy_key, proxy_url in proxies.items():
+        if proxy_url == '__noproxy__':
+            proxies[proxy_key] = None
+            continue
+        if proxy_key == 'no':  # special case
+            continue
+        if proxy_url is not None:
+            # Ensure proxies without a scheme are http.
+            proxy_scheme = urllib.request._parse_proxy(proxy_url)[0]
+            if proxy_scheme is None:
+                proxies[proxy_key] = 'http://' + remove_start(proxy_url, '//')
+
+            replace_scheme = {
+                'socks5': 'socks5h',  # compat: socks5 was treated as socks5h
+                'socks': 'socks4'  # compat: non-standard
+            }
+            if proxy_scheme in replace_scheme:
+                proxies[proxy_key] = urllib.parse.urlunparse(
+                    urllib.parse.urlparse(proxy_url)._replace(scheme=replace_scheme[proxy_scheme]))
+
+
+def clean_headers(headers: HTTPHeaderDict):
+    if 'Youtubedl-No-Compression' in headers:  # compat
+        del headers['Youtubedl-No-Compression']
         headers['Accept-Encoding'] = 'identity'

From 3d2623a898196640f7cc0fc8b70118ff19e6925d Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sun, 9 Jul 2023 13:23:02 +0530
Subject: [PATCH 448/871] [compat, networking] Deprecate old functions (#2861)

Authored by: coletdjnz, pukkandan
---
 test/test_download.py                   |  2 +-
 test/test_networking.py                 | 27 ++++++-----
 test/test_networking_utils.py           | 64 ++++++++++++++++++++-----
 yt_dlp/YoutubeDL.py                     | 12 +++--
 yt_dlp/__init__.py                      |  2 +-
 yt_dlp/compat/_deprecated.py            |  1 -
 yt_dlp/compat/_legacy.py                |  1 +
 yt_dlp/downloader/external.py           |  7 ++-
 yt_dlp/downloader/f4m.py                |  8 ++--
 yt_dlp/downloader/fragment.py           | 19 +++-----
 yt_dlp/downloader/hls.py                |  2 +-
 yt_dlp/downloader/http.py               | 41 ++++++++--------
 yt_dlp/downloader/ism.py                |  4 +-
 yt_dlp/downloader/niconico.py           | 11 ++---
 yt_dlp/downloader/youtube_live_chat.py  | 10 ++--
 yt_dlp/extractor/abematv.py             |  3 +-
 yt_dlp/extractor/adn.py                 | 16 +++----
 yt_dlp/extractor/adobepass.py           | 20 ++++----
 yt_dlp/extractor/ant1newsgr.py          |  4 +-
 yt_dlp/extractor/archiveorg.py          | 12 ++---
 yt_dlp/extractor/atresplayer.py         |  6 +--
 yt_dlp/extractor/bbc.py                 | 14 +++---
 yt_dlp/extractor/bilibili.py            |  4 +-
 yt_dlp/extractor/bitchute.py            |  2 +-
 yt_dlp/extractor/bravotv.py             |  4 +-
 yt_dlp/extractor/brightcove.py          |  6 +--
 yt_dlp/extractor/canalplus.py           |  2 +-
 yt_dlp/extractor/cbsnews.py             |  2 +-
 yt_dlp/extractor/ceskatelevize.py       | 30 ++++++------
 yt_dlp/extractor/cinetecamilano.py      |  4 +-
 yt_dlp/extractor/ciscowebex.py          |  6 +--
 yt_dlp/extractor/common.py              | 41 +++++++++-------
 yt_dlp/extractor/crackle.py             |  4 +-
 yt_dlp/extractor/crunchyroll.py         |  4 +-
 yt_dlp/extractor/cultureunplugged.py    |  6 +--
 yt_dlp/extractor/dacast.py              |  4 +-
 yt_dlp/extractor/dailymotion.py         |  6 +--
 yt_dlp/extractor/discovery.py           |  6 +--
 yt_dlp/extractor/dplay.py               |  8 ++--
 yt_dlp/extractor/eagleplatform.py       |  6 +--
 yt_dlp/extractor/eitb.py                | 10 ++--
 yt_dlp/extractor/eporner.py             |  2 +-
 yt_dlp/extractor/facebook.py            | 14 +++---
 yt_dlp/extractor/fc2.py                 |  6 +--
 yt_dlp/extractor/filmon.py              | 14 +++---
 yt_dlp/extractor/fox.py                 | 10 ++--
 yt_dlp/extractor/foxsports.py           |  5 +-
 yt_dlp/extractor/fujitv.py              |  2 +-
 yt_dlp/extractor/funimation.py          |  6 +--
 yt_dlp/extractor/gdcvault.py            | 15 ++----
 yt_dlp/extractor/generic.py             |  8 ++--
 yt_dlp/extractor/globo.py               |  2 +-
 yt_dlp/extractor/googledrive.py         |  2 +-
 yt_dlp/extractor/hketv.py               |  2 +-
 yt_dlp/extractor/hotnewhiphop.py        | 14 ++----
 yt_dlp/extractor/hotstar.py             |  5 +-
 yt_dlp/extractor/hrti.py                | 10 ++--
 yt_dlp/extractor/ign.py                 | 17 +++----
 yt_dlp/extractor/imggaming.py           |  6 +--
 yt_dlp/extractor/instagram.py           |  6 +--
 yt_dlp/extractor/iprima.py              |  4 +-
 yt_dlp/extractor/kakao.py               |  6 +--
 yt_dlp/extractor/kick.py                |  3 +-
 yt_dlp/extractor/kuwo.py                |  2 +-
 yt_dlp/extractor/la7.py                 |  9 +---
 yt_dlp/extractor/lbry.py                |  4 +-
 yt_dlp/extractor/lecturio.py            |  2 +-
 yt_dlp/extractor/lego.py                |  4 +-
 yt_dlp/extractor/limelight.py           |  6 +--
 yt_dlp/extractor/linuxacademy.py        | 15 +++---
 yt_dlp/extractor/mediasite.py           |  2 +-
 yt_dlp/extractor/megatvcom.py           |  6 +--
 yt_dlp/extractor/mgtv.py                |  6 +--
 yt_dlp/extractor/minds.py               |  2 +-
 yt_dlp/extractor/miomio.py              | 10 ++--
 yt_dlp/extractor/mtv.py                 | 11 ++---
 yt_dlp/extractor/nbc.py                 |  2 +-
 yt_dlp/extractor/nebula.py              |  4 +-
 yt_dlp/extractor/neteasemusic.py        |  6 +--
 yt_dlp/extractor/niconico.py            | 16 +++----
 yt_dlp/extractor/njpwworld.py           |  2 +-
 yt_dlp/extractor/nosvideo.py            |  6 +--
 yt_dlp/extractor/nowness.py             |  8 ++--
 yt_dlp/extractor/nrk.py                 |  5 +-
 yt_dlp/extractor/odkmedia.py            |  6 +--
 yt_dlp/extractor/odnoklassniki.py       |  4 +-
 yt_dlp/extractor/orf.py                 |  2 +-
 yt_dlp/extractor/owncloud.py            |  2 +-
 yt_dlp/extractor/packtpub.py            | 11 ++---
 yt_dlp/extractor/patreon.py             |  6 +--
 yt_dlp/extractor/peloton.py             | 12 ++---
 yt_dlp/extractor/piapro.py              |  2 +-
 yt_dlp/extractor/pladform.py            |  2 +-
 yt_dlp/extractor/platzi.py              |  2 +-
 yt_dlp/extractor/playplustv.py          | 14 ++----
 yt_dlp/extractor/pornhub.py             | 11 +++--
 yt_dlp/extractor/puhutv.py              |  8 ++--
 yt_dlp/extractor/radiko.py              |  2 +-
 yt_dlp/extractor/radiocanada.py         |  6 +--
 yt_dlp/extractor/rcs.py                 |  2 +-
 yt_dlp/extractor/rcti.py                |  4 +-
 yt_dlp/extractor/recurbate.py           |  5 +-
 yt_dlp/extractor/redbulltv.py           |  6 +--
 yt_dlp/extractor/redgifs.py             |  4 +-
 yt_dlp/extractor/regiotv.py             | 10 ++--
 yt_dlp/extractor/rokfin.py              |  4 +-
 yt_dlp/extractor/roosterteeth.py        | 10 ++--
 yt_dlp/extractor/rozhlas.py             |  4 +-
 yt_dlp/extractor/rte.py                 |  6 +--
 yt_dlp/extractor/rts.py                 |  4 +-
 yt_dlp/extractor/rumble.py              |  4 +-
 yt_dlp/extractor/safari.py              |  6 +--
 yt_dlp/extractor/sbs.py                 |  2 +-
 yt_dlp/extractor/sevenplus.py           | 10 ++--
 yt_dlp/extractor/shahid.py              |  8 ++--
 yt_dlp/extractor/sina.py                | 12 ++---
 yt_dlp/extractor/sixplay.py             |  2 +-
 yt_dlp/extractor/slideslive.py          |  2 +-
 yt_dlp/extractor/sonyliv.py             | 10 ++--
 yt_dlp/extractor/soundcloud.py          | 17 +++----
 yt_dlp/extractor/teachable.py           |  2 +-
 yt_dlp/extractor/telemundo.py           |  9 ++--
 yt_dlp/extractor/tennistv.py            |  2 +-
 yt_dlp/extractor/tenplay.py             |  9 ++--
 yt_dlp/extractor/tfo.py                 |  8 +---
 yt_dlp/extractor/theplatform.py         |  4 +-
 yt_dlp/extractor/thisoldhouse.py        |  4 +-
 yt_dlp/extractor/threeqsdn.py           |  4 +-
 yt_dlp/extractor/tiktok.py              |  4 +-
 yt_dlp/extractor/toutv.py               |  6 +--
 yt_dlp/extractor/triller.py             |  4 +-
 yt_dlp/extractor/trueid.py              |  6 +--
 yt_dlp/extractor/tubetugraz.py          |  8 ++--
 yt_dlp/extractor/tubitv.py              |  8 ++--
 yt_dlp/extractor/tumblr.py              |  2 +-
 yt_dlp/extractor/tunein.py              |  4 +-
 yt_dlp/extractor/tv2.py                 | 10 ++--
 yt_dlp/extractor/tvp.py                 |  4 +-
 yt_dlp/extractor/tvplay.py              | 10 ++--
 yt_dlp/extractor/tvplayer.py            | 10 ++--
 yt_dlp/extractor/twitcasting.py         |  4 +-
 yt_dlp/extractor/twitch.py              |  2 +-
 yt_dlp/extractor/twitter.py             |  2 +-
 yt_dlp/extractor/udemy.py               | 15 +++---
 yt_dlp/extractor/vevo.py                | 10 ++--
 yt_dlp/extractor/vice.py                | 10 ++--
 yt_dlp/extractor/videocampus_sachsen.py |  4 +-
 yt_dlp/extractor/vidio.py               |  2 +-
 yt_dlp/extractor/vidlii.py              |  2 +-
 yt_dlp/extractor/viewlift.py            |  6 +--
 yt_dlp/extractor/viidea.py              |  6 +--
 yt_dlp/extractor/vimeo.py               | 31 +++++-------
 yt_dlp/extractor/vk.py                  |  2 +-
 yt_dlp/extractor/vocaroo.py             |  6 +--
 yt_dlp/extractor/vodlocker.py           | 12 ++---
 yt_dlp/extractor/voot.py                |  4 +-
 yt_dlp/extractor/vrt.py                 |  4 +-
 yt_dlp/extractor/vrv.py                 |  7 +--
 yt_dlp/extractor/weibo.py               |  2 +-
 yt_dlp/extractor/weverse.py             |  8 ++--
 yt_dlp/extractor/wistia.py              |  6 +--
 yt_dlp/extractor/wykop.py               |  4 +-
 yt_dlp/extractor/xhamster.py            |  2 +-
 yt_dlp/extractor/xtube.py               |  4 +-
 yt_dlp/extractor/yesjapan.py            |  9 ++--
 yt_dlp/extractor/youtube.py             | 15 +++---
 yt_dlp/extractor/zaiko.py               |  2 +-
 yt_dlp/extractor/zattoo.py              |  5 +-
 yt_dlp/extractor/zype.py                |  6 +--
 yt_dlp/networking/common.py             |  6 +++
 yt_dlp/networking/exceptions.py         | 22 ++++++++-
 yt_dlp/postprocessor/common.py          | 11 ++---
 yt_dlp/update.py                        |  9 ++--
 yt_dlp/utils/_deprecated.py             | 19 --------
 yt_dlp/utils/_legacy.py                 | 62 +++++++++++++++++++++++-
 yt_dlp/utils/_utils.py                  | 47 +-----------------
 176 files changed, 707 insertions(+), 729 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index fd7752cdd..6f00a4ded 100755
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -160,7 +160,7 @@ def try_rm_tcs_files(tcs=None):
                         force_generic_extractor=params.get('force_generic_extractor', False))
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
-                    if not isinstance(err.exc_info[1], (TransportError, UnavailableVideoError)) or (isinstance(err.exc_info[1], HTTPError) and err.exc_info[1].code == 503):
+                    if not isinstance(err.exc_info[1], (TransportError, UnavailableVideoError)) or (isinstance(err.exc_info[1], HTTPError) and err.exc_info[1].status == 503):
                         err.msg = f'{getattr(err, "msg", err)} ({tname})'
                         raise
 
diff --git a/test/test_networking.py b/test/test_networking.py
index 147a4ff49..b60ed283b 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -1057,14 +1057,15 @@ def test_compat_request(self):
             urllib_req = urllib.request.Request('http://foo.bar', data=b'test', method='PUT', headers={'X-Test': '1'})
             urllib_req.add_unredirected_header('Cookie', 'bob=bob')
             urllib_req.timeout = 2
-
-            req = ydl.urlopen(urllib_req).request
-            assert req.url == urllib_req.get_full_url()
-            assert req.data == urllib_req.data
-            assert req.method == urllib_req.get_method()
-            assert 'X-Test' in req.headers
-            assert 'Cookie' in req.headers
-            assert req.extensions.get('timeout') == 2
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', category=DeprecationWarning)
+                req = ydl.urlopen(urllib_req).request
+                assert req.url == urllib_req.get_full_url()
+                assert req.data == urllib_req.data
+                assert req.method == urllib_req.get_method()
+                assert 'X-Test' in req.headers
+                assert 'Cookie' in req.headers
+                assert req.extensions.get('timeout') == 2
 
             with pytest.raises(AssertionError):
                 ydl.urlopen(None)
@@ -1362,7 +1363,9 @@ def test_get_header(self):
 
     def test_compat(self):
         res = Response(io.BytesIO(b''), url='test://', status=404, headers={'test': 'test'})
-        assert res.code == res.getcode() == res.status
-        assert res.geturl() == res.url
-        assert res.info() is res.headers
-        assert res.getheader('test') == res.get_header('test')
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', category=DeprecationWarning)
+            assert res.code == res.getcode() == res.status
+            assert res.geturl() == res.url
+            assert res.info() is res.headers
+            assert res.getheader('test') == res.get_header('test')
diff --git a/test/test_networking_utils.py b/test/test_networking_utils.py
index f9f876af3..ef46f79ed 100644
--- a/test/test_networking_utils.py
+++ b/test/test_networking_utils.py
@@ -8,11 +8,13 @@
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import contextlib
 import io
 import platform
 import random
 import ssl
 import urllib.error
+import warnings
 
 from yt_dlp.cookies import YoutubeDLCookieJar
 from yt_dlp.dependencies import certifi
@@ -202,20 +204,58 @@ def test_compat_http_error(self):
         assert isinstance(error, HTTPError)
         assert isinstance(error, urllib.error.HTTPError)
 
-        assert error.code == 403
-        assert error.getcode() == 403
-        assert error.hdrs is error.response.headers
-        assert error.info() is error.response.headers
-        assert error.headers is error.response.headers
-        assert error.filename == error.response.url
-        assert error.url == error.response.url
-        assert error.geturl() == error.response.url
+        @contextlib.contextmanager
+        def raises_deprecation_warning():
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter('always')
+                yield
+
+                if len(w) == 0:
+                    pytest.fail('Did not raise DeprecationWarning')
+                if len(w) > 1:
+                    pytest.fail(f'Raised multiple warnings: {w}')
+
+                if not issubclass(w[-1].category, DeprecationWarning):
+                    pytest.fail(f'Expected DeprecationWarning, got {w[-1].category}')
+                w.clear()
+
+        with raises_deprecation_warning():
+            assert error.code == 403
+
+        with raises_deprecation_warning():
+            assert error.getcode() == 403
+
+        with raises_deprecation_warning():
+            assert error.hdrs is error.response.headers
+
+        with raises_deprecation_warning():
+            assert error.info() is error.response.headers
+
+        with raises_deprecation_warning():
+            assert error.headers is error.response.headers
+
+        with raises_deprecation_warning():
+            assert error.filename == error.response.url
+
+        with raises_deprecation_warning():
+            assert error.url == error.response.url
+
+        with raises_deprecation_warning():
+            assert error.geturl() == error.response.url
 
         # Passthrough file operations
-        assert error.read() == b'test'
-        assert not error.closed
-        # Technically Response operations are also passed through, which should not be used.
-        assert error.get_header('test') == 'test'
+        with raises_deprecation_warning():
+            assert error.read() == b'test'
+
+        with raises_deprecation_warning():
+            assert not error.closed
+
+        with raises_deprecation_warning():
+            # Technically Response operations are also passed through, which should not be used.
+            assert error.get_header('test') == 'test'
+
+        # Should not raise a warning
+        error.close()
 
     @pytest.mark.skipif(
         platform.python_implementation() == 'PyPy', reason='garbage collector works differently in pypy')
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 29a18aef0..850eb8ae0 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -33,7 +33,7 @@
 from .extractor.common import UnsupportedURLIE
 from .extractor.openload import PhantomJSwrapper
 from .minicurses import format_text
-from .networking import Request, RequestDirector
+from .networking import HEADRequest, Request, RequestDirector
 from .networking.common import _REQUEST_HANDLERS
 from .networking.exceptions import (
     HTTPError,
@@ -41,6 +41,7 @@
     RequestError,
     SSLError,
     _CompatHTTPError,
+    network_exceptions,
 )
 from .plugins import directories as plugin_directories
 from .postprocessor import _PLUGIN_CLASSES as plugin_pps
@@ -80,7 +81,6 @@
     ExtractorError,
     FormatSorter,
     GeoRestrictedError,
-    HEADRequest,
     ISO3166Utils,
     LazyList,
     MaxDownloadsReached,
@@ -122,7 +122,6 @@
     locked_file,
     make_archive_id,
     make_dir,
-    network_exceptions,
     number_of_digits,
     orderedSet,
     orderedSet_from_options,
@@ -135,7 +134,6 @@
     sanitize_filename,
     sanitize_path,
     sanitize_url,
-    std_headers,
     str_or_none,
     strftime_or_none,
     subtitles_filename,
@@ -158,6 +156,7 @@
     HTTPHeaderDict,
     clean_headers,
     clean_proxies,
+    std_headers,
 )
 from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
 
@@ -4019,6 +4018,9 @@ def urlopen(self, req):
         if isinstance(req, str):
             req = Request(req)
         elif isinstance(req, urllib.request.Request):
+            self.deprecation_warning(
+                'Passing a urllib.request.Request object to YoutubeDL.urlopen() is deprecated. '
+                'Use yt_dlp.networking.common.Request instead.')
             req = urllib_req_to_req(req)
         assert isinstance(req, Request)
 
@@ -4242,7 +4244,7 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
                     ret.append((thumb_filename, thumb_filename_final))
                     t['filepath'] = thumb_filename
                 except network_exceptions as err:
-                    if isinstance(err, urllib.error.HTTPError) and err.code == 404:
+                    if isinstance(err, HTTPError) and err.status == 404:
                         self.to_screen(f'[info] {thumb_display_id.title()} does not exist')
                     else:
                         self.report_warning(f'Unable to download {thumb_display_id}: {err}')
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index b81277a57..991dbcda7 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -57,11 +57,11 @@
     read_stdin,
     render_table,
     setproctitle,
-    std_headers,
     traverse_obj,
     variadic,
     write_string,
 )
+from .utils.networking import std_headers
 from .YoutubeDL import YoutubeDL
 
 _IN_CLI = False
diff --git a/yt_dlp/compat/_deprecated.py b/yt_dlp/compat/_deprecated.py
index 342f1f80d..14d37b236 100644
--- a/yt_dlp/compat/_deprecated.py
+++ b/yt_dlp/compat/_deprecated.py
@@ -8,7 +8,6 @@
 
 compat_b64decode = base64.b64decode
 
-compat_HTTPError = urllib.error.HTTPError
 compat_urlparse = urllib.parse
 compat_parse_qs = urllib.parse.parse_qs
 compat_urllib_parse_unquote = urllib.parse.unquote
diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py
index 83bf869a8..912907a02 100644
--- a/yt_dlp/compat/_legacy.py
+++ b/yt_dlp/compat/_legacy.py
@@ -70,6 +70,7 @@ def compat_setenv(key, value, env=os.environ):
 compat_HTMLParser = compat_html_parser_HTMLParser = html.parser.HTMLParser
 compat_http_client = http.client
 compat_http_server = http.server
+compat_HTTPError = urllib.error.HTTPError
 compat_input = input
 compat_integer_types = (int, )
 compat_itertools_count = itertools.count
diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index d4045e58f..e307502db 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -10,6 +10,7 @@
 
 from .fragment import FragmentFD
 from ..compat import functools
+from ..networking import Request
 from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor
 from ..utils import (
     Popen,
@@ -25,7 +26,6 @@
     encodeFilename,
     find_available_port,
     remove_end,
-    sanitized_Request,
     traverse_obj,
 )
 
@@ -357,13 +357,12 @@ def aria2c_rpc(self, rpc_port, rpc_secret, method, params=()):
             'method': method,
             'params': [f'token:{rpc_secret}', *params],
         }).encode('utf-8')
-        request = sanitized_Request(
+        request = Request(
             f'http://localhost:{rpc_port}/jsonrpc',
             data=d, headers={
                 'Content-Type': 'application/json',
                 'Content-Length': f'{len(d)}',
-                'Ytdl-request-proxy': '__noproxy__',
-            })
+            }, proxies={'all': None})
         with self.ydl.urlopen(request) as r:
             resp = json.load(r)
         assert resp.get('id') == sanitycheck, 'Something went wrong with RPC server'
diff --git a/yt_dlp/downloader/f4m.py b/yt_dlp/downloader/f4m.py
index 306f92192..28cbba016 100644
--- a/yt_dlp/downloader/f4m.py
+++ b/yt_dlp/downloader/f4m.py
@@ -3,11 +3,11 @@
 import itertools
 import struct
 import time
-import urllib.error
 import urllib.parse
 
 from .fragment import FragmentFD
 from ..compat import compat_etree_fromstring
+from ..networking.exceptions import HTTPError
 from ..utils import fix_xml_ampersands, xpath_text
 
 
@@ -312,7 +312,7 @@ def real_download(self, filename, info_dict):
         self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
 
         urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
-        man_url = urlh.geturl()
+        man_url = urlh.url
         # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
         # (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244
         # and https://github.com/ytdl-org/youtube-dl/issues/7823)
@@ -407,8 +407,8 @@ def real_download(self, filename, info_dict):
                     if box_type == b'mdat':
                         self._append_fragment(ctx, box_data)
                         break
-            except urllib.error.HTTPError as err:
-                if live and (err.code == 404 or err.code == 410):
+            except HTTPError as err:
+                if live and (err.status == 404 or err.status == 410):
                     # We didn't keep up with the live window. Continue
                     # with the next available fragment.
                     msg = 'Fragment %d unavailable' % frag_i
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 069815326..b4b680dae 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -1,24 +1,19 @@
 import concurrent.futures
 import contextlib
-import http.client
 import json
 import math
 import os
 import struct
 import time
-import urllib.error
 
 from .common import FileDownloader
 from .http import HttpFD
 from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
 from ..compat import compat_os_name
-from ..utils import (
-    DownloadError,
-    RetryManager,
-    encodeFilename,
-    sanitized_Request,
-    traverse_obj,
-)
+from ..networking import Request
+from ..networking.exceptions import HTTPError, IncompleteRead
+from ..utils import DownloadError, RetryManager, encodeFilename, traverse_obj
+from ..utils.networking import HTTPHeaderDict
 
 
 class HttpQuietDownloader(HttpFD):
@@ -75,7 +70,7 @@ def report_skip_fragment(self, frag_index, err=None):
 
     def _prepare_url(self, info_dict, url):
         headers = info_dict.get('http_headers')
-        return sanitized_Request(url, None, headers) if headers else url
+        return Request(url, None, headers) if headers else url
 
     def _prepare_and_start_frag_download(self, ctx, info_dict):
         self._prepare_frag_download(ctx)
@@ -457,7 +452,7 @@ def download_fragment(fragment, ctx):
 
             frag_index = ctx['fragment_index'] = fragment['frag_index']
             ctx['last_error'] = None
-            headers = info_dict.get('http_headers', {}).copy()
+            headers = HTTPHeaderDict(info_dict.get('http_headers'))
             byte_range = fragment.get('byte_range')
             if byte_range:
                 headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
@@ -477,7 +472,7 @@ def error_callback(err, count, retries):
                     if not self._download_fragment(
                             ctx, fragment['url'], info_dict, headers, info_dict.get('request_data')):
                         return
-                except (urllib.error.HTTPError, http.client.IncompleteRead) as err:
+                except (HTTPError, IncompleteRead) as err:
                     retry.error = err
                     continue
                 except DownloadError:  # has own retry settings
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index ab7d496d4..d4b3f0320 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -75,7 +75,7 @@ def real_download(self, filename, info_dict):
         self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
 
         urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
-        man_url = urlh.geturl()
+        man_url = urlh.url
         s = urlh.read().decode('utf-8', 'ignore')
 
         can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index 45d094721..f5237443e 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -1,10 +1,14 @@
 import os
 import random
 import time
-import urllib.error
 
 from .common import FileDownloader
-from ..networking.exceptions import CertificateVerifyError, TransportError
+from ..networking import Request
+from ..networking.exceptions import (
+    CertificateVerifyError,
+    HTTPError,
+    TransportError,
+)
 from ..utils import (
     ContentTooShortError,
     RetryManager,
@@ -14,10 +18,10 @@
     encodeFilename,
     int_or_none,
     parse_http_range,
-    sanitized_Request,
     try_call,
     write_xattr,
 )
+from ..utils.networking import HTTPHeaderDict
 
 
 class HttpFD(FileDownloader):
@@ -36,10 +40,7 @@ class DownloadContext(dict):
         ctx.stream = None
 
         # Disable compression
-        headers = {'Accept-Encoding': 'identity'}
-        add_headers = info_dict.get('http_headers')
-        if add_headers:
-            headers.update(add_headers)
+        headers = HTTPHeaderDict({'Accept-Encoding': 'identity'}, info_dict.get('http_headers'))
 
         is_test = self.params.get('test', False)
         chunk_size = self._TEST_FILE_SIZE if is_test else (
@@ -110,10 +111,10 @@ def establish_connection():
             if try_call(lambda: range_end >= ctx.content_len):
                 range_end = ctx.content_len - 1
 
-            request = sanitized_Request(url, request_data, headers)
+            request = Request(url, request_data, headers)
             has_range = range_start is not None
             if has_range:
-                request.add_header('Range', f'bytes={int(range_start)}-{int_or_none(range_end) or ""}')
+                request.headers['Range'] = f'bytes={int(range_start)}-{int_or_none(range_end) or ""}'
             # Establish connection
             try:
                 ctx.data = self.ydl.urlopen(request)
@@ -144,17 +145,17 @@ def establish_connection():
                         self.report_unable_to_resume()
                     ctx.resume_len = 0
                     ctx.open_mode = 'wb'
-                ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
-            except urllib.error.HTTPError as err:
-                if err.code == 416:
+                ctx.data_len = ctx.content_len = int_or_none(ctx.data.headers.get('Content-length', None))
+            except HTTPError as err:
+                if err.status == 416:
                     # Unable to resume (requested range not satisfiable)
                     try:
                         # Open the connection again without the range header
                         ctx.data = self.ydl.urlopen(
-                            sanitized_Request(url, request_data, headers))
-                        content_length = ctx.data.info()['Content-Length']
-                    except urllib.error.HTTPError as err:
-                        if err.code < 500 or err.code >= 600:
+                            Request(url, request_data, headers))
+                        content_length = ctx.data.headers['Content-Length']
+                    except HTTPError as err:
+                        if err.status < 500 or err.status >= 600:
                             raise
                     else:
                         # Examine the reported length
@@ -182,7 +183,7 @@ def establish_connection():
                             ctx.resume_len = 0
                             ctx.open_mode = 'wb'
                             return
-                elif err.code < 500 or err.code >= 600:
+                elif err.status < 500 or err.status >= 600:
                     # Unexpected HTTP error
                     raise
                 raise RetryDownload(err)
@@ -198,9 +199,9 @@ def close_stream():
                 ctx.stream = None
 
         def download():
-            data_len = ctx.data.info().get('Content-length')
+            data_len = ctx.data.headers.get('Content-length')
 
-            if ctx.data.info().get('Content-encoding'):
+            if ctx.data.headers.get('Content-encoding'):
                 # Content-encoding is present, Content-length is not reliable anymore as we are
                 # doing auto decompression. (See: https://github.com/yt-dlp/yt-dlp/pull/6176)
                 data_len = None
@@ -345,7 +346,7 @@ def retry(e):
 
             # Update file modification time
             if self.params.get('updatetime', True):
-                info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))
+                info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.headers.get('last-modified', None))
 
             self._hook_progress({
                 'downloaded_bytes': byte_counter,
diff --git a/yt_dlp/downloader/ism.py b/yt_dlp/downloader/ism.py
index a157a8ad9..dd688f586 100644
--- a/yt_dlp/downloader/ism.py
+++ b/yt_dlp/downloader/ism.py
@@ -2,9 +2,9 @@
 import io
 import struct
 import time
-import urllib.error
 
 from .fragment import FragmentFD
+from ..networking.exceptions import HTTPError
 from ..utils import RetryManager
 
 u8 = struct.Struct('>B')
@@ -271,7 +271,7 @@ def real_download(self, filename, info_dict):
                         write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
                         extra_state['ism_track_written'] = True
                     self._append_fragment(ctx, frag_content)
-                except urllib.error.HTTPError as err:
+                except HTTPError as err:
                     retry.error = err
                     continue
 
diff --git a/yt_dlp/downloader/niconico.py b/yt_dlp/downloader/niconico.py
index 7d8575c2a..5720f6eb8 100644
--- a/yt_dlp/downloader/niconico.py
+++ b/yt_dlp/downloader/niconico.py
@@ -5,13 +5,8 @@
 from . import get_suitable_downloader
 from .common import FileDownloader
 from .external import FFmpegFD
-from ..utils import (
-    DownloadError,
-    WebSocketsWrapper,
-    sanitized_Request,
-    str_or_none,
-    try_get,
-)
+from ..networking import Request
+from ..utils import DownloadError, WebSocketsWrapper, str_or_none, try_get
 
 
 class NiconicoDmcFD(FileDownloader):
@@ -33,7 +28,7 @@ def real_download(self, filename, info_dict):
         heartbeat_data = heartbeat_info_dict['data'].encode()
         heartbeat_interval = heartbeat_info_dict.get('interval', 30)
 
-        request = sanitized_Request(heartbeat_url, heartbeat_data)
+        request = Request(heartbeat_url, heartbeat_data)
 
         def heartbeat():
             try:
diff --git a/yt_dlp/downloader/youtube_live_chat.py b/yt_dlp/downloader/youtube_live_chat.py
index 5928fecf0..c7a86374a 100644
--- a/yt_dlp/downloader/youtube_live_chat.py
+++ b/yt_dlp/downloader/youtube_live_chat.py
@@ -1,8 +1,8 @@
 import json
 import time
-import urllib.error
 
 from .fragment import FragmentFD
+from ..networking.exceptions import HTTPError
 from ..utils import (
     RegexNotFoundError,
     RetryManager,
@@ -10,6 +10,7 @@
     int_or_none,
     try_get,
 )
+from ..utils.networking import HTTPHeaderDict
 
 
 class YoutubeLiveChatFD(FragmentFD):
@@ -37,10 +38,7 @@ def real_download(self, filename, info_dict):
         start_time = int(time.time() * 1000)
 
         def dl_fragment(url, data=None, headers=None):
-            http_headers = info_dict.get('http_headers', {})
-            if headers:
-                http_headers = http_headers.copy()
-                http_headers.update(headers)
+            http_headers = HTTPHeaderDict(info_dict.get('http_headers'), headers)
             return self._download_fragment(ctx, url, info_dict, http_headers, data)
 
         def parse_actions_replay(live_chat_continuation):
@@ -129,7 +127,7 @@ def download_and_parse_fragment(url, frag_index, request_data=None, headers=None
                             or frag_index == 1 and try_refresh_replay_beginning
                             or parse_actions_replay)
                     return (True, *func(live_chat_continuation))
-                except urllib.error.HTTPError as err:
+                except HTTPError as err:
                     retry.error = err
                     continue
             return False, None, None, None
diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index c9166b6b8..98ece8da7 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -22,7 +22,6 @@
     int_or_none,
     intlist_to_bytes,
     OnDemandPagedList,
-    request_to_url,
     time_seconds,
     traverse_obj,
     update_url_query,
@@ -137,7 +136,7 @@ def _get_videokey_from_ticket(self, ticket):
         return intlist_to_bytes(aes_ecb_decrypt(encvideokey, enckey))
 
     def abematv_license_open(self, url):
-        url = request_to_url(url)
+        url = url.get_full_url() if isinstance(url, urllib.request.Request) else url
         ticket = urllib.parse.urlparse(url).netloc
         response_data = self._get_videokey_from_ticket(ticket)
         return urllib.response.addinfourl(io.BytesIO(response_data), headers={
diff --git a/yt_dlp/extractor/adn.py b/yt_dlp/extractor/adn.py
index f1f55e87f..b59dbc850 100644
--- a/yt_dlp/extractor/adn.py
+++ b/yt_dlp/extractor/adn.py
@@ -6,10 +6,8 @@
 
 from .common import InfoExtractor
 from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
-from ..compat import (
-    compat_HTTPError,
-    compat_b64decode,
-)
+from ..compat import compat_b64decode
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ass_subtitles_timecode,
     bytes_to_intlist,
@@ -142,9 +140,9 @@ def _perform_login(self, username, password):
                 self._HEADERS = {'authorization': 'Bearer ' + access_token}
         except ExtractorError as e:
             message = None
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 resp = self._parse_json(
-                    e.cause.read().decode(), None, fatal=False) or {}
+                    e.cause.response.read().decode(), None, fatal=False) or {}
                 message = resp.get('message') or resp.get('code')
             self.report_warning(message or self._LOGIN_ERR_MESSAGE)
 
@@ -195,14 +193,14 @@ def _real_extract(self, url):
                     })
                 break
             except ExtractorError as e:
-                if not isinstance(e.cause, compat_HTTPError):
+                if not isinstance(e.cause, HTTPError):
                     raise e
 
-                if e.cause.code == 401:
+                if e.cause.status == 401:
                     # This usually goes away with a different random pkcs1pad, so retry
                     continue
 
-                error = self._parse_json(e.cause.read(), video_id)
+                error = self._parse_json(e.cause.response.read(), video_id)
                 message = error.get('message')
                 if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
                     self.raise_geo_restricted(msg=message)
diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py
index 722a534ed..5eed0ca22 100644
--- a/yt_dlp/extractor/adobepass.py
+++ b/yt_dlp/extractor/adobepass.py
@@ -2,11 +2,11 @@
 import json
 import re
 import time
-import urllib.error
 import xml.etree.ElementTree as etree
 
 from .common import InfoExtractor
 from ..compat import compat_urlparse
+from ..networking.exceptions import HTTPError
 from ..utils import (
     NO_DEFAULT,
     ExtractorError,
@@ -1394,7 +1394,7 @@ def post_form(form_page_res, note, data={}):
             form_page, urlh = form_page_res
             post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
             if not re.match(r'https?://', post_url):
-                post_url = compat_urlparse.urljoin(urlh.geturl(), post_url)
+                post_url = compat_urlparse.urljoin(urlh.url, post_url)
             form_data = self._hidden_inputs(form_page)
             form_data.update(data)
             return self._download_webpage_handle(
@@ -1619,7 +1619,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                     hidden_data['history'] = 1
 
                     provider_login_page_res = self._download_webpage_handle(
-                        urlh.geturl(), video_id, 'Sending first bookend',
+                        urlh.url, video_id, 'Sending first bookend',
                         query=hidden_data)
 
                     provider_association_redirect, urlh = post_form(
@@ -1629,7 +1629,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                         })
 
                     provider_refresh_redirect_url = extract_redirect_url(
-                        provider_association_redirect, url=urlh.geturl())
+                        provider_association_redirect, url=urlh.url)
 
                     last_bookend_page, urlh = self._download_webpage_handle(
                         provider_refresh_redirect_url, video_id,
@@ -1638,7 +1638,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                     hidden_data['history'] = 3
 
                     mvpd_confirm_page_res = self._download_webpage_handle(
-                        urlh.geturl(), video_id, 'Sending final bookend',
+                        urlh.url, video_id, 'Sending final bookend',
                         query=hidden_data)
 
                     post_form(mvpd_confirm_page_res, 'Confirming Login')
@@ -1652,7 +1652,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                     hidden_data['history_val'] = 1
 
                     provider_login_redirect_page_res = self._download_webpage_handle(
-                        urlh.geturl(), video_id, 'Sending First Bookend',
+                        urlh.url, video_id, 'Sending First Bookend',
                         query=hidden_data)
 
                     provider_login_redirect_page, urlh = provider_login_redirect_page_res
@@ -1680,7 +1680,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                         })
 
                     provider_refresh_redirect_url = extract_redirect_url(
-                        provider_association_redirect, url=urlh.geturl())
+                        provider_association_redirect, url=urlh.url)
 
                     last_bookend_page, urlh = self._download_webpage_handle(
                         provider_refresh_redirect_url, video_id,
@@ -1690,7 +1690,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                     hidden_data['history_val'] = 3
 
                     mvpd_confirm_page_res = self._download_webpage_handle(
-                        urlh.geturl(), video_id, 'Sending Final Bookend',
+                        urlh.url, video_id, 'Sending Final Bookend',
                         query=hidden_data)
 
                     post_form(mvpd_confirm_page_res, 'Confirming Login')
@@ -1699,7 +1699,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                     # based redirect that should be followed.
                     provider_redirect_page, urlh = provider_redirect_page_res
                     provider_refresh_redirect_url = extract_redirect_url(
-                        provider_redirect_page, url=urlh.geturl())
+                        provider_redirect_page, url=urlh.url)
                     if provider_refresh_redirect_url:
                         provider_redirect_page_res = self._download_webpage_handle(
                             provider_refresh_redirect_url, video_id,
@@ -1724,7 +1724,7 @@ def extract_redirect_url(html, url=None, fatal=False):
                             'requestor_id': requestor_id,
                         }), headers=mvpd_headers)
                 except ExtractorError as e:
-                    if not mso_id and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                    if not mso_id and isinstance(e.cause, HTTPError) and e.cause.status == 401:
                         raise_mvpd_required()
                     raise
                 if '<pendingLogout' in session:
diff --git a/yt_dlp/extractor/ant1newsgr.py b/yt_dlp/extractor/ant1newsgr.py
index 7b384b22d..217e3acc4 100644
--- a/yt_dlp/extractor/ant1newsgr.py
+++ b/yt_dlp/extractor/ant1newsgr.py
@@ -1,8 +1,8 @@
 import urllib.parse
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     ExtractorError,
     determine_ext,
     scale_thumbnails_to_max_format_width,
@@ -121,7 +121,7 @@ def _real_extract(self, url):
         canonical_url = self._request_webpage(
             HEADRequest(url), video_id,
             note='Resolve canonical player URL',
-            errnote='Could not resolve canonical player URL').geturl()
+            errnote='Could not resolve canonical player URL').url
         _, netloc, _, _, query, _ = urllib.parse.urlparse(canonical_url)
         cid = urllib.parse.parse_qs(query)['cid'][0]
 
diff --git a/yt_dlp/extractor/archiveorg.py b/yt_dlp/extractor/archiveorg.py
index 4ccd39825..2541cd6fd 100644
--- a/yt_dlp/extractor/archiveorg.py
+++ b/yt_dlp/extractor/archiveorg.py
@@ -1,16 +1,16 @@
 import json
 import re
-import urllib.error
 import urllib.parse
 
 from .common import InfoExtractor
 from .naver import NaverBaseIE
 from .youtube import YoutubeBaseInfoExtractor, YoutubeIE
-from ..compat import compat_HTTPError, compat_urllib_parse_unquote
+from ..compat import compat_urllib_parse_unquote
+from ..networking import HEADRequest
+from ..networking.exceptions import HTTPError
 from ..utils import (
     KNOWN_EXTENSIONS,
     ExtractorError,
-    HEADRequest,
     bug_reports_message,
     clean_html,
     dict_get,
@@ -899,7 +899,7 @@ def _real_extract(self, url):
                     video_id, note='Fetching archived video file url', expected_status=True)
             except ExtractorError as e:
                 # HTTP Error 404 is expected if the video is not saved.
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                     self.raise_no_formats(
                         'The requested video is not archived, indexed, or there is an issue with web.archive.org (try again later)', expected=True)
                 else:
@@ -926,7 +926,7 @@ def _real_extract(self, url):
         info['thumbnails'] = self._extract_thumbnails(video_id)
 
         if urlh:
-            url = compat_urllib_parse_unquote(urlh.geturl())
+            url = compat_urllib_parse_unquote(urlh.url)
             video_file_url_qs = parse_qs(url)
             # Attempt to recover any ext & format info from playback url & response headers
             format = {'url': url, 'filesize': int_or_none(urlh.headers.get('x-archive-orig-content-length'))}
@@ -1052,7 +1052,7 @@ def _download_archived_page(self, url, video_id, *, timestamp='2', **kwargs):
             try:
                 return self._download_webpage(f'https://web.archive.org/web/{timestamp}id_/{url}', video_id, **kwargs)
             except ExtractorError as e:
-                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                     raise ExtractorError('Page was not archived', expected=True)
                 retry.error = e
                 continue
diff --git a/yt_dlp/extractor/atresplayer.py b/yt_dlp/extractor/atresplayer.py
index a20e7f988..3a44e5265 100644
--- a/yt_dlp/extractor/atresplayer.py
+++ b/yt_dlp/extractor/atresplayer.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -34,8 +34,8 @@ class AtresPlayerIE(InfoExtractor):
     _API_BASE = 'https://api.atresplayer.com/'
 
     def _handle_error(self, e, code):
-        if isinstance(e.cause, compat_HTTPError) and e.cause.code == code:
-            error = self._parse_json(e.cause.read(), None)
+        if isinstance(e.cause, HTTPError) and e.cause.status == code:
+            error = self._parse_json(e.cause.response.read(), None)
             if error.get('error') == 'required_registered':
                 self.raise_login_required()
             raise ExtractorError(error['error_description'], expected=True)
diff --git a/yt_dlp/extractor/bbc.py b/yt_dlp/extractor/bbc.py
index 9d28e70a3..a55cdef2b 100644
--- a/yt_dlp/extractor/bbc.py
+++ b/yt_dlp/extractor/bbc.py
@@ -2,11 +2,11 @@
 import itertools
 import json
 import re
-import urllib.error
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str, compat_urlparse
+from ..compat import compat_str, compat_urlparse
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
@@ -277,7 +277,7 @@ def _perform_login(self, username, password):
             post_url, None, 'Logging in', data=urlencode_postdata(login_form),
             headers={'Referer': self._LOGIN_URL})
 
-        if self._LOGIN_URL in urlh.geturl():
+        if self._LOGIN_URL in urlh.url:
             error = clean_html(get_element_by_class('form-message', response))
             if error:
                 raise ExtractorError(
@@ -388,8 +388,8 @@ def _process_media_selector(self, media_selection, programme_id):
                                 href, programme_id, ext='mp4', entry_protocol='m3u8_native',
                                 m3u8_id=format_id, fatal=False)
                         except ExtractorError as e:
-                            if not (isinstance(e.exc_info[1], urllib.error.HTTPError)
-                                    and e.exc_info[1].code in (403, 404)):
+                            if not (isinstance(e.exc_info[1], HTTPError)
+                                    and e.exc_info[1].status in (403, 404)):
                                 raise
                             fmts = []
                         formats.extend(fmts)
@@ -472,7 +472,7 @@ def _download_playlist(self, playlist_id):
 
             return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:
-            if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
+            if not (isinstance(ee.cause, HTTPError) and ee.cause.status == 404):
                 raise
 
         # fallback to legacy playlist
@@ -983,7 +983,7 @@ def _real_extract(self, url):
                                     # Some playlist URL may fail with 500, at the same time
                                     # the other one may work fine (e.g.
                                     # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
-                                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 500:
+                                    if isinstance(e.cause, HTTPError) and e.cause.status == 500:
                                         continue
                                     raise
                             if entry:
diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index e8714a33a..cb7ab2a17 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -4,11 +4,11 @@
 import itertools
 import math
 import time
-import urllib.error
 import urllib.parse
 
 from .common import InfoExtractor, SearchInfoExtractor
 from ..dependencies import Cryptodome
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     GeoRestrictedError,
@@ -614,7 +614,7 @@ def fetch_page(page_idx):
                 response = self._download_json('https://api.bilibili.com/x/space/wbi/arc/search',
                                                playlist_id, note=f'Downloading page {page_idx}', query=query)
             except ExtractorError as e:
-                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 412:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 412:
                     raise ExtractorError(
                         'Request is blocked by server (412), please add cookies, wait and try later.', expected=True)
                 raise
diff --git a/yt_dlp/extractor/bitchute.py b/yt_dlp/extractor/bitchute.py
index a6779505e..0805b8b46 100644
--- a/yt_dlp/extractor/bitchute.py
+++ b/yt_dlp/extractor/bitchute.py
@@ -2,9 +2,9 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     OnDemandPagedList,
     clean_html,
     get_element_by_class,
diff --git a/yt_dlp/extractor/bravotv.py b/yt_dlp/extractor/bravotv.py
index 13cc1927f..419fe8c9c 100644
--- a/yt_dlp/extractor/bravotv.py
+++ b/yt_dlp/extractor/bravotv.py
@@ -1,6 +1,6 @@
 from .adobepass import AdobePassIE
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     extract_attributes,
     float_or_none,
     get_element_html_by_class,
@@ -155,7 +155,7 @@ def _real_extract(self, url):
             chapters = None
 
         m3u8_url = self._request_webpage(HEADRequest(
-            update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').geturl()
+            update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').url
         if 'mpeg_cenc' in m3u8_url:
             self.report_drm(video_id)
         formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
diff --git a/yt_dlp/extractor/brightcove.py b/yt_dlp/extractor/brightcove.py
index cd0e8ff27..61b18412d 100644
--- a/yt_dlp/extractor/brightcove.py
+++ b/yt_dlp/extractor/brightcove.py
@@ -7,10 +7,10 @@
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_fromstring,
-    compat_HTTPError,
     compat_parse_qs,
     compat_urlparse,
 )
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     dict_get,
@@ -915,8 +915,8 @@ def extract_policy_key():
                 json_data = self._download_json(api_url, video_id, headers=headers)
                 break
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
-                    json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
+                if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
+                    json_data = self._parse_json(e.cause.response.read().decode(), video_id)[0]
                     message = json_data.get('message') or json_data['error_code']
                     if json_data.get('error_subcode') == 'CLIENT_GEO':
                         self.raise_geo_restricted(msg=message)
diff --git a/yt_dlp/extractor/canalplus.py b/yt_dlp/extractor/canalplus.py
index b7e2f9dd4..3ff5c3fbf 100644
--- a/yt_dlp/extractor/canalplus.py
+++ b/yt_dlp/extractor/canalplus.py
@@ -64,7 +64,7 @@ def _real_extract(self, url):
         #     response = self._request_webpage(
         #         HEADRequest(fmt_url), video_id,
         #         'Checking if the video is georestricted')
-        #     if '/blocage' in response.geturl():
+        #     if '/blocage' in response.url:
         #         raise ExtractorError(
         #             'The video is not available in your country',
         #             expected=True)
diff --git a/yt_dlp/extractor/cbsnews.py b/yt_dlp/extractor/cbsnews.py
index 65ecc62f0..5a8ebb847 100644
--- a/yt_dlp/extractor/cbsnews.py
+++ b/yt_dlp/extractor/cbsnews.py
@@ -7,9 +7,9 @@
 from .anvato import AnvatoIE
 from .common import InfoExtractor
 from .paramountplus import ParamountPlusIE
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     UserNotLive,
     determine_ext,
     float_or_none,
diff --git a/yt_dlp/extractor/ceskatelevize.py b/yt_dlp/extractor/ceskatelevize.py
index be2b0bb43..8390160a0 100644
--- a/yt_dlp/extractor/ceskatelevize.py
+++ b/yt_dlp/extractor/ceskatelevize.py
@@ -1,20 +1,20 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_urlparse,
-)
+from ..compat import compat_urllib_parse_unquote, compat_urllib_parse_urlparse
+from ..networking import Request
 from ..utils import (
     ExtractorError,
     float_or_none,
-    sanitized_Request,
     str_or_none,
     traverse_obj,
     urlencode_postdata,
-    USER_AGENTS,
 )
 
+USER_AGENTS = {
+    'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
+}
+
 
 class CeskaTelevizeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(?:ivysilani|porady|zive)/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
@@ -97,7 +97,7 @@ class CeskaTelevizeIE(InfoExtractor):
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
         webpage, urlh = self._download_webpage_handle(url, playlist_id)
-        parsed_url = compat_urllib_parse_urlparse(urlh.geturl())
+        parsed_url = compat_urllib_parse_urlparse(urlh.url)
         site_name = self._og_search_property('site_name', webpage, fatal=False, default='Česká televize')
         playlist_title = self._og_search_title(webpage, default=None)
         if site_name and playlist_title:
@@ -163,16 +163,16 @@ def _real_extract(self, url):
         entries = []
 
         for user_agent in (None, USER_AGENTS['Safari']):
-            req = sanitized_Request(
+            req = Request(
                 'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist/',
                 data=urlencode_postdata(data))
 
-            req.add_header('Content-type', 'application/x-www-form-urlencoded')
-            req.add_header('x-addr', '127.0.0.1')
-            req.add_header('X-Requested-With', 'XMLHttpRequest')
+            req.headers['Content-type'] = 'application/x-www-form-urlencoded'
+            req.headers['x-addr'] = '127.0.0.1'
+            req.headers['X-Requested-With'] = 'XMLHttpRequest'
             if user_agent:
-                req.add_header('User-Agent', user_agent)
-            req.add_header('Referer', url)
+                req.headers['User-Agent'] = user_agent
+            req.headers['Referer'] = url
 
             playlistpage = self._download_json(req, playlist_id, fatal=False)
 
@@ -183,8 +183,8 @@ def _real_extract(self, url):
             if playlist_url == 'error_region':
                 raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-            req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
-            req.add_header('Referer', url)
+            req = Request(compat_urllib_parse_unquote(playlist_url))
+            req.headers['Referer'] = url
 
             playlist = self._download_json(req, playlist_id, fatal=False)
             if not playlist:
diff --git a/yt_dlp/extractor/cinetecamilano.py b/yt_dlp/extractor/cinetecamilano.py
index 5e770ebac..9cffa11e8 100644
--- a/yt_dlp/extractor/cinetecamilano.py
+++ b/yt_dlp/extractor/cinetecamilano.py
@@ -1,6 +1,6 @@
 import json
-import urllib.error
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -40,7 +40,7 @@ def _real_extract(self, url):
                     'Authorization': try_get(self._get_cookies('https://www.cinetecamilano.it'), lambda x: f'Bearer {x["cnt-token"].value}') or ''
                 })
         except ExtractorError as e:
-            if ((isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 500)
+            if ((isinstance(e.cause, HTTPError) and e.cause.status == 500)
                     or isinstance(e.cause, json.JSONDecodeError)):
                 self.raise_login_required(method='cookies')
             raise
diff --git a/yt_dlp/extractor/ciscowebex.py b/yt_dlp/extractor/ciscowebex.py
index 40430505d..85585dffb 100644
--- a/yt_dlp/extractor/ciscowebex.py
+++ b/yt_dlp/extractor/ciscowebex.py
@@ -33,7 +33,7 @@ def _real_extract(self, url):
         if rcid:
             webpage = self._download_webpage(url, None, note='Getting video ID')
             url = self._search_regex(self._VALID_URL, webpage, 'redirection url', group='url')
-        url = self._request_webpage(url, None, note='Resolving final URL').geturl()
+        url = self._request_webpage(url, None, note='Resolving final URL').url
         mobj = self._match_valid_url(url)
         subdomain = mobj.group('subdomain')
         siteurl = mobj.group('siteurl_1') or mobj.group('siteurl_2')
@@ -49,7 +49,7 @@ def _real_extract(self, url):
             'https://%s.webex.com/webappng/api/v1/recordings/%s/stream' % (subdomain, video_id),
             video_id, headers=headers, query={'siteurl': siteurl}, expected_status=(403, 429))
 
-        if urlh.getcode() == 403:
+        if urlh.status == 403:
             if stream['code'] == 53004:
                 self.raise_login_required()
             if stream['code'] == 53005:
@@ -59,7 +59,7 @@ def _real_extract(self, url):
                     'This video is protected by a password, use the --video-password option', expected=True)
             raise ExtractorError(f'{self.IE_NAME} said: {stream["code"]} - {stream["message"]}', expected=True)
 
-        if urlh.getcode() == 429:
+        if urlh.status == 429:
             self.raise_login_required(
                 f'{self.IE_NAME} asks you to solve a CAPTCHA. Solve CAPTCHA in browser and',
                 method='cookies')
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 63156d3ac..d44918776 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -31,8 +31,12 @@
 from ..cookies import LenientSimpleCookie
 from ..downloader.f4m import get_base_url, remove_encrypted_media
 from ..downloader.hls import HlsFD
-from ..networking.common import HEADRequest, Request
-from ..networking.exceptions import network_exceptions
+from ..networking import HEADRequest, Request
+from ..networking.exceptions import (
+    HTTPError,
+    IncompleteRead,
+    network_exceptions,
+)
 from ..utils import (
     IDENTITY,
     JSON_LD_RE,
@@ -729,7 +733,7 @@ def extract(self, url):
             e.ie = e.ie or self.IE_NAME,
             e.traceback = e.traceback or sys.exc_info()[2]
             raise
-        except http.client.IncompleteRead as e:
+        except IncompleteRead as e:
             raise ExtractorError('A network error has occurred.', cause=e, expected=True, video_id=self.get_temp_id(url))
         except (KeyError, StopIteration) as e:
             raise ExtractorError('An extractor error has occurred.', cause=e, video_id=self.get_temp_id(url))
@@ -788,16 +792,19 @@ def IE_NAME(cls):
 
     @staticmethod
     def __can_accept_status_code(err, expected_status):
-        assert isinstance(err, urllib.error.HTTPError)
+        assert isinstance(err, HTTPError)
         if expected_status is None:
             return False
         elif callable(expected_status):
-            return expected_status(err.code) is True
+            return expected_status(err.status) is True
         else:
-            return err.code in variadic(expected_status)
+            return err.status in variadic(expected_status)
 
     def _create_request(self, url_or_request, data=None, headers=None, query=None):
         if isinstance(url_or_request, urllib.request.Request):
+            self._downloader.deprecation_warning(
+                'Passing a urllib.request.Request to _create_request() is deprecated. '
+                'Use yt_dlp.networking.common.Request instead.')
             url_or_request = urllib_req_to_req(url_or_request)
         elif not isinstance(url_or_request, Request):
             url_or_request = Request(url_or_request)
@@ -839,7 +846,7 @@ def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fa
         try:
             return self._downloader.urlopen(self._create_request(url_or_request, data, headers, query))
         except network_exceptions as err:
-            if isinstance(err, urllib.error.HTTPError):
+            if isinstance(err, HTTPError):
                 if self.__can_accept_status_code(err, expected_status):
                     return err.response
 
@@ -973,11 +980,11 @@ def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errno
         if prefix is not None:
             webpage_bytes = prefix + webpage_bytes
         if self.get_param('dump_intermediate_pages', False):
-            self.to_screen('Dumping request to ' + urlh.geturl())
+            self.to_screen('Dumping request to ' + urlh.url)
             dump = base64.b64encode(webpage_bytes).decode('ascii')
             self._downloader.to_screen(dump)
         if self.get_param('write_pages'):
-            filename = self._request_dump_filename(urlh.geturl(), video_id)
+            filename = self._request_dump_filename(urlh.url, video_id)
             self.to_screen(f'Saving request to {filename}')
             with open(filename, 'wb') as outf:
                 outf.write(webpage_bytes)
@@ -1109,7 +1116,7 @@ def _download_webpage(
         while True:
             try:
                 return self.__download_webpage(url_or_request, video_id, note, errnote, None, fatal, *args, **kwargs)
-            except http.client.IncompleteRead as e:
+            except IncompleteRead as e:
                 try_count += 1
                 if try_count >= tries:
                     raise e
@@ -1806,7 +1813,7 @@ def _extract_f4m_formats(self, manifest_url, video_id, preference=None, quality=
             return []
 
         manifest, urlh = res
-        manifest_url = urlh.geturl()
+        manifest_url = urlh.url
 
         return self._parse_f4m_formats(
             manifest, manifest_url, video_id, preference=preference, quality=quality, f4m_id=f4m_id,
@@ -1965,7 +1972,7 @@ def _extract_m3u8_formats_and_subtitles(
             return [], {}
 
         m3u8_doc, urlh = res
-        m3u8_url = urlh.geturl()
+        m3u8_url = urlh.url
 
         return self._parse_m3u8_formats_and_subtitles(
             m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
@@ -2243,7 +2250,7 @@ def _extract_smil_formats_and_subtitles(self, smil_url, video_id, fatal=True, f4
             return [], {}
 
         smil, urlh = res
-        smil_url = urlh.geturl()
+        smil_url = urlh.url
 
         namespace = self._parse_smil_namespace(smil)
 
@@ -2266,7 +2273,7 @@ def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
             return {}
 
         smil, urlh = res
-        smil_url = urlh.geturl()
+        smil_url = urlh.url
 
         return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
 
@@ -2458,7 +2465,7 @@ def _extract_xspf_playlist(self, xspf_url, playlist_id, fatal=True):
             return []
 
         xspf, urlh = res
-        xspf_url = urlh.geturl()
+        xspf_url = urlh.url
 
         return self._parse_xspf(
             xspf, playlist_id, xspf_url=xspf_url,
@@ -2529,7 +2536,7 @@ def _extract_mpd_formats_and_subtitles(
             return [], {}
 
         # We could have been redirected to a new url when we retrieved our mpd file.
-        mpd_url = urlh.geturl()
+        mpd_url = urlh.url
         mpd_base_url = base_url(mpd_url)
 
         return self._parse_mpd_formats_and_subtitles(
@@ -2900,7 +2907,7 @@ def _extract_ism_formats_and_subtitles(self, ism_url, video_id, ism_id=None, not
         if ism_doc is None:
             return [], {}
 
-        return self._parse_ism_formats_and_subtitles(ism_doc, urlh.geturl(), ism_id)
+        return self._parse_ism_formats_and_subtitles(ism_doc, urlh.url, ism_id)
 
     def _parse_ism_formats_and_subtitles(self, ism_doc, ism_url, ism_id=None):
         """
diff --git a/yt_dlp/extractor/crackle.py b/yt_dlp/extractor/crackle.py
index 46100151a..1ef90b5a0 100644
--- a/yt_dlp/extractor/crackle.py
+++ b/yt_dlp/extractor/crackle.py
@@ -4,7 +4,7 @@
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     float_or_none,
@@ -113,7 +113,7 @@ def _real_extract(self, url):
                     errnote='Unable to download media JSON')
             except ExtractorError as e:
                 # 401 means geo restriction, trying next country
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                     continue
                 raise
 
diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py
index 910504ed2..adb3d5dcf 100644
--- a/yt_dlp/extractor/crunchyroll.py
+++ b/yt_dlp/extractor/crunchyroll.py
@@ -1,7 +1,7 @@
 import base64
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -114,7 +114,7 @@ def _call_api(self, path, internal_id, lang, note='api', query={}):
             result = self._call_base_api(
                 path, internal_id, lang, f'Downloading {note} JSON ({self._API_ENDPOINT})', query=query)
         except ExtractorError as error:
-            if isinstance(error.cause, urllib.error.HTTPError) and error.cause.code == 404:
+            if isinstance(error.cause, HTTPError) and error.cause.status == 404:
                 return None
             raise
 
diff --git a/yt_dlp/extractor/cultureunplugged.py b/yt_dlp/extractor/cultureunplugged.py
index 2fb22800f..9c8509f1f 100644
--- a/yt_dlp/extractor/cultureunplugged.py
+++ b/yt_dlp/extractor/cultureunplugged.py
@@ -1,10 +1,8 @@
 import time
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    HEADRequest,
-)
+from ..networking import HEADRequest
+from ..utils import int_or_none
 
 
 class CultureUnpluggedIE(InfoExtractor):
diff --git a/yt_dlp/extractor/dacast.py b/yt_dlp/extractor/dacast.py
index cf683bad4..4e81aa4a7 100644
--- a/yt_dlp/extractor/dacast.py
+++ b/yt_dlp/extractor/dacast.py
@@ -1,9 +1,9 @@
 import hashlib
 import re
 import time
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     classproperty,
@@ -105,7 +105,7 @@ def _real_extract(self, url):
                 formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls')
             except ExtractorError as e:
                 # CDN will randomly respond with 403
-                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                     retry.error = e
                     continue
                 raise
diff --git a/yt_dlp/extractor/dailymotion.py b/yt_dlp/extractor/dailymotion.py
index 2a44718fb..21263d41b 100644
--- a/yt_dlp/extractor/dailymotion.py
+++ b/yt_dlp/extractor/dailymotion.py
@@ -3,7 +3,7 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
@@ -68,9 +68,9 @@ def _call_api(self, object_type, xid, object_fields, note, filter_extra=None):
                         None, 'Downloading Access Token',
                         data=urlencode_postdata(data))['access_token']
                 except ExtractorError as e:
-                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                    if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                         raise ExtractorError(self._parse_json(
-                            e.cause.read().decode(), xid)['error_description'], expected=True)
+                            e.cause.response.read().decode(), xid)['error_description'], expected=True)
                     raise
                 self._set_dailymotion_cookie('access_token' if username else 'client_token', token)
             self._HEADERS['Authorization'] = 'Bearer ' + token
diff --git a/yt_dlp/extractor/discovery.py b/yt_dlp/extractor/discovery.py
index e6e109d5c..75b464353 100644
--- a/yt_dlp/extractor/discovery.py
+++ b/yt_dlp/extractor/discovery.py
@@ -3,8 +3,8 @@
 
 from .discoverygo import DiscoveryGoBaseIE
 from ..compat import compat_urllib_parse_unquote
+from ..networking.exceptions import HTTPError
 from ..utils import ExtractorError
-from ..compat import compat_HTTPError
 
 
 class DiscoveryIE(DiscoveryGoBaseIE):
@@ -100,9 +100,9 @@ def _real_extract(self, url):
                 self._API_BASE_URL + 'streaming/video/' + video_id,
                 display_id, 'Downloading streaming JSON metadata', headers=headers)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
+            if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
                 e_description = self._parse_json(
-                    e.cause.read().decode(), display_id)['description']
+                    e.cause.response.read().decode(), display_id)['description']
                 if 'resource not available for country' in e_description:
                     self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
                 if 'Authorized Networks' in e_description:
diff --git a/yt_dlp/extractor/dplay.py b/yt_dlp/extractor/dplay.py
index cf6d14934..6404752f7 100644
--- a/yt_dlp/extractor/dplay.py
+++ b/yt_dlp/extractor/dplay.py
@@ -2,7 +2,7 @@
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -39,7 +39,7 @@ def _get_auth(self, disco_base, display_id, realm, needs_device_id=True):
         return f'Bearer {token}'
 
     def _process_errors(self, e, geo_countries):
-        info = self._parse_json(e.cause.read().decode('utf-8'), None)
+        info = self._parse_json(e.cause.response.read().decode('utf-8'), None)
         error = info['errors'][0]
         error_code = error.get('code')
         if error_code == 'access.denied.geoblocked':
@@ -87,7 +87,7 @@ def _get_disco_api_info(self, url, display_id, disco_host, realm, country, domai
                     'include': 'images,primaryChannel,show,tags'
                 })
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                 self._process_errors(e, geo_countries)
             raise
         video_id = video['data']['id']
@@ -99,7 +99,7 @@ def _get_disco_api_info(self, url, display_id, disco_host, realm, country, domai
             streaming = self._download_video_playback_info(
                 disco_base, video_id, headers)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 self._process_errors(e, geo_countries)
             raise
         for format_dict in streaming:
diff --git a/yt_dlp/extractor/eagleplatform.py b/yt_dlp/extractor/eagleplatform.py
index 9ebd24d80..739d17912 100644
--- a/yt_dlp/extractor/eagleplatform.py
+++ b/yt_dlp/extractor/eagleplatform.py
@@ -2,7 +2,7 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -111,8 +111,8 @@ def _download_json(self, url_or_request, video_id, *args, **kwargs):
             response = super(EaglePlatformIE, self)._download_json(
                 url_or_request, video_id, *args, **kwargs)
         except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError):
-                response = self._parse_json(ee.cause.read().decode('utf-8'), video_id)
+            if isinstance(ee.cause, HTTPError):
+                response = self._parse_json(ee.cause.response.read().decode('utf-8'), video_id)
                 self._handle_error(response)
             raise
         return response
diff --git a/yt_dlp/extractor/eitb.py b/yt_dlp/extractor/eitb.py
index bd027da6b..66afbb6bb 100644
--- a/yt_dlp/extractor/eitb.py
+++ b/yt_dlp/extractor/eitb.py
@@ -1,10 +1,6 @@
 from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    int_or_none,
-    parse_iso8601,
-    sanitized_Request,
-)
+from ..networking import Request
+from ..utils import float_or_none, int_or_none, parse_iso8601
 
 
 class EitbIE(InfoExtractor):
@@ -54,7 +50,7 @@ def _real_extract(self, url):
 
         hls_url = media.get('HLS_SURL')
         if hls_url:
-            request = sanitized_Request(
+            request = Request(
                 'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
                 headers={'Referer': url})
             token_data = self._download_json(
diff --git a/yt_dlp/extractor/eporner.py b/yt_dlp/extractor/eporner.py
index a2337979b..aee2dee58 100644
--- a/yt_dlp/extractor/eporner.py
+++ b/yt_dlp/extractor/eporner.py
@@ -52,7 +52,7 @@ def _real_extract(self, url):
 
         webpage, urlh = self._download_webpage_handle(url, display_id)
 
-        video_id = self._match_id(urlh.geturl())
+        video_id = self._match_id(urlh.url)
 
         hash = self._search_regex(
             r'hash\s*[:=]\s*["\']([\da-f]{32})', webpage, 'hash')
diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 9d871eb28..9f4d3fb78 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -8,6 +8,8 @@
     compat_str,
     compat_urllib_parse_unquote,
 )
+from ..networking import Request
+from ..networking.exceptions import network_exceptions
 from ..utils import (
     ExtractorError,
     clean_html,
@@ -19,11 +21,9 @@
     int_or_none,
     js_to_json,
     merge_dicts,
-    network_exceptions,
     parse_count,
     parse_qs,
     qualities,
-    sanitized_Request,
     traverse_obj,
     try_get,
     url_or_none,
@@ -319,7 +319,7 @@ class FacebookIE(InfoExtractor):
     }
 
     def _perform_login(self, username, password):
-        login_page_req = sanitized_Request(self._LOGIN_URL)
+        login_page_req = Request(self._LOGIN_URL)
         self._set_cookie('facebook.com', 'locale', 'en_US')
         login_page = self._download_webpage(login_page_req, None,
                                             note='Downloading login page',
@@ -340,8 +340,8 @@ def _perform_login(self, username, password):
             'timezone': '-60',
             'trynum': '1',
         }
-        request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        request = Request(self._LOGIN_URL, urlencode_postdata(login_form))
+        request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
         try:
             login_results = self._download_webpage(request, None,
                                                    note='Logging in', errnote='unable to fetch login page')
@@ -367,8 +367,8 @@ def _perform_login(self, username, password):
                 'h': h,
                 'name_action_selected': 'dont_save',
             }
-            check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
-            check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+            check_req = Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
+            check_req.headers['Content-Type'] = 'application/x-www-form-urlencoded'
             check_response = self._download_webpage(check_req, None,
                                                     note='Confirming login')
             if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
diff --git a/yt_dlp/extractor/fc2.py b/yt_dlp/extractor/fc2.py
index dd5e088fc..ba19b6cab 100644
--- a/yt_dlp/extractor/fc2.py
+++ b/yt_dlp/extractor/fc2.py
@@ -3,11 +3,11 @@
 from .common import InfoExtractor
 from ..compat import compat_parse_qs
 from ..dependencies import websockets
+from ..networking import Request
 from ..utils import (
     ExtractorError,
     WebSocketsWrapper,
     js_to_json,
-    sanitized_Request,
     traverse_obj,
     update_url_query,
     urlencode_postdata,
@@ -57,7 +57,7 @@ def _login(self):
         }
 
         login_data = urlencode_postdata(login_form_strs)
-        request = sanitized_Request(
+        request = Request(
             'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
 
         login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
@@ -66,7 +66,7 @@ def _login(self):
             return False
 
         # this is also needed
-        login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
+        login_redir = Request('http://id.fc2.com/?mode=redirect&login=done')
         self._download_webpage(
             login_redir, None, note='Login redirect', errnote='Login redirect failed')
 
diff --git a/yt_dlp/extractor/filmon.py b/yt_dlp/extractor/filmon.py
index 9a93cb984..0cd18f494 100644
--- a/yt_dlp/extractor/filmon.py
+++ b/yt_dlp/extractor/filmon.py
@@ -1,8 +1,6 @@
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_HTTPError,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     qualities,
     strip_or_none,
@@ -40,8 +38,8 @@ def _real_extract(self, url):
                 'https://www.filmon.com/api/vod/movie?id=%s' % video_id,
                 video_id)['response']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
-                errmsg = self._parse_json(e.cause.read().decode(), video_id)['reason']
+            if isinstance(e.cause, HTTPError):
+                errmsg = self._parse_json(e.cause.response.read().decode(), video_id)['reason']
                 raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
             raise
 
@@ -124,8 +122,8 @@ def _real_extract(self, url):
             channel_data = self._download_json(
                 'http://www.filmon.com/api-v2/channel/' + channel_id, channel_id)['data']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
-                errmsg = self._parse_json(e.cause.read().decode(), channel_id)['message']
+            if isinstance(e.cause, HTTPError):
+                errmsg = self._parse_json(e.cause.response.read().decode(), channel_id)['message']
                 raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
             raise
 
diff --git a/yt_dlp/extractor/fox.py b/yt_dlp/extractor/fox.py
index 15c0c48c1..8fb4ada6b 100644
--- a/yt_dlp/extractor/fox.py
+++ b/yt_dlp/extractor/fox.py
@@ -3,10 +3,10 @@
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_HTTPError,
     compat_str,
     compat_urllib_parse_unquote,
 )
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -68,9 +68,9 @@ def _call_api(self, path, video_id, data=None):
                 'https://api3.fox.com/v2.0/' + path,
                 video_id, data=data, headers=headers)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 entitlement_issues = self._parse_json(
-                    e.cause.read().decode(), video_id)['entitlementIssues']
+                    e.cause.response.read().decode(), video_id)['entitlementIssues']
                 for e in entitlement_issues:
                     if e.get('errorCode') == 1005:
                         raise ExtractorError(
@@ -123,8 +123,8 @@ def _real_extract(self, url):
         try:
             m3u8_url = self._download_json(release_url, video_id)['playURL']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                error = self._parse_json(e.cause.read().decode(), video_id)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                error = self._parse_json(e.cause.response.read().decode(), video_id)
                 if error.get('exception') == 'GeoLocationBlocked':
                     self.raise_geo_restricted(countries=['US'])
                 raise ExtractorError(error['description'], expected=True)
diff --git a/yt_dlp/extractor/foxsports.py b/yt_dlp/extractor/foxsports.py
index f906a1718..8e89ccf84 100644
--- a/yt_dlp/extractor/foxsports.py
+++ b/yt_dlp/extractor/foxsports.py
@@ -1,6 +1,7 @@
 from .common import InfoExtractor
 from .uplynk import UplynkPreplayIE
-from ..utils import HEADRequest, float_or_none, make_archive_id, smuggle_url
+from ..networking import HEADRequest
+from ..utils import float_or_none, make_archive_id, smuggle_url
 
 
 class FoxSportsIE(InfoExtractor):
@@ -35,7 +36,7 @@ def _real_extract(self, url):
                 'x-api-key': 'cf289e299efdfa39fb6316f259d1de93',
             })
         preplay_url = self._request_webpage(
-            HEADRequest(data['url']), video_id, 'Fetching preplay URL').geturl()
+            HEADRequest(data['url']), video_id, 'Fetching preplay URL').url
 
         return {
             '_type': 'url_transparent',
diff --git a/yt_dlp/extractor/fujitv.py b/yt_dlp/extractor/fujitv.py
index 668bb2743..77e826e2d 100644
--- a/yt_dlp/extractor/fujitv.py
+++ b/yt_dlp/extractor/fujitv.py
@@ -1,5 +1,5 @@
-from ..utils import HEADRequest
 from .common import InfoExtractor
+from ..networking import HEADRequest
 
 
 class FujiTVFODPlus7IE(InfoExtractor):
diff --git a/yt_dlp/extractor/funimation.py b/yt_dlp/extractor/funimation.py
index 47c316664..41de85cc6 100644
--- a/yt_dlp/extractor/funimation.py
+++ b/yt_dlp/extractor/funimation.py
@@ -3,7 +3,7 @@
 import string
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -46,8 +46,8 @@ def _perform_login(self, username, password):
                 }))
             FunimationBaseIE._TOKEN = data['token']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                error = self._parse_json(e.cause.read().decode(), None)['error']
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                error = self._parse_json(e.cause.response.read().decode(), None)['error']
                 raise ExtractorError(error, expected=True)
             raise
 
diff --git a/yt_dlp/extractor/gdcvault.py b/yt_dlp/extractor/gdcvault.py
index 2878bbd88..4265feb61 100644
--- a/yt_dlp/extractor/gdcvault.py
+++ b/yt_dlp/extractor/gdcvault.py
@@ -2,13 +2,8 @@
 
 from .common import InfoExtractor
 from .kaltura import KalturaIE
-from ..utils import (
-    HEADRequest,
-    remove_start,
-    sanitized_Request,
-    smuggle_url,
-    urlencode_postdata,
-)
+from ..networking import HEADRequest, Request
+from ..utils import remove_start, smuggle_url, urlencode_postdata
 
 
 class GDCVaultIE(InfoExtractor):
@@ -138,8 +133,8 @@ def _login(self, webpage_url, display_id):
             'password': password,
         }
 
-        request = sanitized_Request(login_url, urlencode_postdata(login_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        request = Request(login_url, urlencode_postdata(login_form))
+        request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
         self._download_webpage(request, display_id, 'Logging in')
         start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
         self._download_webpage(logout_url, display_id, 'Logging out')
@@ -163,7 +158,7 @@ def _real_extract(self, url):
             video_url = 'http://www.gdcvault.com' + direct_url
             # resolve the url so that we can detect the correct extension
             video_url = self._request_webpage(
-                HEADRequest(video_url), video_id).geturl()
+                HEADRequest(video_url), video_id).url
 
             return {
                 'id': video_id,
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 87cf11d6b..8fa4c6221 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -2431,7 +2431,7 @@ def _real_extract(self, url):
             'Accept-Encoding': 'identity',
             **smuggled_data.get('http_headers', {})
         })
-        new_url = full_response.geturl()
+        new_url = full_response.url
         url = urllib.parse.urlparse(url)._replace(scheme=urllib.parse.urlparse(new_url).scheme).geturl()
         if new_url != extract_basic_auth(url)[0]:
             self.report_following_redirect(new_url)
@@ -2529,12 +2529,12 @@ def _real_extract(self, url):
                 return self.playlist_result(
                     self._parse_xspf(
                         doc, video_id, xspf_url=url,
-                        xspf_base_url=full_response.geturl()),
+                        xspf_base_url=full_response.url),
                     video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                 info_dict['formats'], info_dict['subtitles'] = self._parse_mpd_formats_and_subtitles(
                     doc,
-                    mpd_base_url=full_response.geturl().rpartition('/')[0],
+                    mpd_base_url=full_response.url.rpartition('/')[0],
                     mpd_url=url)
                 self._extra_manifest_info(info_dict, url)
                 self.report_detected('DASH manifest')
@@ -2572,7 +2572,7 @@ def _extract_embeds(self, url, webpage, *, urlh=None, info_dict={}):
         info_dict = types.MappingProxyType(info_dict)  # Prevents accidental mutation
         video_id = traverse_obj(info_dict, 'display_id', 'id') or self._generic_id(url)
         url, smuggled_data = unsmuggle_url(url, {})
-        actual_url = urlh.geturl() if urlh else url
+        actual_url = urlh.url if urlh else url
 
         # Sometimes embedded video player is hidden behind percent encoding
         # (e.g. https://github.com/ytdl-org/youtube-dl/issues/2448)
diff --git a/yt_dlp/extractor/globo.py b/yt_dlp/extractor/globo.py
index a7be2cb76..df98f093c 100644
--- a/yt_dlp/extractor/globo.py
+++ b/yt_dlp/extractor/globo.py
@@ -8,8 +8,8 @@
 from ..compat import (
     compat_str,
 )
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     ExtractorError,
     float_or_none,
     orderedSet,
diff --git a/yt_dlp/extractor/googledrive.py b/yt_dlp/extractor/googledrive.py
index 8a4cd1690..2fdec20f6 100644
--- a/yt_dlp/extractor/googledrive.py
+++ b/yt_dlp/extractor/googledrive.py
@@ -228,7 +228,7 @@ def add_source_format(urlh):
                     # Using original URLs may result in redirect loop due to
                     # google.com's cookies mistakenly used for googleusercontent.com
                     # redirect URLs (see #23919).
-                    'url': urlh.geturl(),
+                    'url': urlh.url,
                     'ext': determine_ext(title, 'mp4').lower(),
                     'format_id': 'source',
                     'quality': 1,
diff --git a/yt_dlp/extractor/hketv.py b/yt_dlp/extractor/hketv.py
index 10879564f..e026996da 100644
--- a/yt_dlp/extractor/hketv.py
+++ b/yt_dlp/extractor/hketv.py
@@ -126,7 +126,7 @@ def _real_extract(self, url):
             # If we ever wanted to provide the final resolved URL that
             # does not require cookies, albeit with a shorter lifespan:
             #     urlh = self._downloader.urlopen(file_url)
-            #     resolved_url = urlh.geturl()
+            #     resolved_url = urlh.url
             label = fmt.get('label')
             h = self._FORMAT_HEIGHTS.get(label)
             w = h * width // height if h and width and height else None
diff --git a/yt_dlp/extractor/hotnewhiphop.py b/yt_dlp/extractor/hotnewhiphop.py
index f8570cb86..3007fbb53 100644
--- a/yt_dlp/extractor/hotnewhiphop.py
+++ b/yt_dlp/extractor/hotnewhiphop.py
@@ -1,11 +1,7 @@
 from .common import InfoExtractor
 from ..compat import compat_b64decode
-from ..utils import (
-    ExtractorError,
-    HEADRequest,
-    sanitized_Request,
-    urlencode_postdata,
-)
+from ..networking import HEADRequest, Request
+from ..utils import ExtractorError, urlencode_postdata
 
 
 class HotNewHipHopIE(InfoExtractor):
@@ -36,9 +32,9 @@ def _real_extract(self, url):
             ('mediaType', 's'),
             ('mediaId', video_id),
         ])
-        r = sanitized_Request(
+        r = Request(
             'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
-        r.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        r.headers['Content-Type'] = 'application/x-www-form-urlencoded'
         mkd = self._download_json(
             r, video_id, note='Requesting media key',
             errnote='Could not download media key')
@@ -50,7 +46,7 @@ def _real_extract(self, url):
         req = self._request_webpage(
             redirect_req, video_id,
             note='Resolving final URL', errnote='Could not resolve final URL')
-        video_url = req.geturl()
+        video_url = req.url
         if video_url.endswith('.html'):
             raise ExtractorError('Redirect failed')
 
diff --git a/yt_dlp/extractor/hotstar.py b/yt_dlp/extractor/hotstar.py
index 591e23b8a..324e9f51d 100644
--- a/yt_dlp/extractor/hotstar.py
+++ b/yt_dlp/extractor/hotstar.py
@@ -6,7 +6,8 @@
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -233,7 +234,7 @@ def _real_extract(self, url):
                         'height': int_or_none(playback_set.get('height')),
                     }]
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                     geo_restricted = True
                 continue
 
diff --git a/yt_dlp/extractor/hrti.py b/yt_dlp/extractor/hrti.py
index cfec80d14..57b76e46b 100644
--- a/yt_dlp/extractor/hrti.py
+++ b/yt_dlp/extractor/hrti.py
@@ -1,13 +1,13 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking import Request
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
     parse_age_limit,
-    sanitized_Request,
     try_get,
 )
 
@@ -42,7 +42,7 @@ def _initialize_pre_login(self):
             'application_version': self._APP_VERSION
         }
 
-        req = sanitized_Request(self._API_URL, data=json.dumps(app_data).encode('utf-8'))
+        req = Request(self._API_URL, data=json.dumps(app_data).encode('utf-8'))
         req.get_method = lambda: 'PUT'
 
         resources = self._download_json(
@@ -73,8 +73,8 @@ def _perform_login(self, username, password):
                 self._login_url, None, note='Logging in', errnote='Unable to log in',
                 data=json.dumps(auth_data).encode('utf-8'))
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 406:
-                auth_info = self._parse_json(e.cause.read().encode('utf-8'), None)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 406:
+                auth_info = self._parse_json(e.cause.response.read().encode('utf-8'), None)
             else:
                 raise
 
diff --git a/yt_dlp/extractor/ign.py b/yt_dlp/extractor/ign.py
index e4db7f9fa..64875f8ce 100644
--- a/yt_dlp/extractor/ign.py
+++ b/yt_dlp/extractor/ign.py
@@ -1,8 +1,9 @@
 import re
-import urllib.error
+import urllib.parse
 
 from .common import InfoExtractor
 from ..compat import compat_parse_qs
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -27,9 +28,9 @@ def _checked_call_api(self, slug):
         try:
             return self._call_api(slug)
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                 e.cause.args = e.cause.args or [
-                    e.cause.geturl(), e.cause.getcode(), e.cause.reason]
+                    e.cause.response.url, e.cause.status, e.cause.reason]
                 raise ExtractorError(
                     'Content not found: expired?', cause=e.cause,
                     expected=True)
@@ -226,7 +227,7 @@ def _real_extract(self, url):
             parsed_url._replace(path=parsed_url.path.rsplit('/', 1)[0] + '/embed'))
 
         webpage, urlh = self._download_webpage_handle(embed_url, video_id)
-        new_url = urlh.geturl()
+        new_url = urlh.url
         ign_url = compat_parse_qs(
             urllib.parse.urlparse(new_url).query).get('url', [None])[-1]
         if ign_url:
@@ -323,14 +324,14 @@ def _checked_call_api(self, slug):
         try:
             return self._call_api(slug)
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError):
+            if isinstance(e.cause, HTTPError):
                 e.cause.args = e.cause.args or [
-                    e.cause.geturl(), e.cause.getcode(), e.cause.reason]
-                if e.cause.code == 404:
+                    e.cause.response.url, e.cause.status, e.cause.reason]
+                if e.cause.status == 404:
                     raise ExtractorError(
                         'Content not found: expired?', cause=e.cause,
                         expected=True)
-                elif e.cause.code == 503:
+                elif e.cause.status == 503:
                     self.report_warning(error_to_compat_str(e.cause))
                     return
             raise
diff --git a/yt_dlp/extractor/imggaming.py b/yt_dlp/extractor/imggaming.py
index 8e220fd9f..a40aa2176 100644
--- a/yt_dlp/extractor/imggaming.py
+++ b/yt_dlp/extractor/imggaming.py
@@ -1,7 +1,7 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -52,9 +52,9 @@ def _extract_dve_api_url(self, media_id, media_type):
             return self._call_api(
                 stream_path, media_id)['playerUrlCallback']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 raise ExtractorError(
-                    self._parse_json(e.cause.read().decode(), media_id)['messages'][0],
+                    self._parse_json(e.cause.response.read().decode(), media_id)['messages'][0],
                     expected=True)
             raise
 
diff --git a/yt_dlp/extractor/instagram.py b/yt_dlp/extractor/instagram.py
index 02335138f..bfc4b7b88 100644
--- a/yt_dlp/extractor/instagram.py
+++ b/yt_dlp/extractor/instagram.py
@@ -3,9 +3,9 @@
 import json
 import re
 import time
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     decode_base_n,
@@ -442,7 +442,7 @@ def _real_extract(self, url):
             shared_data = self._search_json(
                 r'window\._sharedData\s*=', webpage, 'shared data', video_id, fatal=False) or {}
 
-            if shared_data and self._LOGIN_URL not in urlh.geturl():
+            if shared_data and self._LOGIN_URL not in urlh.url:
                 media.update(traverse_obj(
                     shared_data, ('entry_data', 'PostPage', 0, 'graphql', 'shortcode_media'),
                     ('entry_data', 'PostPage', 0, 'media'), expected_type=dict) or {})
@@ -589,7 +589,7 @@ def _extract_graphql(self, data, url):
                 except ExtractorError as e:
                     # if it's an error caused by a bad query, and there are
                     # more GIS templates to try, ignore it and keep trying
-                    if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                    if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                         if gis_tmpl != gis_tmpls[-1]:
                             continue
                     raise
diff --git a/yt_dlp/extractor/iprima.py b/yt_dlp/extractor/iprima.py
index e58e9c2ee..6dec1510d 100644
--- a/yt_dlp/extractor/iprima.py
+++ b/yt_dlp/extractor/iprima.py
@@ -81,7 +81,7 @@ def _perform_login(self, username, password):
             note='Logging in')
 
         # a profile may need to be selected first, even when there is only a single one
-        if '/profile-select' in login_handle.geturl():
+        if '/profile-select' in login_handle.url:
             profile_id = self._search_regex(
                 r'data-identifier\s*=\s*["\']?(\w+)', profile_select_html, 'profile id')
 
@@ -89,7 +89,7 @@ def _perform_login(self, username, password):
                 f'{self._AUTH_ROOT}/user/profile-select-perform/{profile_id}', None,
                 query={'continueUrl': '/user/login?redirect_uri=/user/'}, note='Selecting profile')
 
-        code = traverse_obj(login_handle.geturl(), ({parse_qs}, 'code', 0))
+        code = traverse_obj(login_handle.url, ({parse_qs}, 'code', 0))
         if not code:
             raise ExtractorError('Login failed', expected=True)
 
diff --git a/yt_dlp/extractor/kakao.py b/yt_dlp/extractor/kakao.py
index 1f0f0a5d5..43055e89d 100644
--- a/yt_dlp/extractor/kakao.py
+++ b/yt_dlp/extractor/kakao.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -101,8 +101,8 @@ def _real_extract(self, url):
                     cdn_api_base, video_id, query=query,
                     note='Downloading video URL for profile %s' % profile_name)
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                    resp = self._parse_json(e.cause.read().decode(), video_id)
+                if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                    resp = self._parse_json(e.cause.response.read().decode(), video_id)
                     if resp.get('code') == 'GeoBlocked':
                         self.raise_geo_restricted()
                 raise
diff --git a/yt_dlp/extractor/kick.py b/yt_dlp/extractor/kick.py
index be1dfd4b1..d12437242 100644
--- a/yt_dlp/extractor/kick.py
+++ b/yt_dlp/extractor/kick.py
@@ -1,7 +1,6 @@
 from .common import InfoExtractor
-
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     UserNotLive,
     float_or_none,
     merge_dicts,
diff --git a/yt_dlp/extractor/kuwo.py b/yt_dlp/extractor/kuwo.py
index cfec1c50f..e8a061a10 100644
--- a/yt_dlp/extractor/kuwo.py
+++ b/yt_dlp/extractor/kuwo.py
@@ -91,7 +91,7 @@ def _real_extract(self, url):
         webpage, urlh = self._download_webpage_handle(
             url, song_id, note='Download song detail info',
             errnote='Unable to get song detail info')
-        if song_id not in urlh.geturl() or '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
+        if song_id not in urlh.url or '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
             raise ExtractorError('this song has been offline because of copyright issues', expected=True)
 
         song_name = self._html_search_regex(
diff --git a/yt_dlp/extractor/la7.py b/yt_dlp/extractor/la7.py
index 36bfaf5c3..a3cd12b00 100644
--- a/yt_dlp/extractor/la7.py
+++ b/yt_dlp/extractor/la7.py
@@ -1,13 +1,8 @@
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    HEADRequest,
-    int_or_none,
-    parse_duration,
-    unified_strdate,
-)
+from ..networking import HEADRequest
+from ..utils import float_or_none, int_or_none, parse_duration, unified_strdate
 
 
 class LA7IE(InfoExtractor):
diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py
index 23d3daf13..6af64f0df 100644
--- a/yt_dlp/extractor/lbry.py
+++ b/yt_dlp/extractor/lbry.py
@@ -3,9 +3,9 @@
 import urllib.parse
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     OnDemandPagedList,
     UnsupportedError,
     determine_ext,
@@ -266,7 +266,7 @@ def _real_extract(self, url):
             # HEAD request returns redirect response to m3u8 URL if available
             final_url = self._request_webpage(
                 HEADRequest(streaming_url), display_id, headers=headers,
-                note='Downloading streaming redirect url info').geturl()
+                note='Downloading streaming redirect url info').url
 
         elif result.get('value_type') == 'stream':
             claim_id, is_live = result['signing_channel']['claim_id'], True
diff --git a/yt_dlp/extractor/lecturio.py b/yt_dlp/extractor/lecturio.py
index 973764c63..bb059d3a2 100644
--- a/yt_dlp/extractor/lecturio.py
+++ b/yt_dlp/extractor/lecturio.py
@@ -25,7 +25,7 @@ def _perform_login(self, username, password):
             self._LOGIN_URL, None, 'Downloading login popup')
 
         def is_logged(url_handle):
-            return self._LOGIN_URL not in url_handle.geturl()
+            return self._LOGIN_URL not in url_handle.url
 
         # Already logged in
         if is_logged(urlh):
diff --git a/yt_dlp/extractor/lego.py b/yt_dlp/extractor/lego.py
index 811b44758..46fc7a9b6 100644
--- a/yt_dlp/extractor/lego.py
+++ b/yt_dlp/extractor/lego.py
@@ -1,7 +1,7 @@
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -75,7 +75,7 @@ def _real_extract(self, url):
                     'videoId': '%s_%s' % (uuid.UUID(video_id), locale),
                 }, headers=self.geo_verification_headers())
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 451:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 451:
                 self.raise_geo_restricted(countries=countries)
             raise
 
diff --git a/yt_dlp/extractor/limelight.py b/yt_dlp/extractor/limelight.py
index e11ec43d6..4e50f106f 100644
--- a/yt_dlp/extractor/limelight.py
+++ b/yt_dlp/extractor/limelight.py
@@ -1,7 +1,7 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     float_or_none,
@@ -69,8 +69,8 @@ def _call_playlist_service(self, item_id, method, fatal=True, referer=None):
                 item_id, 'Downloading PlaylistService %s JSON' % method,
                 fatal=fatal, headers=headers)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                error = self._parse_json(e.cause.read().decode(), item_id)['detail']['contentAccessPermission']
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                error = self._parse_json(e.cause.response.read().decode(), item_id)['detail']['contentAccessPermission']
                 if error == 'CountryDisabled':
                     self.raise_geo_restricted()
                 raise ExtractorError(error, expected=True)
diff --git a/yt_dlp/extractor/linuxacademy.py b/yt_dlp/extractor/linuxacademy.py
index 7bb64e17c..0b1644293 100644
--- a/yt_dlp/extractor/linuxacademy.py
+++ b/yt_dlp/extractor/linuxacademy.py
@@ -2,11 +2,8 @@
 import random
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_b64decode,
-    compat_HTTPError,
-    compat_str,
-)
+from ..compat import compat_b64decode, compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -107,7 +104,7 @@ def random_string():
             'sso': 'true',
         })
 
-        login_state_url = urlh.geturl()
+        login_state_url = urlh.url
 
         try:
             login_page = self._download_webpage(
@@ -119,8 +116,8 @@ def random_string():
                     'Referer': login_state_url,
                 })
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                error = self._parse_json(e.cause.read(), None)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                error = self._parse_json(e.cause.response.read(), None)
                 message = error.get('description') or error['code']
                 raise ExtractorError(
                     '%s said: %s' % (self.IE_NAME, message), expected=True)
@@ -137,7 +134,7 @@ def random_string():
             })
 
         access_token = self._search_regex(
-            r'access_token=([^=&]+)', urlh.geturl(),
+            r'access_token=([^=&]+)', urlh.url,
             'access token', default=None)
         if not access_token:
             access_token = self._parse_json(
diff --git a/yt_dlp/extractor/mediasite.py b/yt_dlp/extractor/mediasite.py
index fe549c49f..7ea78ab69 100644
--- a/yt_dlp/extractor/mediasite.py
+++ b/yt_dlp/extractor/mediasite.py
@@ -171,7 +171,7 @@ def _real_extract(self, url):
         query = mobj.group('query')
 
         webpage, urlh = self._download_webpage_handle(url, resource_id)  # XXX: add UrlReferrer?
-        redirect_url = urlh.geturl()
+        redirect_url = urlh.url
 
         # XXX: might have also extracted UrlReferrer and QueryString from the html
         service_path = compat_urlparse.urljoin(redirect_url, self._html_search_regex(
diff --git a/yt_dlp/extractor/megatvcom.py b/yt_dlp/extractor/megatvcom.py
index 2f3f11f51..93c7e7dc0 100644
--- a/yt_dlp/extractor/megatvcom.py
+++ b/yt_dlp/extractor/megatvcom.py
@@ -1,14 +1,14 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
+    ExtractorError,
     clean_html,
     determine_ext,
-    ExtractorError,
     extract_attributes,
     get_element_by_class,
     get_element_html_by_id,
-    HEADRequest,
     parse_qs,
     unescapeHTML,
     unified_timestamp,
@@ -160,5 +160,5 @@ def _real_extract(self, url):
         canonical_url = self._request_webpage(
             HEADRequest(canonical_url), video_id,
             note='Resolve canonical URL',
-            errnote='Could not resolve canonical URL').geturl()
+            errnote='Could not resolve canonical URL').url
         return self.url_result(canonical_url, MegaTVComIE.ie_key(), video_id)
diff --git a/yt_dlp/extractor/mgtv.py b/yt_dlp/extractor/mgtv.py
index 06edcb396..31ccf004e 100644
--- a/yt_dlp/extractor/mgtv.py
+++ b/yt_dlp/extractor/mgtv.py
@@ -1,9 +1,9 @@
 import base64
 import time
-import urllib.error
 import uuid
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -86,8 +86,8 @@ def _real_extract(self, url):
                     'type': 'pch5'
                 }, headers=self.geo_verification_headers())['data']
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
-                error = self._parse_json(e.cause.read().decode(), None)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                error = self._parse_json(e.cause.response.read().decode(), None)
                 if error.get('code') == 40005:
                     self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
                 raise ExtractorError(error['msg'], expected=True)
diff --git a/yt_dlp/extractor/minds.py b/yt_dlp/extractor/minds.py
index 2fb17920c..27a6e3805 100644
--- a/yt_dlp/extractor/minds.py
+++ b/yt_dlp/extractor/minds.py
@@ -106,7 +106,7 @@ def _real_extract(self, url):
         if poster:
             urlh = self._request_webpage(poster, video_id, fatal=False)
             if urlh:
-                thumbnail = urlh.geturl()
+                thumbnail = urlh.url
 
         return {
             'id': video_id,
diff --git a/yt_dlp/extractor/miomio.py b/yt_dlp/extractor/miomio.py
index a0a041ea5..8df8cba19 100644
--- a/yt_dlp/extractor/miomio.py
+++ b/yt_dlp/extractor/miomio.py
@@ -2,12 +2,8 @@
 
 from .common import InfoExtractor
 from ..compat import compat_urlparse
-from ..utils import (
-    xpath_text,
-    int_or_none,
-    ExtractorError,
-    sanitized_Request,
-)
+from ..networking import Request
+from ..utils import ExtractorError, int_or_none, xpath_text
 
 
 class MioMioIE(InfoExtractor):
@@ -61,7 +57,7 @@ def _extract_mioplayer(self, webpage, video_id, title, http_headers):
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
             video_id)
 
-        vid_config_request = sanitized_Request(
+        vid_config_request = Request(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
             headers=http_headers)
 
diff --git a/yt_dlp/extractor/mtv.py b/yt_dlp/extractor/mtv.py
index d91be6270..0d700b9a8 100644
--- a/yt_dlp/extractor/mtv.py
+++ b/yt_dlp/extractor/mtv.py
@@ -2,16 +2,15 @@
 
 from .common import InfoExtractor
 from ..compat import compat_str
+from ..networking import HEADRequest, Request
 from ..utils import (
     ExtractorError,
+    RegexNotFoundError,
     find_xpath_attr,
     fix_xml_ampersands,
     float_or_none,
-    HEADRequest,
     int_or_none,
     join_nonempty,
-    RegexNotFoundError,
-    sanitized_Request,
     strip_or_none,
     timeconvert,
     try_get,
@@ -51,15 +50,15 @@ def _get_thumbnail_url(self, uri, itemdoc):
 
     def _extract_mobile_video_formats(self, mtvn_id):
         webpage_url = self._MOBILE_TEMPLATE % mtvn_id
-        req = sanitized_Request(webpage_url)
+        req = Request(webpage_url)
         # Otherwise we get a webpage that would execute some javascript
-        req.add_header('User-Agent', 'curl/7')
+        req.headers['User-Agent'] = 'curl/7'
         webpage = self._download_webpage(req, mtvn_id,
                                          'Downloading mobile page')
         metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
         req = HEADRequest(metrics_url)
         response = self._request_webpage(req, mtvn_id, 'Resolving url')
-        url = response.geturl()
+        url = response.url
         # Transform the url to get the best quality:
         url = re.sub(r'.+pxE=mp4', 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=0+_pxK=18639+_pxE=mp4', url, 1)
         return [{'url': url, 'ext': 'mp4'}]
diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index ddc89a7c2..299b05174 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -6,9 +6,9 @@
 from .theplatform import ThePlatformIE, default_ns
 from .adobepass import AdobePassIE
 from ..compat import compat_urllib_parse_unquote
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     RegexNotFoundError,
     UserNotLive,
     clean_html,
diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py
index 7a5a02dfa..4f3e691b7 100644
--- a/yt_dlp/extractor/nebula.py
+++ b/yt_dlp/extractor/nebula.py
@@ -1,8 +1,8 @@
 import itertools
 import json
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import ExtractorError, make_archive_id, parse_iso8601, remove_start
 
 _BASE_URL_RE = r'https?://(?:www\.|beta\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)'
@@ -48,7 +48,7 @@ def inner_call():
             return inner_call()
         except ExtractorError as exc:
             # if 401 or 403, attempt credential re-auth and retry
-            if exc.cause and isinstance(exc.cause, urllib.error.HTTPError) and exc.cause.code in (401, 403):
+            if exc.cause and isinstance(exc.cause, HTTPError) and exc.cause.status in (401, 403):
                 self.to_screen(f'Reauthenticating to Nebula and retrying, because last {auth_type} call resulted in error {exc.cause.code}')
                 self._perform_login()
                 return inner_call()
diff --git a/yt_dlp/extractor/neteasemusic.py b/yt_dlp/extractor/neteasemusic.py
index 595709899..5b7307bc8 100644
--- a/yt_dlp/extractor/neteasemusic.py
+++ b/yt_dlp/extractor/neteasemusic.py
@@ -11,6 +11,7 @@
 from .common import InfoExtractor
 from ..aes import aes_ecb_encrypt, pkcs7_padding
 from ..compat import compat_urllib_parse_urlencode
+from ..networking import Request
 from ..utils import (
     ExtractorError,
     bytes_to_intlist,
@@ -18,7 +19,6 @@
     float_or_none,
     int_or_none,
     intlist_to_bytes,
-    sanitized_Request,
     try_get,
 )
 
@@ -146,8 +146,8 @@ def convert_milliseconds(cls, ms):
         return int(round(ms / 1000.0))
 
     def query_api(self, endpoint, video_id, note):
-        req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
-        req.add_header('Referer', self._API_BASE)
+        req = Request('%s%s' % (self._API_BASE, endpoint))
+        req.headers['Referer'] = self._API_BASE
         return self._download_json(req, video_id, note)
 
 
diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py
index 89e8e6093..fa2d709d2 100644
--- a/yt_dlp/extractor/niconico.py
+++ b/yt_dlp/extractor/niconico.py
@@ -8,10 +8,8 @@
 from urllib.parse import urlparse
 
 from .common import InfoExtractor, SearchInfoExtractor
-from ..compat import (
-    compat_HTTPError,
-)
 from ..dependencies import websockets
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
@@ -396,7 +394,7 @@ def _real_extract(self, url):
             webpage, handle = self._download_webpage_handle(
                 'https://www.nicovideo.jp/watch/' + video_id, video_id)
             if video_id.startswith('so'):
-                video_id = self._match_id(handle.geturl())
+                video_id = self._match_id(handle.url)
 
             api_data = self._parse_json(self._html_search_regex(
                 'data-api-data="([^"]+)"', webpage,
@@ -407,9 +405,9 @@ def _real_extract(self, url):
                     'https://www.nicovideo.jp/api/watch/v3/%s?_frontendId=6&_frontendVersion=0&actionTrackId=AAAAAAAAAA_%d' % (video_id, round(time.time() * 1000)), video_id,
                     note='Downloading API JSON', errnote='Unable to fetch data')['data']
             except ExtractorError:
-                if not isinstance(e.cause, compat_HTTPError):
+                if not isinstance(e.cause, HTTPError):
                     raise
-                webpage = e.cause.read().decode('utf-8', 'replace')
+                webpage = e.cause.response.read().decode('utf-8', 'replace')
                 error_msg = self._html_search_regex(
                     r'(?s)<section\s+class="(?:(?:ErrorMessage|WatchExceptionPage-message)\s*)+">(.+?)</section>',
                     webpage, 'error reason', default=None)
@@ -742,7 +740,7 @@ def _real_extract(self, url):
         try:
             mylist = self._call_api(list_id, 'list', {'pageSize': 1})
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 self.raise_login_required('You have to be logged in to get your history')
             raise
         return self.playlist_result(self._entries(list_id), list_id, **self._parse_owner(mylist))
@@ -951,8 +949,8 @@ def _real_extract(self, url):
             'frontend_id': traverse_obj(embedded_data, ('site', 'frontendId')) or '9',
         })
 
-        hostname = remove_start(urlparse(urlh.geturl()).hostname, 'sp.')
-        cookies = try_get(urlh.geturl(), self._downloader._calc_cookies)
+        hostname = remove_start(urlparse(urlh.url).hostname, 'sp.')
+        cookies = try_get(urlh.url, self._downloader._calc_cookies)
         latency = try_get(self._configuration_arg('latency'), lambda x: x[0])
         if latency not in self._KNOWN_LATENCY:
             latency = 'high'
diff --git a/yt_dlp/extractor/njpwworld.py b/yt_dlp/extractor/njpwworld.py
index 7b8a526f0..607838133 100644
--- a/yt_dlp/extractor/njpwworld.py
+++ b/yt_dlp/extractor/njpwworld.py
@@ -51,7 +51,7 @@ def _perform_login(self, username, password):
             data=urlencode_postdata({'login_id': username, 'pw': password}),
             headers={'Referer': 'https://front.njpwworld.com/auth'})
         # /auth/login will return 302 for successful logins
-        if urlh.geturl() == self._LOGIN_URL:
+        if urlh.url == self._LOGIN_URL:
             self.report_warning('unable to login')
             return False
 
diff --git a/yt_dlp/extractor/nosvideo.py b/yt_dlp/extractor/nosvideo.py
index b6d3ea40c..7e9688c0b 100644
--- a/yt_dlp/extractor/nosvideo.py
+++ b/yt_dlp/extractor/nosvideo.py
@@ -1,9 +1,9 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import Request
 from ..utils import (
     ExtractorError,
-    sanitized_Request,
     urlencode_postdata,
     xpath_text,
     xpath_with_ns,
@@ -36,8 +36,8 @@ def _real_extract(self, url):
             'op': 'download1',
             'method_free': 'Continue to Video',
         }
-        req = sanitized_Request(url, urlencode_postdata(fields))
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+        req = Request(url, urlencode_postdata(fields))
+        req.headers['Content-type'] = 'application/x-www-form-urlencoded'
         webpage = self._download_webpage(req, video_id,
                                          'Downloading download page')
         if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
diff --git a/yt_dlp/extractor/nowness.py b/yt_dlp/extractor/nowness.py
index fc9043bce..a3c29f62c 100644
--- a/yt_dlp/extractor/nowness.py
+++ b/yt_dlp/extractor/nowness.py
@@ -4,10 +4,8 @@
 )
 from .common import InfoExtractor
 from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-)
+from ..networking import Request
+from ..utils import ExtractorError
 
 
 class NownessBaseIE(InfoExtractor):
@@ -40,7 +38,7 @@ def _extract_url_result(self, post):
 
     def _api_request(self, url, request_path):
         display_id = self._match_id(url)
-        request = sanitized_Request(
+        request = Request(
             'http://api.nowness.com/api/' + request_path % display_id,
             headers={
                 'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
diff --git a/yt_dlp/extractor/nrk.py b/yt_dlp/extractor/nrk.py
index 88d08e5e3..384865acc 100644
--- a/yt_dlp/extractor/nrk.py
+++ b/yt_dlp/extractor/nrk.py
@@ -3,7 +3,8 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -148,7 +149,7 @@ def call_playback_api(item, query=None):
             try:
                 return self._call_api(f'playback/{item}/program/{video_id}', video_id, item, query=query)
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                     return self._call_api(f'playback/{item}/{video_id}', video_id, item, query=query)
                 raise
 
diff --git a/yt_dlp/extractor/odkmedia.py b/yt_dlp/extractor/odkmedia.py
index 2960860d6..b852160b9 100644
--- a/yt_dlp/extractor/odkmedia.py
+++ b/yt_dlp/extractor/odkmedia.py
@@ -1,7 +1,7 @@
 import json
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     GeoRestrictedError,
@@ -74,8 +74,8 @@ def _real_extract(self, url):
                 f'https://odkmedia.io/odc/api/v2/playback/{video_info["id"]}/', display_id,
                 headers={'Authorization': '', 'service-name': 'odc'})
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError):
-                error_data = self._parse_json(e.cause.read(), display_id)['detail']
+            if isinstance(e.cause, HTTPError):
+                error_data = self._parse_json(e.cause.response.read(), display_id)['detail']
                 raise GeoRestrictedError(error_data)
 
         formats, subtitles = [], {}
diff --git a/yt_dlp/extractor/odnoklassniki.py b/yt_dlp/extractor/odnoklassniki.py
index e63714e84..1be45d8ad 100644
--- a/yt_dlp/extractor/odnoklassniki.py
+++ b/yt_dlp/extractor/odnoklassniki.py
@@ -7,9 +7,9 @@
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
 )
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     float_or_none,
     int_or_none,
     qualities,
@@ -448,7 +448,7 @@ def _extract_mobile(self, url):
         json_data = self._parse_json(unescapeHTML(json_data), video_id) or {}
 
         redirect_url = self._request_webpage(HEADRequest(
-            json_data['videoSrc']), video_id, 'Requesting download URL').geturl()
+            json_data['videoSrc']), video_id, 'Requesting download URL').url
         self._clear_cookies(redirect_url)
 
         return {
diff --git a/yt_dlp/extractor/orf.py b/yt_dlp/extractor/orf.py
index e9d23a4d1..cc3c003fa 100644
--- a/yt_dlp/extractor/orf.py
+++ b/yt_dlp/extractor/orf.py
@@ -2,11 +2,11 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     clean_html,
     determine_ext,
     float_or_none,
-    HEADRequest,
     InAdvancePagedList,
     int_or_none,
     join_nonempty,
diff --git a/yt_dlp/extractor/owncloud.py b/yt_dlp/extractor/owncloud.py
index e1d5682f8..79fd830bb 100644
--- a/yt_dlp/extractor/owncloud.py
+++ b/yt_dlp/extractor/owncloud.py
@@ -44,7 +44,7 @@ def _real_extract(self, url):
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
         if re.search(r'<label[^>]+for="password"', webpage):
-            webpage = self._verify_video_password(webpage, urlh.geturl(), video_id)
+            webpage = self._verify_video_password(webpage, urlh.url, video_id)
 
         hidden_inputs = self._hidden_inputs(webpage)
         title = hidden_inputs.get('filename')
diff --git a/yt_dlp/extractor/packtpub.py b/yt_dlp/extractor/packtpub.py
index 51778d8a2..56203306f 100644
--- a/yt_dlp/extractor/packtpub.py
+++ b/yt_dlp/extractor/packtpub.py
@@ -1,10 +1,7 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    # compat_str,
-    compat_HTTPError,
-)
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -54,8 +51,8 @@ def _perform_login(self, username, password):
                     'password': password,
                 }).encode())['data']['access']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 404):
-                message = self._parse_json(e.cause.read().decode(), None)['message']
+            if isinstance(e.cause, HTTPError) and e.cause.status in (400, 401, 404):
+                message = self._parse_json(e.cause.response.read().decode(), None)['message']
                 raise ExtractorError(message, expected=True)
             raise
 
@@ -70,7 +67,7 @@ def _real_extract(self, url):
                 'https://services.packtpub.com/products-v1/products/%s/%s/%s' % (course_id, chapter_id, video_id), video_id,
                 'Downloading JSON video', headers=headers)['data']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                 self.raise_login_required('This video is locked')
             raise
 
diff --git a/yt_dlp/extractor/patreon.py b/yt_dlp/extractor/patreon.py
index e93e37eb9..447087436 100644
--- a/yt_dlp/extractor/patreon.py
+++ b/yt_dlp/extractor/patreon.py
@@ -1,10 +1,10 @@
 import itertools
-from urllib.error import HTTPError
 
 from .common import InfoExtractor
 from .vimeo import VimeoIE
 
 from ..compat import compat_urllib_parse_unquote
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     determine_ext,
@@ -37,9 +37,9 @@ def _call_api(self, ep, item_id, query=None, headers=None, fatal=True, note=None
                 item_id, note='Downloading API JSON' if not note else note,
                 query=query, fatal=fatal, headers=headers)
         except ExtractorError as e:
-            if not isinstance(e.cause, HTTPError) or mimetype2ext(e.cause.headers.get('Content-Type')) != 'json':
+            if not isinstance(e.cause, HTTPError) or mimetype2ext(e.cause.response.headers.get('Content-Type')) != 'json':
                 raise
-            err_json = self._parse_json(self._webpage_read_content(e.cause, None, item_id), item_id, fatal=False)
+            err_json = self._parse_json(self._webpage_read_content(e.cause.response, None, item_id), item_id, fatal=False)
             err_message = traverse_obj(err_json, ('errors', ..., 'detail'), get_all=False)
             if err_message:
                 raise ExtractorError(f'Patreon said: {err_message}', expected=True)
diff --git a/yt_dlp/extractor/peloton.py b/yt_dlp/extractor/peloton.py
index 4835822cf..786429988 100644
--- a/yt_dlp/extractor/peloton.py
+++ b/yt_dlp/extractor/peloton.py
@@ -3,7 +3,7 @@
 import urllib.parse
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -83,8 +83,8 @@ def _login(self, video_id):
                 }).encode(),
                 headers={'Content-Type': 'application/json', 'User-Agent': 'web'})
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                json_string = self._webpage_read_content(e.cause, None, video_id)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                json_string = self._webpage_read_content(e.cause.response, None, video_id)
                 res = self._parse_json(json_string, video_id)
                 raise ExtractorError(res['message'], expected=res['message'] == 'Login failed')
             else:
@@ -96,8 +96,8 @@ def _get_token(self, video_id):
                 'https://api.onepeloton.com/api/subscription/stream', video_id, note='Downloading token',
                 data=json.dumps({}).encode(), headers={'Content-Type': 'application/json'})
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                json_string = self._webpage_read_content(e.cause, None, video_id)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                json_string = self._webpage_read_content(e.cause.response, None, video_id)
                 res = self._parse_json(json_string, video_id)
                 raise ExtractorError(res['message'], expected=res['message'] == 'Stream limit reached')
             else:
@@ -109,7 +109,7 @@ def _real_extract(self, url):
         try:
             self._start_session(video_id)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 self._login(video_id)
                 self._start_session(video_id)
             else:
diff --git a/yt_dlp/extractor/piapro.py b/yt_dlp/extractor/piapro.py
index eb5923d11..5f39e0639 100644
--- a/yt_dlp/extractor/piapro.py
+++ b/yt_dlp/extractor/piapro.py
@@ -69,7 +69,7 @@ def _perform_login(self, username, password):
         if urlh is False:
             login_ok = False
         else:
-            parts = compat_urlparse.urlparse(urlh.geturl())
+            parts = compat_urlparse.urlparse(urlh.url)
             if parts.path != '/':
                 login_ok = False
         if not login_ok:
diff --git a/yt_dlp/extractor/pladform.py b/yt_dlp/extractor/pladform.py
index dcf18e1f3..00500686f 100644
--- a/yt_dlp/extractor/pladform.py
+++ b/yt_dlp/extractor/pladform.py
@@ -78,7 +78,7 @@ def fail(text):
                 expected=True)
 
         if not video:
-            targetUrl = self._request_webpage(url, video_id, note='Resolving final URL').geturl()
+            targetUrl = self._request_webpage(url, video_id, note='Resolving final URL').url
             if targetUrl == url:
                 raise ExtractorError('Can\'t parse page')
             return self.url_result(targetUrl)
diff --git a/yt_dlp/extractor/platzi.py b/yt_dlp/extractor/platzi.py
index b8a441494..166b98c4a 100644
--- a/yt_dlp/extractor/platzi.py
+++ b/yt_dlp/extractor/platzi.py
@@ -36,7 +36,7 @@ def _perform_login(self, username, password):
             headers={'Referer': self._LOGIN_URL})
 
         # login succeeded
-        if 'platzi.com/login' not in urlh.geturl():
+        if 'platzi.com/login' not in urlh.url:
             return
 
         login_error = self._webpage_read_content(
diff --git a/yt_dlp/extractor/playplustv.py b/yt_dlp/extractor/playplustv.py
index 316f220f7..a4439c8bc 100644
--- a/yt_dlp/extractor/playplustv.py
+++ b/yt_dlp/extractor/playplustv.py
@@ -1,13 +1,9 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
-from ..utils import (
-    clean_html,
-    ExtractorError,
-    int_or_none,
-    PUTRequest,
-)
+from ..networking import PUTRequest
+from ..networking.exceptions import HTTPError
+from ..utils import ExtractorError, clean_html, int_or_none
 
 
 class PlayPlusTVIE(InfoExtractor):
@@ -47,9 +43,9 @@ def _perform_login(self, username, password):
         try:
             self._token = self._download_json(req, None)['token']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 raise ExtractorError(self._parse_json(
-                    e.cause.read(), None)['errorMessage'], expected=True)
+                    e.cause.response.read(), None)['errorMessage'], expected=True)
             raise
 
         self._profile = self._call_api('Profiles')['list'][0]['_id']
diff --git a/yt_dlp/extractor/pornhub.py b/yt_dlp/extractor/pornhub.py
index 2f5a572a5..f08414030 100644
--- a/yt_dlp/extractor/pornhub.py
+++ b/yt_dlp/extractor/pornhub.py
@@ -3,11 +3,12 @@
 import math
 import operator
 import re
-import urllib.request
 
 from .common import InfoExtractor
 from .openload import PhantomJSwrapper
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking import Request
+from ..networking.exceptions import HTTPError
 from ..utils import (
     NO_DEFAULT,
     ExtractorError,
@@ -46,8 +47,8 @@ def dl(*args, **kwargs):
                 r'document\.cookie\s*=\s*["\']RNKEY=',
                 r'document\.location\.reload\(true\)')):
             url_or_request = args[0]
-            url = (url_or_request.get_full_url()
-                   if isinstance(url_or_request, urllib.request.Request)
+            url = (url_or_request.url
+                   if isinstance(url_or_request, Request)
                    else url_or_request)
             phantom = PhantomJSwrapper(self, required_version='2.0')
             phantom.get(url, html=webpage)
@@ -602,7 +603,7 @@ def download_page(base_url, num, fallback=False):
                 base_url, item_id, note, query={'page': num})
 
         def is_404(e):
-            return isinstance(e.cause, compat_HTTPError) and e.cause.code == 404
+            return isinstance(e.cause, HTTPError) and e.cause.status == 404
 
         base_url = url
         has_page = page is not None
diff --git a/yt_dlp/extractor/puhutv.py b/yt_dlp/extractor/puhutv.py
index 482e5705f..4b8e5e90d 100644
--- a/yt_dlp/extractor/puhutv.py
+++ b/yt_dlp/extractor/puhutv.py
@@ -1,8 +1,6 @@
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -72,7 +70,7 @@ def _real_extract(self, url):
                 display_id, 'Downloading video JSON',
                 headers=self.geo_verification_headers())
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 self.raise_geo_restricted()
             raise
 
diff --git a/yt_dlp/extractor/radiko.py b/yt_dlp/extractor/radiko.py
index 7fdf78283..cef68eba0 100644
--- a/yt_dlp/extractor/radiko.py
+++ b/yt_dlp/extractor/radiko.py
@@ -41,7 +41,7 @@ def _auth_client(self):
                 'x-radiko-device': 'pc',
                 'x-radiko-user': 'dummy_user',
             })
-        auth1_header = auth1_handle.info()
+        auth1_header = auth1_handle.headers
 
         auth_token = auth1_header['X-Radiko-AuthToken']
         kl = int(auth1_header['X-Radiko-KeyLength'])
diff --git a/yt_dlp/extractor/radiocanada.py b/yt_dlp/extractor/radiocanada.py
index 72c21d502..1a5a6355a 100644
--- a/yt_dlp/extractor/radiocanada.py
+++ b/yt_dlp/extractor/radiocanada.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -74,8 +74,8 @@ def _call_api(self, path, video_id=None, app_code=None, query=None):
             return self._download_json(
                 'https://services.radio-canada.ca/media/' + path, video_id, query=query)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 422):
-                data = self._parse_json(e.cause.read().decode(), None)
+            if isinstance(e.cause, HTTPError) and e.cause.status in (401, 422):
+                data = self._parse_json(e.cause.response.read().decode(), None)
                 error = data.get('error_description') or data['errorMessage']['text']
                 raise ExtractorError(error, expected=True)
             raise
diff --git a/yt_dlp/extractor/rcs.py b/yt_dlp/extractor/rcs.py
index 2440858ca..028d3d90b 100644
--- a/yt_dlp/extractor/rcs.py
+++ b/yt_dlp/extractor/rcs.py
@@ -1,9 +1,9 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     base_url,
     clean_html,
     extract_attributes,
diff --git a/yt_dlp/extractor/rcti.py b/yt_dlp/extractor/rcti.py
index 27b4ad7bb..79d9c8e31 100644
--- a/yt_dlp/extractor/rcti.py
+++ b/yt_dlp/extractor/rcti.py
@@ -3,7 +3,7 @@
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     dict_get,
     ExtractorError,
@@ -186,7 +186,7 @@ def _real_extract(self, url):
         try:
             formats = self._extract_m3u8_formats(video_url, display_id, 'mp4', headers={'Referer': 'https://www.rctiplus.com/'})
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 self.raise_geo_restricted(countries=['ID'], metadata_available=True)
             else:
                 raise e
diff --git a/yt_dlp/extractor/recurbate.py b/yt_dlp/extractor/recurbate.py
index 5534cf3c3..d7294cb14 100644
--- a/yt_dlp/extractor/recurbate.py
+++ b/yt_dlp/extractor/recurbate.py
@@ -1,6 +1,5 @@
-import urllib.error
-
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import ExtractorError, merge_dicts
 
 
@@ -25,7 +24,7 @@ def _real_extract(self, url):
         try:
             webpage = self._download_webpage(url, video_id)
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
             raise
         token = self._html_search_regex(r'data-token="([^"]+)"', webpage, 'token')
diff --git a/yt_dlp/extractor/redbulltv.py b/yt_dlp/extractor/redbulltv.py
index a01bc8434..d1de2490f 100644
--- a/yt_dlp/extractor/redbulltv.py
+++ b/yt_dlp/extractor/redbulltv.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     float_or_none,
     ExtractorError,
@@ -68,9 +68,9 @@ def extract_info(self, video_id):
                 headers={'Authorization': token}
             )
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                 error_message = self._parse_json(
-                    e.cause.read().decode(), video_id)['error']
+                    e.cause.response.read().decode(), video_id)['error']
                 raise ExtractorError('%s said: %s' % (
                     self.IE_NAME, error_message), expected=True)
             raise
diff --git a/yt_dlp/extractor/redgifs.py b/yt_dlp/extractor/redgifs.py
index 098fb8185..f9453202b 100644
--- a/yt_dlp/extractor/redgifs.py
+++ b/yt_dlp/extractor/redgifs.py
@@ -1,8 +1,8 @@
 import functools
-import urllib
 
 from .common import InfoExtractor
 from ..compat import compat_parse_qs
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -82,7 +82,7 @@ def _call_api(self, ep, video_id, *args, **kwargs):
                     f'https://api.redgifs.com/v2/{ep}', video_id, headers=headers, *args, **kwargs)
                 break
             except ExtractorError as e:
-                if first_attempt and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                if first_attempt and isinstance(e.cause, HTTPError) and e.cause.status == 401:
                     del self._API_HEADERS['authorization']  # refresh the token
                     continue
                 raise
diff --git a/yt_dlp/extractor/regiotv.py b/yt_dlp/extractor/regiotv.py
index 6114841fb..edb6ae5bc 100644
--- a/yt_dlp/extractor/regiotv.py
+++ b/yt_dlp/extractor/regiotv.py
@@ -1,10 +1,6 @@
 from .common import InfoExtractor
-
-from ..utils import (
-    sanitized_Request,
-    xpath_text,
-    xpath_with_ns,
-)
+from ..networking import Request
+from ..utils import xpath_text, xpath_with_ns
 
 
 class RegioTVIE(InfoExtractor):
@@ -33,7 +29,7 @@ def _real_extract(self, url):
 
         SOAP_TEMPLATE = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><{0} xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">{1}</key></{0}></soap:Body></soap:Envelope>'
 
-        request = sanitized_Request(
+        request = Request(
             'http://v.telvi.de/',
             SOAP_TEMPLATE.format('GetHTML5VideoData', key).encode('utf-8'))
         video_data = self._download_xml(request, video_id, 'Downloading video XML')
diff --git a/yt_dlp/extractor/rokfin.py b/yt_dlp/extractor/rokfin.py
index 0e40eb32a..4a4d40bef 100644
--- a/yt_dlp/extractor/rokfin.py
+++ b/yt_dlp/extractor/rokfin.py
@@ -245,7 +245,7 @@ def _perform_login(self, username, password):
             f'{self._AUTH_BASE}/token', None,
             note='getting access credentials', errnote='error getting access credentials',
             data=urlencode_postdata({
-                'code': urllib.parse.parse_qs(urllib.parse.urldefrag(urlh.geturl()).fragment).get('code')[0],
+                'code': urllib.parse.parse_qs(urllib.parse.urldefrag(urlh.url).fragment).get('code')[0],
                 'client_id': 'web',
                 'grant_type': 'authorization_code',
                 'redirect_uri': 'https://rokfin.com/silent-check-sso.html'
@@ -269,7 +269,7 @@ def _download_json_using_access_token(self, url_or_request, video_id, headers={}
 
         json_string, urlh = self._download_webpage_handle(
             url_or_request, video_id, headers=headers, query=query, expected_status=401)
-        if not auth_token or urlh.code != 401 or refresh_token is None:
+        if not auth_token or urlh.status != 401 or refresh_token is None:
             return self._parse_json(json_string, video_id)
 
         self._access_mgmt_tokens = self._download_json(
diff --git a/yt_dlp/extractor/roosterteeth.py b/yt_dlp/extractor/roosterteeth.py
index 776fbfbc0..94e673b13 100644
--- a/yt_dlp/extractor/roosterteeth.py
+++ b/yt_dlp/extractor/roosterteeth.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -35,8 +35,8 @@ def _perform_login(self, username, password):
                 }))
         except ExtractorError as e:
             msg = 'Unable to login'
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                resp = self._parse_json(e.cause.read().decode(), None, fatal=False)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                resp = self._parse_json(e.cause.response.read().decode(), None, fatal=False)
                 if resp:
                     error = resp.get('extra_info') or resp.get('error_description') or resp.get('error')
                     if error:
@@ -138,8 +138,8 @@ def _real_extract(self, url):
             m3u8_url = video_data['attributes']['url']
             # XXX: additional URL at video_data['links']['download']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                if self._parse_json(e.cause.read().decode(), display_id).get('access') is False:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                if self._parse_json(e.cause.response.read().decode(), display_id).get('access') is False:
                     self.raise_login_required(
                         '%s is only available for FIRST members' % display_id)
             raise
diff --git a/yt_dlp/extractor/rozhlas.py b/yt_dlp/extractor/rozhlas.py
index 5f83d42e8..63134322d 100644
--- a/yt_dlp/extractor/rozhlas.py
+++ b/yt_dlp/extractor/rozhlas.py
@@ -1,7 +1,7 @@
 import itertools
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     extract_attributes,
@@ -81,7 +81,7 @@ def _extract_formats(self, entry, audio_id):
                             'vcodec': 'none',
                         })
                 except ExtractorError as e:
-                    if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 429:
+                    if isinstance(e.cause, HTTPError) and e.cause.status == 429:
                         retry.error = e.cause
                     else:
                         self.report_warning(e.msg)
diff --git a/yt_dlp/extractor/rte.py b/yt_dlp/extractor/rte.py
index aedaa5b55..7ba80d4ba 100644
--- a/yt_dlp/extractor/rte.py
+++ b/yt_dlp/extractor/rte.py
@@ -1,7 +1,7 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     float_or_none,
     parse_iso8601,
@@ -31,8 +31,8 @@ def _real_extract(self, url):
             except ExtractorError as ee:
                 if num < len(ENDPOINTS) or formats:
                     continue
-                if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
-                    error_info = self._parse_json(ee.cause.read().decode(), item_id, fatal=False)
+                if isinstance(ee.cause, HTTPError) and ee.cause.status == 404:
+                    error_info = self._parse_json(ee.cause.response.read().decode(), item_id, fatal=False)
                     if error_info:
                         raise ExtractorError(
                             '%s said: %s' % (self.IE_NAME, error_info['message']),
diff --git a/yt_dlp/extractor/rts.py b/yt_dlp/extractor/rts.py
index 81c4d7cac..9f73d1811 100644
--- a/yt_dlp/extractor/rts.py
+++ b/yt_dlp/extractor/rts.py
@@ -136,8 +136,8 @@ def download_json(internal_id):
 
             if not entries:
                 page, urlh = self._download_webpage_handle(url, display_id)
-                if re.match(self._VALID_URL, urlh.geturl()).group('id') != media_id:
-                    return self.url_result(urlh.geturl(), 'RTS')
+                if re.match(self._VALID_URL, urlh.url).group('id') != media_id:
+                    return self.url_result(urlh.url, 'RTS')
 
                 # article with videos on rhs
                 videos = re.findall(
diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py
index 82f3f0f8c..f8bf4a182 100644
--- a/yt_dlp/extractor/rumble.py
+++ b/yt_dlp/extractor/rumble.py
@@ -2,7 +2,7 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     UnsupportedError,
@@ -371,7 +371,7 @@ def entries(self, url, playlist_id):
             try:
                 webpage = self._download_webpage(f'{url}?page={page}', playlist_id, note='Downloading page %d' % page)
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                     break
                 raise
             for video_url in re.findall(r'class=video-item--a\s?href=([^>]+\.html)', webpage):
diff --git a/yt_dlp/extractor/safari.py b/yt_dlp/extractor/safari.py
index 450a661e9..8d322d710 100644
--- a/yt_dlp/extractor/safari.py
+++ b/yt_dlp/extractor/safari.py
@@ -28,13 +28,13 @@ def _perform_login(self, username, password):
             'Downloading login page')
 
         def is_logged(urlh):
-            return 'learning.oreilly.com/home/' in urlh.geturl()
+            return 'learning.oreilly.com/home/' in urlh.url
 
         if is_logged(urlh):
             self.LOGGED_IN = True
             return
 
-        redirect_url = urlh.geturl()
+        redirect_url = urlh.url
         parsed_url = compat_urlparse.urlparse(redirect_url)
         qs = compat_parse_qs(parsed_url.query)
         next_uri = compat_urlparse.urljoin(
@@ -129,7 +129,7 @@ def _real_extract(self, url):
 
             webpage, urlh = self._download_webpage_handle(url, video_id)
 
-            mobj = re.match(self._VALID_URL, urlh.geturl())
+            mobj = re.match(self._VALID_URL, urlh.url)
             reference_id = mobj.group('reference_id')
             if not reference_id:
                 reference_id = self._search_regex(
diff --git a/yt_dlp/extractor/sbs.py b/yt_dlp/extractor/sbs.py
index 119106e8e..7a9115047 100644
--- a/yt_dlp/extractor/sbs.py
+++ b/yt_dlp/extractor/sbs.py
@@ -1,6 +1,6 @@
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     float_or_none,
     int_or_none,
     parse_duration,
diff --git a/yt_dlp/extractor/sevenplus.py b/yt_dlp/extractor/sevenplus.py
index 222bf6ce7..6c688d150 100644
--- a/yt_dlp/extractor/sevenplus.py
+++ b/yt_dlp/extractor/sevenplus.py
@@ -2,10 +2,8 @@
 import re
 
 from .brightcove import BrightcoveNewBaseIE
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     try_get,
@@ -97,9 +95,9 @@ def _real_extract(self, url):
                     'videoType': 'vod',
                 }, headers=headers)['media']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 raise ExtractorError(self._parse_json(
-                    e.cause.read().decode(), episode_id)[0]['error_code'], expected=True)
+                    e.cause.response.read().decode(), episode_id)[0]['error_code'], expected=True)
             raise
 
         for source in media.get('sources', {}):
diff --git a/yt_dlp/extractor/shahid.py b/yt_dlp/extractor/shahid.py
index 26a0bff40..d509e8879 100644
--- a/yt_dlp/extractor/shahid.py
+++ b/yt_dlp/extractor/shahid.py
@@ -3,7 +3,7 @@
 import re
 
 from .aws import AWSIE
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -22,7 +22,7 @@ class ShahidBaseIE(AWSIE):
 
     def _handle_error(self, e):
         fail_data = self._parse_json(
-            e.cause.read().decode('utf-8'), None, fatal=False)
+            e.cause.response.read().decode('utf-8'), None, fatal=False)
         if fail_data:
             faults = fail_data.get('faults', [])
             faults_message = ', '.join([clean_html(fault['userMessage']) for fault in faults if fault.get('userMessage')])
@@ -40,7 +40,7 @@ def _call_api(self, path, video_id, request=None):
                 'secret_key': '4WUUJWuFvtTkXbhaWTDv7MhO+0LqoYDWfEnUXoWn',
             }, video_id, query)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
+            if isinstance(e.cause, HTTPError):
                 self._handle_error(e)
             raise
 
@@ -88,7 +88,7 @@ def _perform_login(self, username, password):
                     'Content-Type': 'application/json; charset=UTF-8',
                 })['user']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
+            if isinstance(e.cause, HTTPError):
                 self._handle_error(e)
             raise
 
diff --git a/yt_dlp/extractor/sina.py b/yt_dlp/extractor/sina.py
index aeba4e377..984281188 100644
--- a/yt_dlp/extractor/sina.py
+++ b/yt_dlp/extractor/sina.py
@@ -1,12 +1,12 @@
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     ExtractorError,
-    int_or_none,
-    update_url_query,
-    qualities,
-    get_element_by_attribute,
     clean_html,
+    get_element_by_attribute,
+    int_or_none,
+    qualities,
+    update_url_query,
 )
 
 
@@ -60,7 +60,7 @@ def _real_extract(self, url):
                 self.to_screen('Getting video id')
                 request = HEADRequest(url)
                 _, urlh = self._download_webpage_handle(request, 'NA', False)
-                return self._real_extract(urlh.geturl())
+                return self._real_extract(urlh.url)
             else:
                 pseudo_id = mobj.group('pseudo_id')
                 webpage = self._download_webpage(url, pseudo_id)
diff --git a/yt_dlp/extractor/sixplay.py b/yt_dlp/extractor/sixplay.py
index a6fb6c1f5..ef93b9276 100644
--- a/yt_dlp/extractor/sixplay.py
+++ b/yt_dlp/extractor/sixplay.py
@@ -79,7 +79,7 @@ def _real_extract(self, url):
                             headers=self.geo_verification_headers())
                         if not urlh:
                             continue
-                        asset_url = urlh.geturl()
+                        asset_url = urlh.url
                     asset_url = asset_url.replace('_drmnp.ism/', '_unpnp.ism/')
                     for i in range(3, 0, -1):
                         asset_url = asset_url = asset_url.replace('_sd1/', '_sd%d/' % i)
diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py
index 3d36edbbc..25f867a60 100644
--- a/yt_dlp/extractor/slideslive.py
+++ b/yt_dlp/extractor/slideslive.py
@@ -426,7 +426,7 @@ def _real_extract(self, url):
             video_id, headers=traverse_obj(parse_qs(url), {
                 'Referer': ('embed_parent_url', -1),
                 'Origin': ('embed_container_origin', -1)}))
-        redirect_url = urlh.geturl()
+        redirect_url = urlh.url
         if 'domain_not_allowed' in redirect_url:
             domain = traverse_obj(parse_qs(redirect_url), ('allowed_domains[]', ...), get_all=False)
             if not domain:
diff --git a/yt_dlp/extractor/sonyliv.py b/yt_dlp/extractor/sonyliv.py
index 5ebe20df7..437957259 100644
--- a/yt_dlp/extractor/sonyliv.py
+++ b/yt_dlp/extractor/sonyliv.py
@@ -6,7 +6,7 @@
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -123,12 +123,12 @@ def _call_api(self, version, path, video_id):
                 'https://apiv2.sonyliv.com/AGL/%s/A/ENG/WEB/%s' % (version, path),
                 video_id, headers=self._HEADERS)['resultObj']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 406 and self._parse_json(
-                    e.cause.read().decode(), video_id)['message'] == 'Please subscribe to watch this content':
+            if isinstance(e.cause, HTTPError) and e.cause.status == 406 and self._parse_json(
+                    e.cause.response.read().decode(), video_id)['message'] == 'Please subscribe to watch this content':
                 self.raise_login_required(self._LOGIN_HINT, method=None)
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 message = self._parse_json(
-                    e.cause.read().decode(), video_id)['message']
+                    e.cause.response.read().decode(), video_id)['message']
                 if message == 'Geoblocked Country':
                     self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
                 raise ExtractorError(message)
diff --git a/yt_dlp/extractor/soundcloud.py b/yt_dlp/extractor/soundcloud.py
index 979f23f44..a7c2afd49 100644
--- a/yt_dlp/extractor/soundcloud.py
+++ b/yt_dlp/extractor/soundcloud.py
@@ -7,15 +7,13 @@
     InfoExtractor,
     SearchInfoExtractor
 )
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
+from ..compat import compat_str
+from ..networking import HEADRequest, Request
+from ..networking.exceptions import HTTPError
 from ..utils import (
     error_to_compat_str,
     ExtractorError,
     float_or_none,
-    HEADRequest,
     int_or_none,
     KNOWN_EXTENSIONS,
     mimetype2ext,
@@ -26,7 +24,6 @@
     update_url_query,
     url_or_none,
     urlhandle_detect_ext,
-    sanitized_Request,
 )
 
 
@@ -103,7 +100,7 @@ def _download_json(self, *args, **kwargs):
             try:
                 return super()._download_json(*args, **kwargs)
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
+                if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
                     self._store_client_id(None)
                     self._update_client_id()
                     continue
@@ -123,7 +120,7 @@ def _perform_login(self, username, password):
         self._access_token = password
         query = self._API_AUTH_QUERY_TEMPLATE % self._CLIENT_ID
         payload = {'session': {'access_token': self._access_token}}
-        token_verification = sanitized_Request(self._API_VERIFY_AUTH_TOKEN % query, json.dumps(payload).encode('utf-8'))
+        token_verification = Request(self._API_VERIFY_AUTH_TOKEN % query, json.dumps(payload).encode('utf-8'))
         response = self._download_json(token_verification, None, note='Verifying login token...', fatal=False)
         if response is not False:
             self._HEADERS = {'Authorization': 'OAuth ' + self._access_token}
@@ -212,7 +209,7 @@ def _extract_info_dict(self, info, full_title=None, secret_token=None, extract_f
                 urlh = self._request_webpage(
                     HEADRequest(redirect_url), track_id, fatal=False)
                 if urlh:
-                    format_url = urlh.geturl()
+                    format_url = urlh.url
                     format_urls.add(format_url)
                     formats.append({
                         'format_id': 'download',
@@ -669,7 +666,7 @@ def _entries(self, url, playlist_id):
                 except ExtractorError as e:
                     # Downloading page may result in intermittent 502 HTTP error
                     # See https://github.com/yt-dlp/yt-dlp/issues/872
-                    if not isinstance(e.cause, compat_HTTPError) or e.cause.code != 502:
+                    if not isinstance(e.cause, HTTPError) or e.cause.status != 502:
                         raise
                     retry.error = e
                     continue
diff --git a/yt_dlp/extractor/teachable.py b/yt_dlp/extractor/teachable.py
index c212a4926..01906bda9 100644
--- a/yt_dlp/extractor/teachable.py
+++ b/yt_dlp/extractor/teachable.py
@@ -56,7 +56,7 @@ def is_logged(webpage):
             self._logged_in = True
             return
 
-        login_url = urlh.geturl()
+        login_url = urlh.url
 
         login_form = self._hidden_inputs(login_page)
 
diff --git a/yt_dlp/extractor/telemundo.py b/yt_dlp/extractor/telemundo.py
index 88f29cb83..54e74a6c0 100644
--- a/yt_dlp/extractor/telemundo.py
+++ b/yt_dlp/extractor/telemundo.py
@@ -1,9 +1,6 @@
 from .common import InfoExtractor
-from ..utils import (
-    try_get,
-    unified_timestamp,
-    HEADRequest,
-)
+from ..networking import HEADRequest
+from ..utils import try_get, unified_timestamp
 
 
 class TelemundoIE(InfoExtractor):
@@ -38,7 +35,7 @@ def _real_extract(self, url):
 
         m3u8_url = self._request_webpage(HEADRequest(
             redirect_url + '?format=redirect&manifest=m3u&format=redirect&Tracking=true&Embedded=true&formats=MPEG4'),
-            video_id, 'Processing m3u8').geturl()
+            video_id, 'Processing m3u8').url
         formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
         date = unified_timestamp(try_get(
             metadata, lambda x: x['props']['initialState']['video']['associatedPlaylists'][0]['videos'][0]['datePublished'].split(' ', 1)[1]))
diff --git a/yt_dlp/extractor/tennistv.py b/yt_dlp/extractor/tennistv.py
index bc64226bf..c1b4a3312 100644
--- a/yt_dlp/extractor/tennistv.py
+++ b/yt_dlp/extractor/tennistv.py
@@ -86,7 +86,7 @@ def _perform_login(self, username, password):
             })
 
         self.get_token(None, {
-            'code': urllib.parse.parse_qs(handle.geturl())['code'][-1],
+            'code': urllib.parse.parse_qs(handle.url)['code'][-1],
             'grant_type': 'authorization_code',
             'client_id': 'tennis-tv-web',
             'redirect_uri': 'https://www.tennistv.com/resources/v1.1.10/html/silent-check-sso.html'
diff --git a/yt_dlp/extractor/tenplay.py b/yt_dlp/extractor/tenplay.py
index 633032e31..c7097cf02 100644
--- a/yt_dlp/extractor/tenplay.py
+++ b/yt_dlp/extractor/tenplay.py
@@ -2,11 +2,8 @@
 import base64
 
 from .common import InfoExtractor
-from ..utils import (
-    HEADRequest,
-    int_or_none,
-    urlencode_postdata,
-)
+from ..networking import HEADRequest
+from ..utils import int_or_none, urlencode_postdata
 
 
 class TenPlayIE(InfoExtractor):
@@ -94,7 +91,7 @@ def _real_extract(self, url):
             data.get('playbackApiEndpoint'), content_id, 'Downloading video JSON',
             headers=headers).get('source')
         m3u8_url = self._request_webpage(HEADRequest(
-            _video_url), content_id).geturl()
+            _video_url), content_id).url
         if '10play-not-in-oz' in m3u8_url:
             self.raise_geo_restricted(countries=['AU'])
         formats = self._extract_m3u8_formats(m3u8_url, content_id, 'mp4')
diff --git a/yt_dlp/extractor/tfo.py b/yt_dlp/extractor/tfo.py
index a24789cb3..d417f50e1 100644
--- a/yt_dlp/extractor/tfo.py
+++ b/yt_dlp/extractor/tfo.py
@@ -1,12 +1,8 @@
 import json
 
 from .common import InfoExtractor
-from ..utils import (
-    HEADRequest,
-    ExtractorError,
-    int_or_none,
-    clean_html,
-)
+from ..networking import HEADRequest
+from ..utils import ExtractorError, clean_html, int_or_none
 
 
 class TFOIE(InfoExtractor):
diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py
index e659b8ee1..537f6f6cd 100644
--- a/yt_dlp/extractor/theplatform.py
+++ b/yt_dlp/extractor/theplatform.py
@@ -7,13 +7,13 @@
 
 from .once import OnceIE
 from .adobepass import AdobePassIE
+from ..networking import Request
 from ..utils import (
     determine_ext,
     ExtractorError,
     float_or_none,
     int_or_none,
     parse_qs,
-    sanitized_Request,
     unsmuggle_url,
     update_url_query,
     xpath_with_ns,
@@ -270,7 +270,7 @@ def _real_extract(self, url):
             source_url = smuggled_data.get('source_url')
             if source_url:
                 headers['Referer'] = source_url
-            request = sanitized_Request(url, headers=headers)
+            request = Request(url, headers=headers)
             webpage = self._download_webpage(request, video_id)
             smil_url = self._search_regex(
                 r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
diff --git a/yt_dlp/extractor/thisoldhouse.py b/yt_dlp/extractor/thisoldhouse.py
index 55b6413ae..cc7beeea5 100644
--- a/yt_dlp/extractor/thisoldhouse.py
+++ b/yt_dlp/extractor/thisoldhouse.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..utils import HEADRequest
+from ..networking import HEADRequest
 
 
 class ThisOldHouseIE(InfoExtractor):
@@ -50,6 +50,6 @@ def _real_extract(self, url):
             r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?thisoldhouse\.(?:chorus\.build|com)/videos/zype/([0-9a-f]{24})[^\'"]*)[\'"]',
             webpage, 'video url')
         if 'subscription_required=true' in video_url or 'c-entry-group-labels__image' in webpage:
-            return self.url_result(self._request_webpage(HEADRequest(video_url), display_id).geturl(), 'Zype', display_id)
+            return self.url_result(self._request_webpage(HEADRequest(video_url), display_id).url, 'Zype', display_id)
         video_id = self._search_regex(r'(?:https?:)?//(?:www\.)?thisoldhouse\.(?:chorus\.build|com)/videos/zype/([0-9a-f]{24})', video_url, 'video id')
         return self.url_result(self._ZYPE_TMPL % video_id, 'Zype', video_id)
diff --git a/yt_dlp/extractor/threeqsdn.py b/yt_dlp/extractor/threeqsdn.py
index b1041902b..7841f8da6 100644
--- a/yt_dlp/extractor/threeqsdn.py
+++ b/yt_dlp/extractor/threeqsdn.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -90,7 +90,7 @@ def _real_extract(self, url):
             config = self._download_json(
                 url.replace('://playout.3qsdn.com/', '://playout.3qsdn.com/config/'), video_id)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 self.raise_geo_restricted()
             raise
 
diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index 2f491c317..48de61f93 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -7,9 +7,9 @@
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote, compat_urllib_parse_urlparse
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     LazyList,
     UnsupportedError,
     UserNotLive,
@@ -1084,7 +1084,7 @@ class TikTokVMIE(InfoExtractor):
 
     def _real_extract(self, url):
         new_url = self._request_webpage(
-            HEADRequest(url), self._match_id(url), headers={'User-Agent': 'facebookexternalhit/1.1'}).geturl()
+            HEADRequest(url), self._match_id(url), headers={'User-Agent': 'facebookexternalhit/1.1'}).url
         if self.suitable(new_url):  # Prevent infinite loop in case redirect fails
             raise UnsupportedError(new_url)
         return self.url_result(new_url)
diff --git a/yt_dlp/extractor/toutv.py b/yt_dlp/extractor/toutv.py
index f60c199f0..ced1224fa 100644
--- a/yt_dlp/extractor/toutv.py
+++ b/yt_dlp/extractor/toutv.py
@@ -1,7 +1,7 @@
 import json
 
 from .radiocanada import RadioCanadaIE
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -52,8 +52,8 @@ def _perform_login(self, username, password):
                     'Content-Type': 'application/json;charset=utf-8',
                 })['access_token']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                error = self._parse_json(e.cause.read().decode(), None)['Message']
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                error = self._parse_json(e.cause.response.read().decode(), None)['Message']
                 raise ExtractorError(error, expected=True)
             raise
         self._claims = self._call_api('validation/v2/getClaims')['claims']
diff --git a/yt_dlp/extractor/triller.py b/yt_dlp/extractor/triller.py
index 6a4dadb9b..c5d01c827 100644
--- a/yt_dlp/extractor/triller.py
+++ b/yt_dlp/extractor/triller.py
@@ -3,9 +3,9 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     UnsupportedError,
     determine_ext,
     int_or_none,
@@ -327,7 +327,7 @@ class TrillerShortIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        real_url = self._request_webpage(HEADRequest(url), self._match_id(url)).geturl()
+        real_url = self._request_webpage(HEADRequest(url), self._match_id(url)).url
         if self.suitable(real_url):  # Prevent infinite loop in case redirect fails
             raise UnsupportedError(real_url)
         return self.url_result(real_url)
diff --git a/yt_dlp/extractor/trueid.py b/yt_dlp/extractor/trueid.py
index 696343627..86f0990e8 100644
--- a/yt_dlp/extractor/trueid.py
+++ b/yt_dlp/extractor/trueid.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -88,9 +88,9 @@ def _real_extract(self, url):
             stream_data = self._download_json(
                 f'https://{domain}/cmsPostProxy/contents/video/{video_id}/streamer?os=android', video_id, data=b'')['data']
         except ExtractorError as e:
-            if not isinstance(e.cause, compat_HTTPError):
+            if not isinstance(e.cause, HTTPError):
                 raise e
-            errmsg = self._parse_json(e.cause.read().decode(), video_id)['meta']['message']
+            errmsg = self._parse_json(e.cause.response.read().decode(), video_id)['meta']['message']
             if 'country' in errmsg:
                 self.raise_geo_restricted(
                     errmsg, [initial_data['display_country']] if initial_data.get('display_country') else None, True)
diff --git a/yt_dlp/extractor/tubetugraz.py b/yt_dlp/extractor/tubetugraz.py
index 2199fea19..a351e4e55 100644
--- a/yt_dlp/extractor/tubetugraz.py
+++ b/yt_dlp/extractor/tubetugraz.py
@@ -22,7 +22,7 @@ def _perform_login(self, username, password):
             return
 
         content, urlh = self._download_webpage_handle(
-            urlh.geturl(), None, fatal=False, headers={'referer': urlh.geturl()},
+            urlh.url, None, fatal=False, headers={'referer': urlh.url},
             note='logging in', errnote='unable to log in',
             data=urlencode_postdata({
                 'lang': 'de',
@@ -30,7 +30,7 @@ def _perform_login(self, username, password):
                 'j_username': username,
                 'j_password': password
             }))
-        if not urlh or urlh.geturl() == 'https://tube.tugraz.at/paella/ui/index.html':
+        if not urlh or urlh.url == 'https://tube.tugraz.at/paella/ui/index.html':
             return
 
         if not self._html_search_regex(
@@ -40,14 +40,14 @@ def _perform_login(self, username, password):
             return
 
         content, urlh = self._download_webpage_handle(
-            urlh.geturl(), None, fatal=False, headers={'referer': urlh.geturl()},
+            urlh.url, None, fatal=False, headers={'referer': urlh.url},
             note='logging in with TFA', errnote='unable to log in with TFA',
             data=urlencode_postdata({
                 'lang': 'de',
                 '_eventId_proceed': '',
                 'j_tokenNumber': self._get_tfa_info(),
             }))
-        if not urlh or urlh.geturl() == 'https://tube.tugraz.at/paella/ui/index.html':
+        if not urlh or urlh.url == 'https://tube.tugraz.at/paella/ui/index.html':
             return
 
         self.report_warning('unable to login: incorrect TFA code')
diff --git a/yt_dlp/extractor/tubitv.py b/yt_dlp/extractor/tubitv.py
index de8b5da69..bd46bc363 100644
--- a/yt_dlp/extractor/tubitv.py
+++ b/yt_dlp/extractor/tubitv.py
@@ -1,13 +1,13 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import Request
 from ..utils import (
     ExtractorError,
     int_or_none,
     js_to_json,
-    sanitized_Request,
-    urlencode_postdata,
     traverse_obj,
+    urlencode_postdata,
 )
 
 
@@ -72,8 +72,8 @@ def _perform_login(self, username, password):
             'password': password,
         }
         payload = urlencode_postdata(form_data)
-        request = sanitized_Request(self._LOGIN_URL, payload)
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        request = Request(self._LOGIN_URL, payload)
+        request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
         login_page = self._download_webpage(
             request, None, False, 'Wrong login info')
         if not re.search(r'id="tubi-logout"', login_page):
diff --git a/yt_dlp/extractor/tumblr.py b/yt_dlp/extractor/tumblr.py
index 88d4ae32d..a26bdcaae 100644
--- a/yt_dlp/extractor/tumblr.py
+++ b/yt_dlp/extractor/tumblr.py
@@ -274,7 +274,7 @@ def _real_extract(self, url):
         url = f'http://{blog}.tumblr.com/post/{video_id}/'
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
-        redirect_url = urlh.geturl()
+        redirect_url = urlh.url
 
         api_only = bool(self._search_regex(
             r'(tumblr.com|^)/(safe-mode|login_required|blog/view)',
diff --git a/yt_dlp/extractor/tunein.py b/yt_dlp/extractor/tunein.py
index e02121bd8..fd2fe132c 100644
--- a/yt_dlp/extractor/tunein.py
+++ b/yt_dlp/extractor/tunein.py
@@ -225,10 +225,10 @@ def _real_extract(self, url):
         urlh = self._request_webpage(
             url, redirect_id, note='Downloading redirect page')
 
-        url = urlh.geturl()
+        url = urlh.url
         url_parsed = urllib.parse.urlparse(url)
         if url_parsed.port == 443:
-            url = url_parsed._replace(netloc=url_parsed.hostname).geturl()
+            url = url_parsed._replace(netloc=url_parsed.hostname).url
 
         self.to_screen('Following redirect: %s' % url)
         return self.url_result(url)
diff --git a/yt_dlp/extractor/tv2.py b/yt_dlp/extractor/tv2.py
index c51e63371..f6b452dc8 100644
--- a/yt_dlp/extractor/tv2.py
+++ b/yt_dlp/extractor/tv2.py
@@ -1,7 +1,7 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -57,8 +57,8 @@ def _real_extract(self, url):
                                            headers={'content-type': 'application/json'},
                                            data='{"device":{"id":"1-1-1","name":"Nettleser (HTML)"}}'.encode())['playback']
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                    error = self._parse_json(e.cause.read().decode(), video_id)['error']
+                if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                    error = self._parse_json(e.cause.response.read().decode(), video_id)['error']
                     error_code = error.get('code')
                     if error_code == 'ASSET_PLAYBACK_INVALID_GEO_LOCATION':
                         self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
@@ -211,8 +211,8 @@ def _real_extract(self, url):
                     api_base + '/play.json?protocol=%s&videoFormat=SMIL+ISMUSP' % protocol,
                     video_id, 'Downloading play JSON')['playback']
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                    error = self._parse_json(e.cause.read().decode(), video_id)['error']
+                if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                    error = self._parse_json(e.cause.response.read().decode(), video_id)['error']
                     error_code = error.get('code')
                     if error_code == 'ASSET_PLAYBACK_INVALID_GEO_LOCATION':
                         self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
diff --git a/yt_dlp/extractor/tvp.py b/yt_dlp/extractor/tvp.py
index c686044fa..2aa0dd870 100644
--- a/yt_dlp/extractor/tvp.py
+++ b/yt_dlp/extractor/tvp.py
@@ -488,9 +488,9 @@ def _call_api(self, resource, video_id, query={}, **kwargs):
             f'{self._API_BASE_URL}/{resource}', video_id,
             query={'lang': 'pl', 'platform': 'BROWSER', **query},
             expected_status=lambda x: is_valid(x) or 400 <= x < 500, **kwargs)
-        if is_valid(urlh.getcode()):
+        if is_valid(urlh.status):
             return document
-        raise ExtractorError(f'Woronicza said: {document.get("code")} (HTTP {urlh.getcode()})')
+        raise ExtractorError(f'Woronicza said: {document.get("code")} (HTTP {urlh.status})')
 
     def _parse_video(self, video, with_url=True):
         info_dict = traverse_obj(video, {
diff --git a/yt_dlp/extractor/tvplay.py b/yt_dlp/extractor/tvplay.py
index e056f9872..48a6efe1c 100644
--- a/yt_dlp/extractor/tvplay.py
+++ b/yt_dlp/extractor/tvplay.py
@@ -1,10 +1,8 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -129,8 +127,8 @@ def _real_extract(self, url):
                 'http://playapi.mtgx.tv/v3/videos/stream/%s' % video_id,
                 video_id, 'Downloading streams JSON')
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                msg = self._parse_json(e.cause.read().decode('utf-8'), video_id)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                msg = self._parse_json(e.cause.response.read().decode('utf-8'), video_id)
                 raise ExtractorError(msg['msg'], expected=True)
             raise
 
diff --git a/yt_dlp/extractor/tvplayer.py b/yt_dlp/extractor/tvplayer.py
index b05355f87..228c2366e 100644
--- a/yt_dlp/extractor/tvplayer.py
+++ b/yt_dlp/extractor/tvplayer.py
@@ -1,8 +1,6 @@
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     extract_attributes,
     try_get,
@@ -64,9 +62,9 @@ def _real_extract(self, url):
                     'validate': validate,
                 }))['tvplayer']['response']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
+            if isinstance(e.cause, HTTPError):
                 response = self._parse_json(
-                    e.cause.read().decode(), resource_id)['tvplayer']['response']
+                    e.cause.response.read().decode(), resource_id)['tvplayer']['response']
                 raise ExtractorError(
                     '%s said: %s' % (self.IE_NAME, response['error']), expected=True)
             raise
diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py
index 2548dae04..dff353a4f 100644
--- a/yt_dlp/extractor/twitcasting.py
+++ b/yt_dlp/extractor/twitcasting.py
@@ -107,9 +107,9 @@ def _real_extract(self, url):
                 url, video_id, data=request_data,
                 headers={'Origin': 'https://twitcasting.tv'},
                 note='Trying video password')
-        if urlh.geturl() != url and request_data:
+        if urlh.url != url and request_data:
             webpage = self._download_webpage(
-                urlh.geturl(), video_id, data=request_data,
+                urlh.url, video_id, data=request_data,
                 headers={'Origin': 'https://twitcasting.tv'},
                 note='Retrying authentication')
         # has to check here as the first request can contain password input form even if the password is correct
diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index c8ee52014..3297ef091 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -71,7 +71,7 @@ def login_step(page, urlh, note, data):
             form = self._hidden_inputs(page)
             form.update(data)
 
-            page_url = urlh.geturl()
+            page_url = urlh.url
             post_url = self._search_regex(
                 r'<form[^>]+action=(["\'])(?P<url>.+?)\1', page,
                 'post url', default=self._LOGIN_POST_URL, group='url')
diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index fc157ac22..4015277a8 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1596,7 +1596,7 @@ def _real_extract(self, url):
         if eid:
             id = eid
             url = self._BASE_URL + id
-        new_url = self._request_webpage(url, id, headers={'User-Agent': 'curl'}).geturl()
+        new_url = self._request_webpage(url, id, headers={'User-Agent': 'curl'}).url
         __UNSAFE_LINK = "https://twitter.com/safety/unsafe_link_warning?unsafe_link="
         if new_url.startswith(__UNSAFE_LINK):
             new_url = new_url.replace(__UNSAFE_LINK, "")
diff --git a/yt_dlp/extractor/udemy.py b/yt_dlp/extractor/udemy.py
index 329e5da2d..5c296051a 100644
--- a/yt_dlp/extractor/udemy.py
+++ b/yt_dlp/extractor/udemy.py
@@ -1,8 +1,9 @@
 import re
-import urllib.request
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str, compat_urlparse
+from ..compat import compat_str, compat_urlparse
+from ..networking import Request
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -10,7 +11,6 @@
     float_or_none,
     int_or_none,
     js_to_json,
-    sanitized_Request,
     smuggle_url,
     try_get,
     unescapeHTML,
@@ -153,11 +153,10 @@ def _download_json(self, url_or_request, *args, **kwargs):
                 headers['X-Udemy-Bearer-Token'] = cookie.value
                 headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value
 
-        if isinstance(url_or_request, urllib.request.Request):
-            for header, value in headers.items():
-                url_or_request.add_header(header, value)
+        if isinstance(url_or_request, Request):
+            url_or_request.headers.update(headers)
         else:
-            url_or_request = sanitized_Request(url_or_request, headers=headers)
+            url_or_request = Request(url_or_request, headers=headers)
 
         response = super(UdemyIE, self)._download_json(url_or_request, *args, **kwargs)
         self._handle_error(response)
@@ -212,7 +211,7 @@ def _real_extract(self, url):
             lecture = self._download_lecture(course_id, lecture_id)
         except ExtractorError as e:
             # Error could possibly mean we are not enrolled in the course
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 webpage = webpage or self._download_webpage(url, lecture_id)
                 self._enroll_course(url, webpage, course_id)
                 lecture = self._download_lecture(course_id, lecture_id)
diff --git a/yt_dlp/extractor/vevo.py b/yt_dlp/extractor/vevo.py
index da4ce49ca..aa40227a7 100644
--- a/yt_dlp/extractor/vevo.py
+++ b/yt_dlp/extractor/vevo.py
@@ -2,10 +2,8 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_HTTPError,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -184,8 +182,8 @@ def _call_api(self, path, *args, **kwargs):
         try:
             data = self._download_json(self._api_url_template % path, *args, **kwargs)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
-                errors = self._parse_json(e.cause.read().decode(), None)['errors']
+            if isinstance(e.cause, HTTPError):
+                errors = self._parse_json(e.cause.response.read().decode(), None)['errors']
                 error_message = ', '.join([error['message'] for error in errors])
                 raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
             raise
diff --git a/yt_dlp/extractor/vice.py b/yt_dlp/extractor/vice.py
index d1a3b48aa..8a7126853 100644
--- a/yt_dlp/extractor/vice.py
+++ b/yt_dlp/extractor/vice.py
@@ -7,10 +7,8 @@
 from .adobepass import AdobePassIE
 from .common import InfoExtractor
 from .youtube import YoutubeIE
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -140,8 +138,8 @@ def _real_extract(self, url):
                 'https://vms.vice.com/%s/video/preplay/%s' % (locale, video_id),
                 video_id, query=query)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401):
-                error = json.loads(e.cause.read().decode())
+            if isinstance(e.cause, HTTPError) and e.cause.status in (400, 401):
+                error = json.loads(e.cause.response.read().decode())
                 error_message = error.get('error_description') or error['details']
                 raise ExtractorError('%s said: %s' % (
                     self.IE_NAME, error_message), expected=True)
diff --git a/yt_dlp/extractor/videocampus_sachsen.py b/yt_dlp/extractor/videocampus_sachsen.py
index 982ab3dd0..37bc7d718 100644
--- a/yt_dlp/extractor/videocampus_sachsen.py
+++ b/yt_dlp/extractor/videocampus_sachsen.py
@@ -2,7 +2,7 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import ExtractorError, OnDemandPagedList, urlencode_postdata
 
 
@@ -169,7 +169,7 @@ def _real_extract(self, url):
                 f'https://{host}/media/hlsMedium/key/{video_id}/format/auto/ext/mp4/learning/0/path/m3u8',
                 video_id, 'mp4', m3u8_id='hls', fatal=True)
         except ExtractorError as e:
-            if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (404, 500):
+            if not isinstance(e.cause, HTTPError) or e.cause.status not in (404, 500):
                 raise
 
         formats.append({'url': f'https://{host}/getMedium/{video_id}.mp4'})
diff --git a/yt_dlp/extractor/vidio.py b/yt_dlp/extractor/vidio.py
index 23e1aaf20..770aa284d 100644
--- a/yt_dlp/extractor/vidio.py
+++ b/yt_dlp/extractor/vidio.py
@@ -39,7 +39,7 @@ def is_logged_in():
         login_post, login_post_urlh = self._download_webpage_handle(
             self._LOGIN_URL, None, 'Logging in', data=urlencode_postdata(login_form), expected_status=[302, 401])
 
-        if login_post_urlh.getcode() == 401:
+        if login_post_urlh.status == 401:
             if get_element_by_class('onboarding-content-register-popup__title', login_post):
                 raise ExtractorError(
                     'Unable to log in: The provided email has not registered yet.', expected=True)
diff --git a/yt_dlp/extractor/vidlii.py b/yt_dlp/extractor/vidlii.py
index cde4274d9..44353b7fc 100644
--- a/yt_dlp/extractor/vidlii.py
+++ b/yt_dlp/extractor/vidlii.py
@@ -1,8 +1,8 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     format_field,
     float_or_none,
     get_element_by_id,
diff --git a/yt_dlp/extractor/viewlift.py b/yt_dlp/extractor/viewlift.py
index 381260114..8f686f05d 100644
--- a/yt_dlp/extractor/viewlift.py
+++ b/yt_dlp/extractor/viewlift.py
@@ -1,7 +1,7 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -46,8 +46,8 @@ def _call_api(self, site, path, video_id, url, query):
             return self._download_json(
                 self._API_BASE + path, video_id, headers={'Authorization': self._TOKENS.get(site)}, query=query)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                webpage = e.cause.read().decode()
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                webpage = e.cause.response.read().decode()
                 try:
                     error_message = traverse_obj(json.loads(webpage), 'errorMessage', 'message')
                 except json.JSONDecodeError:
diff --git a/yt_dlp/extractor/viidea.py b/yt_dlp/extractor/viidea.py
index 4cdf2677b..649ffe395 100644
--- a/yt_dlp/extractor/viidea.py
+++ b/yt_dlp/extractor/viidea.py
@@ -2,10 +2,10 @@
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_HTTPError,
     compat_str,
     compat_urlparse,
 )
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     js_to_json,
@@ -133,9 +133,9 @@ def _real_extract(self, url):
                 '%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
                 lecture_id)['lecture'][0]
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 msg = self._parse_json(
-                    e.cause.read().decode('utf-8'), lecture_id)
+                    e.cause.response.read().decode('utf-8'), lecture_id)
                 raise ExtractorError(msg['detail'], expected=True)
             raise
 
diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py
index d81d9c551..e72fa50fa 100644
--- a/yt_dlp/extractor/vimeo.py
+++ b/yt_dlp/extractor/vimeo.py
@@ -2,20 +2,16 @@
 import functools
 import re
 import itertools
-import urllib.error
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-    compat_urlparse,
-)
+from ..compat import compat_str, compat_urlparse
+from ..networking import HEADRequest, Request
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     determine_ext,
     ExtractorError,
     get_element_by_class,
-    HEADRequest,
     js_to_json,
     int_or_none,
     merge_dicts,
@@ -23,7 +19,6 @@
     parse_filesize,
     parse_iso8601,
     parse_qs,
-    sanitized_Request,
     smuggle_url,
     str_or_none,
     try_get,
@@ -72,7 +67,7 @@ def _perform_login(self, username, password):
                     'Referer': self._LOGIN_URL,
                 })
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 418:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 418:
                 raise ExtractorError(
                     'Unable to log in: bad username or password',
                     expected=True)
@@ -809,7 +804,7 @@ def _try_album_password(self, url):
                         'X-Requested-With': 'XMLHttpRequest',
                     })
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                     raise ExtractorError('Wrong password', expected=True)
                 raise
 
@@ -832,10 +827,10 @@ def _real_extract(self, url):
             # Retrieve video webpage to extract further information
             webpage, urlh = self._download_webpage_handle(
                 url, video_id, headers=headers)
-            redirect_url = urlh.geturl()
+            redirect_url = urlh.url
         except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                errmsg = ee.cause.read()
+            if isinstance(ee.cause, HTTPError) and ee.cause.status == 403:
+                errmsg = ee.cause.response.read()
                 if b'Because of its privacy settings, this video cannot be played here' in errmsg:
                     raise ExtractorError(
                         'Cannot download embed-only video without embedding '
@@ -1154,7 +1149,7 @@ def _fetch_page(self, album_id, authorization, hashed_pass, page):
                     'Authorization': 'jwt ' + authorization,
                 })['data']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                 return
         for video in videos:
             link = video.get('link')
@@ -1196,7 +1191,7 @@ def _real_extract(self, url):
                         'X-Requested-With': 'XMLHttpRequest',
                     })['hashed_pass']
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                     raise ExtractorError('Wrong password', expected=True)
                 raise
         entries = OnDemandPagedList(functools.partial(
@@ -1309,10 +1304,10 @@ class VimeoWatchLaterIE(VimeoChannelIE):  # XXX: Do not subclass from concrete I
 
     def _page_url(self, base_url, pagenum):
         url = '%s/page:%d/' % (base_url, pagenum)
-        request = sanitized_Request(url)
+        request = Request(url)
         # Set the header to get a partial html page with the ids,
         # the normal page doesn't contain them.
-        request.add_header('X-Requested-With', 'XMLHttpRequest')
+        request.headers['X-Requested-With'] = 'XMLHttpRequest'
         return request
 
     def _real_extract(self, url):
@@ -1432,7 +1427,7 @@ def _real_extract(self, url):
                     **self._hidden_inputs(password_form),
                 }), note='Logging in with video password')
             except ExtractorError as e:
-                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 418:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 418:
                     raise ExtractorError('Wrong video password', expected=True)
                 raise
 
diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py
index 575369028..6b7379d46 100644
--- a/yt_dlp/extractor/vk.py
+++ b/yt_dlp/extractor/vk.py
@@ -36,7 +36,7 @@ class VKBaseIE(InfoExtractor):
 
     def _download_webpage_handle(self, url_or_request, video_id, *args, fatal=True, **kwargs):
         response = super()._download_webpage_handle(url_or_request, video_id, *args, fatal=fatal, **kwargs)
-        challenge_url, cookie = response[1].geturl() if response else '', None
+        challenge_url, cookie = response[1].url if response else '', None
         if challenge_url.startswith('https://vk.com/429.html?'):
             cookie = self._get_cookies(challenge_url).get('hash429')
         if not cookie:
diff --git a/yt_dlp/extractor/vocaroo.py b/yt_dlp/extractor/vocaroo.py
index 704e25c22..d98fbfd2d 100644
--- a/yt_dlp/extractor/vocaroo.py
+++ b/yt_dlp/extractor/vocaroo.py
@@ -1,8 +1,6 @@
 from .common import InfoExtractor
-from ..utils import (
-    HEADRequest,
-    float_or_none,
-)
+from ..networking import HEADRequest
+from ..utils import float_or_none
 
 
 class VocarooIE(InfoExtractor):
diff --git a/yt_dlp/extractor/vodlocker.py b/yt_dlp/extractor/vodlocker.py
index 1c7236ed3..b215d6c9d 100644
--- a/yt_dlp/extractor/vodlocker.py
+++ b/yt_dlp/extractor/vodlocker.py
@@ -1,10 +1,6 @@
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    NO_DEFAULT,
-    sanitized_Request,
-    urlencode_postdata,
-)
+from ..networking import Request
+from ..utils import NO_DEFAULT, ExtractorError, urlencode_postdata
 
 
 class VodlockerIE(InfoExtractor):
@@ -37,8 +33,8 @@ def _real_extract(self, url):
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!
             post = urlencode_postdata(fields)
-            req = sanitized_Request(url, post)
-            req.add_header('Content-type', 'application/x-www-form-urlencoded')
+            req = Request(url, post)
+            req.headers['Content-type'] = 'application/x-www-form-urlencoded'
             webpage = self._download_webpage(
                 req, video_id, 'Downloading video page')
 
diff --git a/yt_dlp/extractor/voot.py b/yt_dlp/extractor/voot.py
index dd41647aa..b19a27934 100644
--- a/yt_dlp/extractor/voot.py
+++ b/yt_dlp/extractor/voot.py
@@ -1,10 +1,10 @@
 import json
 import time
-import urllib.error
 import uuid
 
 from .common import InfoExtractor
 from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -140,7 +140,7 @@ def _real_extract(self, url):
                     'voottoken': self._TOKEN,
                 })['m3u8']
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 400:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                 self._check_token_expiry()
             raise
 
diff --git a/yt_dlp/extractor/vrt.py b/yt_dlp/extractor/vrt.py
index 005835712..497233d95 100644
--- a/yt_dlp/extractor/vrt.py
+++ b/yt_dlp/extractor/vrt.py
@@ -1,10 +1,10 @@
 import functools
 import json
 import time
-import urllib.error
 import urllib.parse
 
 from .gigya import GigyaBaseIE
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     clean_html,
@@ -263,7 +263,7 @@ def _perform_login(self, username, password):
                         '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
                     }))
             except ExtractorError as e:
-                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                     retry.error = e
                     continue
                 raise
diff --git a/yt_dlp/extractor/vrv.py b/yt_dlp/extractor/vrv.py
index ad9dc568a..523c442e6 100644
--- a/yt_dlp/extractor/vrv.py
+++ b/yt_dlp/extractor/vrv.py
@@ -8,7 +8,8 @@
 import urllib.parse
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_urllib_parse_urlencode
+from ..compat import compat_urllib_parse_urlencode
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -54,8 +55,8 @@ def _call_api(self, path, video_id, note, data=None):
                 '?'.join([base_url, encoded_query]), video_id,
                 note='Downloading %s JSON metadata' % note, headers=headers, data=data)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                raise ExtractorError(json.loads(e.cause.read().decode())['message'], expected=True)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                raise ExtractorError(json.loads(e.cause.response.read().decode())['message'], expected=True)
             raise
 
     def _call_cms(self, path, video_id, note):
diff --git a/yt_dlp/extractor/weibo.py b/yt_dlp/extractor/weibo.py
index 81a23b9df..bc9a71abe 100644
--- a/yt_dlp/extractor/weibo.py
+++ b/yt_dlp/extractor/weibo.py
@@ -31,7 +31,7 @@ def _real_extract(self, url):
         # to get Referer url for genvisitor
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
-        visitor_url = urlh.geturl()
+        visitor_url = urlh.url
 
         if 'passport.weibo.com' in visitor_url:
             # first visit
diff --git a/yt_dlp/extractor/weverse.py b/yt_dlp/extractor/weverse.py
index 8f2a7ee06..9a08b8e43 100644
--- a/yt_dlp/extractor/weverse.py
+++ b/yt_dlp/extractor/weverse.py
@@ -5,13 +5,13 @@
 import json
 import re
 import time
-import urllib.error
 import urllib.parse
 import uuid
 
 from .common import InfoExtractor
 from .naver import NaverBaseIE
 from .youtube import YoutubeIE
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     UserNotLive,
@@ -59,7 +59,7 @@ def _perform_login(self, username, password):
                     'password': password,
                 }, separators=(',', ':')).encode(), headers=headers, note='Logging in')
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 raise ExtractorError('Invalid password provided', expected=True)
             raise
 
@@ -97,10 +97,10 @@ def _call_api(self, ep, video_id, data=None, note='Downloading API JSON'):
                     'wmd': wmd,
                 })
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 self.raise_login_required(
                     'Session token has expired. Log in again or refresh cookies in browser')
-            elif isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+            elif isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 raise ExtractorError('Your account does not have access to this content', expected=True)
             raise
 
diff --git a/yt_dlp/extractor/wistia.py b/yt_dlp/extractor/wistia.py
index 884fa4b5f..bce5e8326 100644
--- a/yt_dlp/extractor/wistia.py
+++ b/yt_dlp/extractor/wistia.py
@@ -1,12 +1,12 @@
 import re
-import urllib.error
 import urllib.parse
 from base64 import b64decode
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     determine_ext,
     float_or_none,
     int_or_none,
@@ -365,7 +365,7 @@ def _real_extract(self, url):
 
         try:
             data = self._download_embed_config('channel', channel_id, url)
-        except (ExtractorError, urllib.error.HTTPError):
+        except (ExtractorError, HTTPError):
             # Some channels give a 403 from the JSON API
             self.report_warning('Failed to download channel data from API, falling back to webpage.')
             webpage = self._download_webpage(f'https://fast.wistia.net/embed/channel/{channel_id}', channel_id)
diff --git a/yt_dlp/extractor/wykop.py b/yt_dlp/extractor/wykop.py
index 0fa6d524d..1d29cc89b 100644
--- a/yt_dlp/extractor/wykop.py
+++ b/yt_dlp/extractor/wykop.py
@@ -1,7 +1,7 @@
 import json
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     format_field,
@@ -43,7 +43,7 @@ def _call_api(self, path, video_id, note='Downloading JSON metadata'):
             try:
                 return self._do_call_api(path, video_id, note, headers={'Authorization': f'Bearer {token}'})
             except ExtractorError as e:
-                if not retrying and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                if not retrying and isinstance(e.cause, HTTPError) and e.cause.status == 403:
                     token = self._get_token(True)
                     continue
                 raise
diff --git a/yt_dlp/extractor/xhamster.py b/yt_dlp/extractor/xhamster.py
index 7af6c8f03..37224799b 100644
--- a/yt_dlp/extractor/xhamster.py
+++ b/yt_dlp/extractor/xhamster.py
@@ -183,7 +183,7 @@ def get_height(s):
                         'height': get_height(quality),
                         'filesize': format_sizes.get(quality),
                         'http_headers': {
-                            'Referer': urlh.geturl(),
+                            'Referer': urlh.url,
                         },
                     })
             xplayer_sources = try_get(
diff --git a/yt_dlp/extractor/xtube.py b/yt_dlp/extractor/xtube.py
index ce4480c7d..db8292589 100644
--- a/yt_dlp/extractor/xtube.py
+++ b/yt_dlp/extractor/xtube.py
@@ -2,12 +2,12 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import Request
 from ..utils import (
     int_or_none,
     js_to_json,
     orderedSet,
     parse_duration,
-    sanitized_Request,
     str_to_int,
     url_or_none,
 )
@@ -186,7 +186,7 @@ def _real_extract(self, url):
 
         entries = []
         for pagenum in itertools.count(1):
-            request = sanitized_Request(
+            request = Request(
                 'http://www.xtube.com/profile/%s/videos/%d' % (user_id, pagenum),
                 headers={
                     'Cookie': 'popunder=4',
diff --git a/yt_dlp/extractor/yesjapan.py b/yt_dlp/extractor/yesjapan.py
index b45fa8f14..94e41660d 100644
--- a/yt_dlp/extractor/yesjapan.py
+++ b/yt_dlp/extractor/yesjapan.py
@@ -1,9 +1,6 @@
 from .common import InfoExtractor
-from ..utils import (
-    HEADRequest,
-    get_element_by_attribute,
-    parse_iso8601,
-)
+from ..networking import HEADRequest
+from ..utils import get_element_by_attribute, parse_iso8601
 
 
 class YesJapanIE(InfoExtractor):
@@ -42,7 +39,7 @@ def _real_extract(self, url):
         req = self._request_webpage(
             redirect_req, video_id, note='Resolving final URL', errnote='Could not resolve final URL', fatal=False)
         if req:
-            video_url = req.geturl()
+            video_url = req.url
 
         formats = [{
             'format_id': 'sd',
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 826bbb20e..2b3776aa1 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -15,13 +15,13 @@
 import threading
 import time
 import traceback
-import urllib.error
 import urllib.parse
 
 from .common import InfoExtractor, SearchInfoExtractor
 from .openload import PhantomJSwrapper
 from ..compat import functools
 from ..jsinterp import JSInterpreter
+from ..networking.exceptions import HTTPError, network_exceptions
 from ..utils import (
     NO_DEFAULT,
     ExtractorError,
@@ -41,7 +41,6 @@
     join_nonempty,
     js_to_json,
     mimetype2ext,
-    network_exceptions,
     orderedSet,
     parse_codecs,
     parse_count,
@@ -959,15 +958,15 @@ def _extract_response(self, item_id, query, note='Downloading API JSON', headers
             except ExtractorError as e:
                 if not isinstance(e.cause, network_exceptions):
                     return self._error_or_warning(e, fatal=fatal)
-                elif not isinstance(e.cause, urllib.error.HTTPError):
+                elif not isinstance(e.cause, HTTPError):
                     retry.error = e
                     continue
 
-                first_bytes = e.cause.read(512)
+                first_bytes = e.cause.response.read(512)
                 if not is_html(first_bytes):
                     yt_error = try_get(
                         self._parse_json(
-                            self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
+                            self._webpage_read_content(e.cause.response, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
                         lambda x: x['error']['message'], str)
                     if yt_error:
                         self._report_alerts([('ERROR', yt_error)], fatal=False)
@@ -975,7 +974,7 @@ def _extract_response(self, item_id, query, note='Downloading API JSON', headers
                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
-                if e.cause.code not in (403, 429):
+                if e.cause.status not in (403, 429):
                     retry.error = e
                     continue
                 return self._error_or_warning(e, fatal=fatal)
@@ -2837,7 +2836,7 @@ def _extract_sequence_from_mpd(refresh_sequence, immediate):
             # Obtain from MPD's maximum seq value
             old_mpd_url = mpd_url
             last_error = ctx.pop('last_error', None)
-            expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
+            expire_fast = immediate or last_error and isinstance(last_error, HTTPError) and last_error.status == 403
             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
                                                or (mpd_url, stream_number, False))
             if not refresh_sequence:
@@ -5263,7 +5262,7 @@ def _extract_webpage(self, url, item_id, fatal=True):
                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
             except ExtractorError as e:
                 if isinstance(e.cause, network_exceptions):
-                    if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
+                    if not isinstance(e.cause, HTTPError) or e.cause.status not in (403, 429):
                         retry.error = e
                         continue
                 self._error_or_warning(e, fatal=fatal)
diff --git a/yt_dlp/extractor/zaiko.py b/yt_dlp/extractor/zaiko.py
index 84cee4445..0ccacbb6a 100644
--- a/yt_dlp/extractor/zaiko.py
+++ b/yt_dlp/extractor/zaiko.py
@@ -16,7 +16,7 @@
 class ZaikoBaseIE(InfoExtractor):
     def _download_real_webpage(self, url, video_id):
         webpage, urlh = self._download_webpage_handle(url, video_id)
-        final_url = urlh.geturl()
+        final_url = urlh.url
         if 'zaiko.io/login' in final_url:
             self.raise_login_required()
         elif '/_buy/' in final_url:
diff --git a/yt_dlp/extractor/zattoo.py b/yt_dlp/extractor/zattoo.py
index 22620c0a3..6bd9ea064 100644
--- a/yt_dlp/extractor/zattoo.py
+++ b/yt_dlp/extractor/zattoo.py
@@ -2,7 +2,8 @@
 from uuid import uuid4
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -36,7 +37,7 @@ def _perform_login(self, username, password):
                     'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                 })
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                 raise ExtractorError(
                     'Unable to login: incorrect username and/or password',
                     expected=True)
diff --git a/yt_dlp/extractor/zype.py b/yt_dlp/extractor/zype.py
index 8cf994505..2f3b4c47f 100644
--- a/yt_dlp/extractor/zype.py
+++ b/yt_dlp/extractor/zype.py
@@ -1,7 +1,7 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     dict_get,
     ExtractorError,
@@ -37,9 +37,9 @@ def _real_extract(self, url):
             response = self._download_json(re.sub(
                 r'\.(?:js|html)\?', '.json?', url), video_id)['response']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 403):
+            if isinstance(e.cause, HTTPError) and e.cause.status in (400, 401, 403):
                 raise ExtractorError(self._parse_json(
-                    e.cause.read().decode(), video_id)['message'], expected=True)
+                    e.cause.response.read().decode(), video_id)['message'], expected=True)
             raise
 
         body = response['body']
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
index e4b362827..458eca39f 100644
--- a/yt_dlp/networking/common.py
+++ b/yt_dlp/networking/common.py
@@ -24,6 +24,7 @@
 from ..utils import (
     bug_reports_message,
     classproperty,
+    deprecation_warning,
     error_to_str,
     escape_url,
     update_url_query,
@@ -507,16 +508,21 @@ def get_header(self, name, default=None):
     # The following methods are for compatability reasons and are deprecated
     @property
     def code(self):
+        deprecation_warning('Response.code is deprecated, use Response.status', stacklevel=2)
         return self.status
 
     def getcode(self):
+        deprecation_warning('Response.getcode() is deprecated, use Response.status', stacklevel=2)
         return self.status
 
     def geturl(self):
+        deprecation_warning('Response.geturl() is deprecated, use Response.url', stacklevel=2)
         return self.url
 
     def info(self):
+        deprecation_warning('Response.info() is deprecated, use Response.headers', stacklevel=2)
         return self.headers
 
     def getheader(self, name, default=None):
+        deprecation_warning('Response.getheader() is deprecated, use Response.get_header', stacklevel=2)
         return self.get_header(name, default)
diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py
index 6fe8afb92..10afc9ccb 100644
--- a/yt_dlp/networking/exceptions.py
+++ b/yt_dlp/networking/exceptions.py
@@ -3,7 +3,7 @@
 import typing
 import urllib.error
 
-from ..utils import YoutubeDLError
+from ..utils import YoutubeDLError, deprecation_warning
 
 if typing.TYPE_CHECKING:
     from .common import RequestHandler, Response
@@ -137,6 +137,7 @@ def reason(self, value):
 
     @property
     def headers(self):
+        deprecation_warning('HTTPError.headers is deprecated, use HTTPError.response.headers instead')
         return self._http_error.response.headers
 
     @headers.setter
@@ -144,16 +145,20 @@ def headers(self, value):
         return
 
     def info(self):
+        deprecation_warning('HTTPError.info() is deprecated, use HTTPError.response.headers instead')
         return self.response.headers
 
     def getcode(self):
+        deprecation_warning('HTTPError.getcode is deprecated, use HTTPError.status instead')
         return self.status
 
     def geturl(self):
+        deprecation_warning('HTTPError.geturl is deprecated, use HTTPError.response.url instead')
         return self.response.url
 
     @property
     def code(self):
+        deprecation_warning('HTTPError.code is deprecated, use HTTPError.status instead')
         return self.status
 
     @code.setter
@@ -162,6 +167,7 @@ def code(self, value):
 
     @property
     def url(self):
+        deprecation_warning('HTTPError.url is deprecated, use HTTPError.response.url instead')
         return self.response.url
 
     @url.setter
@@ -170,6 +176,7 @@ def url(self, value):
 
     @property
     def hdrs(self):
+        deprecation_warning('HTTPError.hdrs is deprecated, use HTTPError.response.headers instead')
         return self.response.headers
 
     @hdrs.setter
@@ -178,6 +185,7 @@ def hdrs(self, value):
 
     @property
     def filename(self):
+        deprecation_warning('HTTPError.filename is deprecated, use HTTPError.response.url instead')
         return self.response.url
 
     @filename.setter
@@ -185,6 +193,18 @@ def filename(self, value):
         return
 
     def __getattr__(self, name):
+        # File operations are passed through the response.
+        # Warn for some commonly used ones
+        passthrough_warnings = {
+            'read': 'response.read()',
+            # technically possibly due to passthrough, but we should discourage this
+            'get_header': 'response.get_header()',
+            'readable': 'response.readable()',
+            'closed': 'response.closed',
+            'tell': 'response.tell()',
+        }
+        if name in passthrough_warnings:
+            deprecation_warning(f'HTTPError.{name} is deprecated, use HTTPError.{passthrough_warnings[name]} instead')
         return super().__getattr__(name)
 
     def __str__(self):
diff --git a/yt_dlp/postprocessor/common.py b/yt_dlp/postprocessor/common.py
index 08b0fe1ff..8cef86c43 100644
--- a/yt_dlp/postprocessor/common.py
+++ b/yt_dlp/postprocessor/common.py
@@ -1,16 +1,15 @@
 import functools
 import json
 import os
-import urllib.error
 
+from ..networking import Request
+from ..networking.exceptions import HTTPError, network_exceptions
 from ..utils import (
     PostProcessingError,
     RetryManager,
     _configuration_args,
     deprecation_warning,
     encodeFilename,
-    network_exceptions,
-    sanitized_Request,
 )
 
 
@@ -203,13 +202,13 @@ def _download_json(self, url, *, expected_http_errors=(404,)):
         self.write_debug(f'{self.PP_NAME} query: {url}')
         for retry in RetryManager(self.get_param('extractor_retries', 3), self._retry_download):
             try:
-                rsp = self._downloader.urlopen(sanitized_Request(url))
+                rsp = self._downloader.urlopen(Request(url))
             except network_exceptions as e:
-                if isinstance(e, urllib.error.HTTPError) and e.code in expected_http_errors:
+                if isinstance(e, HTTPError) and e.status in expected_http_errors:
                     return None
                 retry.error = PostProcessingError(f'Unable to communicate with {self.PP_NAME} API: {e}')
                 continue
-        return json.loads(rsp.read().decode(rsp.info().get_param('charset') or 'utf-8'))
+        return json.loads(rsp.read().decode(rsp.headers.get_param('charset') or 'utf-8'))
 
 
 class AudioConversionError(PostProcessingError):  # Deprecated
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index 4790075eb..d708b09e3 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -7,19 +7,18 @@
 import re
 import subprocess
 import sys
-import urllib.error
 from zipimport import zipimporter
 
 from .compat import functools  # isort: split
 from .compat import compat_realpath, compat_shlex_quote
+from .networking import Request
+from .networking.exceptions import HTTPError, network_exceptions
 from .utils import (
     Popen,
     cached_method,
     deprecation_warning,
-    network_exceptions,
     remove_end,
     remove_start,
-    sanitized_Request,
     shell_quote,
     system_identifier,
     version_tuple,
@@ -190,7 +189,7 @@ def _tag(self):
     def _get_version_info(self, tag):
         url = f'{API_BASE_URL}/{self._target_repo}/releases/{tag}'
         self.ydl.write_debug(f'Fetching release info: {url}')
-        return json.loads(self.ydl.urlopen(sanitized_Request(url, headers={
+        return json.loads(self.ydl.urlopen(Request(url, headers={
             'Accept': 'application/vnd.github+json',
             'User-Agent': 'yt-dlp',
             'X-GitHub-Api-Version': '2022-11-28',
@@ -315,7 +314,7 @@ def update(self):
         try:
             newcontent = self._download(self.release_name, self._tag)
         except network_exceptions as e:
-            if isinstance(e, urllib.error.HTTPError) and e.code == 404:
+            if isinstance(e, HTTPError) and e.status == 404:
                 return self._report_error(
                     f'The requested tag {self._label(self.target_channel, self.target_tag)} does not exist', True)
             return self._report_network_error(f'fetch updates: {e}')
diff --git a/yt_dlp/utils/_deprecated.py b/yt_dlp/utils/_deprecated.py
index e55d42354..a8ae8ecb5 100644
--- a/yt_dlp/utils/_deprecated.py
+++ b/yt_dlp/utils/_deprecated.py
@@ -10,16 +10,6 @@
 
 
 from ._utils import preferredencoding
-from ..networking._urllib import HTTPHandler
-
-# isort: split
-from .networking import random_user_agent, std_headers  # noqa: F401
-from ..networking._urllib import PUTRequest  # noqa: F401
-from ..networking._urllib import SUPPORTED_ENCODINGS, HEADRequest  # noqa: F401
-from ..networking._urllib import ProxyHandler as PerRequestProxyHandler  # noqa: F401
-from ..networking._urllib import RedirectHandler as YoutubeDLRedirectHandler  # noqa: F401
-from ..networking._urllib import make_socks_conn_class, update_Request  # noqa: F401
-from ..networking.exceptions import network_exceptions  # noqa: F401
 
 
 def encodeFilename(s, for_subprocess=False):
@@ -47,12 +37,3 @@ def decodeOption(optval):
 
 def error_to_compat_str(err):
     return str(err)
-
-
-class YoutubeDLHandler(HTTPHandler):
-    def __init__(self, params, *args, **kwargs):
-        self._params = params
-        super().__init__(*args, **kwargs)
-
-
-YoutubeDLHTTPSHandler = YoutubeDLHandler
diff --git a/yt_dlp/utils/_legacy.py b/yt_dlp/utils/_legacy.py
index 96ac468b1..077000971 100644
--- a/yt_dlp/utils/_legacy.py
+++ b/yt_dlp/utils/_legacy.py
@@ -1,17 +1,30 @@
 """No longer used and new code should not use. Exists only for API compat."""
-
 import platform
 import struct
 import sys
+import urllib.error
 import urllib.parse
+import urllib.request
 import zlib
 
 from ._utils import Popen, decode_base_n, preferredencoding
 from .traversal import traverse_obj
 from ..dependencies import certifi, websockets
+from ..networking._helper import make_ssl_context
+from ..networking._urllib import HTTPHandler
 
 # isort: split
+from .networking import random_user_agent, std_headers  # noqa: F401
 from ..cookies import YoutubeDLCookieJar  # noqa: F401
+from ..networking._urllib import PUTRequest  # noqa: F401
+from ..networking._urllib import SUPPORTED_ENCODINGS, HEADRequest  # noqa: F401
+from ..networking._urllib import ProxyHandler as PerRequestProxyHandler  # noqa: F401
+from ..networking._urllib import RedirectHandler as YoutubeDLRedirectHandler  # noqa: F401
+from ..networking._urllib import (  # noqa: F401
+    make_socks_conn_class,
+    update_Request,
+)
+from ..networking.exceptions import HTTPError, network_exceptions  # noqa: F401
 
 has_certifi = bool(certifi)
 has_websockets = bool(websockets)
@@ -176,5 +189,52 @@ def handle_youtubedl_headers(headers):
     return filtered_headers
 
 
+def request_to_url(req):
+    if isinstance(req, urllib.request.Request):
+        return req.get_full_url()
+    else:
+        return req
+
+
+def sanitized_Request(url, *args, **kwargs):
+    from ..utils import escape_url, extract_basic_auth, sanitize_url
+    url, auth_header = extract_basic_auth(escape_url(sanitize_url(url)))
+    if auth_header is not None:
+        headers = args[1] if len(args) >= 2 else kwargs.setdefault('headers', {})
+        headers['Authorization'] = auth_header
+    return urllib.request.Request(url, *args, **kwargs)
+
+
+class YoutubeDLHandler(HTTPHandler):
+    def __init__(self, params, *args, **kwargs):
+        self._params = params
+        super().__init__(*args, **kwargs)
+
+
+YoutubeDLHTTPSHandler = YoutubeDLHandler
+
+
+class YoutubeDLCookieProcessor(urllib.request.HTTPCookieProcessor):
+    def __init__(self, cookiejar=None):
+        urllib.request.HTTPCookieProcessor.__init__(self, cookiejar)
+
+    def http_response(self, request, response):
+        return urllib.request.HTTPCookieProcessor.http_response(self, request, response)
+
+    https_request = urllib.request.HTTPCookieProcessor.http_request
+    https_response = http_response
+
+
+def make_HTTPS_handler(params, **kwargs):
+    return YoutubeDLHTTPSHandler(params, context=make_ssl_context(
+        verify=not params.get('nocheckcertificate'),
+        client_certificate=params.get('client_certificate'),
+        client_certificate_key=params.get('client_certificate_key'),
+        client_certificate_password=params.get('client_certificate_password'),
+        legacy_support=params.get('legacyserverconnect'),
+        use_certifi='no-certifi' not in params.get('compat_opts', []),
+    ), **kwargs)
+
+
 def process_communicate_or_kill(p, *args, **kwargs):
     return Popen.communicate_or_kill(p, *args, **kwargs)
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index d0e328716..2e619f9ea 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -62,11 +62,6 @@
 compiled_regex_type = type(re.compile(''))
 
 
-USER_AGENTS = {
-    'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
-}
-
-
 class NO_DEFAULT:
     pass
 
@@ -727,14 +722,6 @@ def extract_basic_auth(url):
     return url, f'Basic {auth_payload.decode()}'
 
 
-def sanitized_Request(url, *args, **kwargs):
-    url, auth_header = extract_basic_auth(escape_url(sanitize_url(url)))
-    if auth_header is not None:
-        headers = args[1] if len(args) >= 2 else kwargs.setdefault('headers', {})
-        headers['Authorization'] = auth_header
-    return urllib.request.Request(url, *args, **kwargs)
-
-
 def expand_path(s):
     """Expand shell variables and ~"""
     return os.path.expandvars(compat_expanduser(s))
@@ -894,19 +881,6 @@ def formatSeconds(secs, delim=':', msec=False):
     return '%s.%03d' % (ret, time.milliseconds) if msec else ret
 
 
-def make_HTTPS_handler(params, **kwargs):
-    from ._deprecated import YoutubeDLHTTPSHandler
-    from ..networking._helper import make_ssl_context
-    return YoutubeDLHTTPSHandler(params, context=make_ssl_context(
-        verify=not params.get('nocheckcertificate'),
-        client_certificate=params.get('client_certificate'),
-        client_certificate_key=params.get('client_certificate_key'),
-        client_certificate_password=params.get('client_certificate_password'),
-        legacy_support=params.get('legacyserverconnect'),
-        use_certifi='no-certifi' not in params.get('compat_opts', []),
-    ), **kwargs)
-
-
 def bug_reports_message(before=';'):
     from ..update import REPOSITORY
 
@@ -1143,17 +1117,6 @@ def is_path_like(f):
     return isinstance(f, (str, bytes, os.PathLike))
 
 
-class YoutubeDLCookieProcessor(urllib.request.HTTPCookieProcessor):
-    def __init__(self, cookiejar=None):
-        urllib.request.HTTPCookieProcessor.__init__(self, cookiejar)
-
-    def http_response(self, request, response):
-        return urllib.request.HTTPCookieProcessor.http_response(self, request, response)
-
-    https_request = urllib.request.HTTPCookieProcessor.http_request
-    https_response = http_response
-
-
 def extract_timezone(date_str):
     m = re.search(
         r'''(?x)
@@ -1455,6 +1418,7 @@ def write_string(s, out=None, encoding=None):
     out.flush()
 
 
+# TODO: Use global logger
 def deprecation_warning(msg, *, printer=None, stacklevel=0, **kwargs):
     from .. import _IN_CLI
     if _IN_CLI:
@@ -2005,13 +1969,6 @@ def url_or_none(url):
     return url if re.match(r'^(?:(?:https?|rt(?:m(?:pt?[es]?|fp)|sp[su]?)|mms|ftps?):)?//', url) else None
 
 
-def request_to_url(req):
-    if isinstance(req, urllib.request.Request):
-        return req.get_full_url()
-    else:
-        return req
-
-
 def strftime_or_none(timestamp, date_format='%Y%m%d', default=None):
     datetime_object = None
     try:
@@ -5525,7 +5482,7 @@ def info(self, message):
 
     def warning(self, message, *, once=False):
         if self._ydl:
-            self._ydl.report_warning(message, only_once=once)
+            self._ydl.report_warning(message, once)
 
     def error(self, message, *, is_error=True):
         if self._ydl:

From 131d132da5c98c6c78bd7eed4b37f4458561b3d9 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 15 Jul 2023 16:39:45 +0530
Subject: [PATCH 449/871] [build] Make sure deprecated modules are added

---
 setup.py                            | 3 ++-
 yt_dlp/__pyinstaller/hook-yt_dlp.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index ccfcf4252..a2f9f55c3 100644
--- a/setup.py
+++ b/setup.py
@@ -65,7 +65,8 @@ def py2exe_params():
             'excludes': ['Crypto', 'Cryptodome'],  # py2exe cannot import Crypto
             'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
             # Modules that are only imported dynamically must be added here
-            'includes': ['yt_dlp.compat._legacy'],
+            'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
+                         'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
         },
         'zipfile': None,
     }
diff --git a/yt_dlp/__pyinstaller/hook-yt_dlp.py b/yt_dlp/__pyinstaller/hook-yt_dlp.py
index 63dcdffe0..88c2b8b28 100644
--- a/yt_dlp/__pyinstaller/hook-yt_dlp.py
+++ b/yt_dlp/__pyinstaller/hook-yt_dlp.py
@@ -18,7 +18,8 @@ def pycryptodome_module():
 
 
 def get_hidden_imports():
-    yield 'yt_dlp.compat._legacy'
+    yield from ('yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated')
+    yield from ('yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated')
     yield pycryptodome_module()
     yield from collect_submodules('websockets')
     # These are auto-detected, but explicitly add them just in case

From 2b029ca0a9f9105c4f7626993fa60e54c9782749 Mon Sep 17 00:00:00 2001
From: Aaruni Kaushik <aaruni96@users.noreply.github.com>
Date: Sat, 15 Jul 2023 21:15:08 +0200
Subject: [PATCH 450/871] [cleanup] Add color to `download-archive` message
 (#5138)

Authored by: aaruni96, Grub4K, pukkandan
Closes #4913
---
 yt_dlp/YoutubeDL.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 850eb8ae0..c49960782 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1492,7 +1492,10 @@ def check_filter():
             return ret
 
         if self.in_download_archive(info_dict):
-            reason = '%s has already been recorded in the archive' % video_title
+            reason = ''.join((
+                format_field(info_dict, 'id', f'{self._format_screen("%s", self.Styles.ID)}: '),
+                format_field(info_dict, 'title', f'{self._format_screen("%s", self.Styles.EMPHASIS)} '),
+                'has already been recorded in the archive'))
             break_opt, break_err = 'break_on_existing', ExistingVideoReached
         else:
             try:
@@ -1553,7 +1556,8 @@ def extract_info(self, url, download=True, ie_key=None, extra_info=None,
 
             temp_id = ie.get_temp_id(url)
             if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': key}):
-                self.to_screen(f'[{key}] {temp_id}: has already been recorded in the archive')
+                self.to_screen(f'[download] {self._format_screen(temp_id, self.Styles.ID)}: '
+                               'has already been recorded in the archive')
                 if self.params.get('break_on_existing', False):
                     raise ExistingVideoReached()
                 break

From 6c5211cebeacfc53ad5d5ddf4a659be76039656f Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 15 Jul 2023 15:22:10 -0500
Subject: [PATCH 451/871] [core] Fix HTTP headers and cookie handling

- Remove `Cookie` header from `http_headers` immediately after loading into cookiejar
- Restore compat for `--load-info-json` cookies
- Add more tests
- Fix improper passing of Cookie header by `MailRu` extractor

Closes #7558
Authored by: bashonly, pukkandan
---
 test/test_YoutubeDL.py          | 85 +++++++++++++++++++++++++++++----
 test/test_YoutubeDLCookieJar.py |  8 ++++
 yt_dlp/YoutubeDL.py             | 46 ++++++++++++------
 yt_dlp/downloader/common.py     |  6 ---
 yt_dlp/extractor/mailru.py      |  8 ++--
 5 files changed, 120 insertions(+), 33 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index c15c7704c..b4f770ca5 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -11,7 +11,7 @@
 import copy
 import json
 
-from test.helper import FakeYDL, assertRegexpMatches
+from test.helper import FakeYDL, assertRegexpMatches, try_rm
 from yt_dlp import YoutubeDL
 from yt_dlp.compat import compat_os_name
 from yt_dlp.extractor import YoutubeIE
@@ -24,6 +24,8 @@
     int_or_none,
     match_filter_func,
 )
+from yt_dlp.utils.traversal import traverse_obj
+
 
 TEST_URL = 'http://localhost/sample.mp4'
 
@@ -1227,10 +1229,10 @@ def cookie(name, value, version=None, domain='', path='', secure=False, expires=
 
         _test_url = 'https://yt.dlp/test'
 
-        def test(encoded_cookies, cookies, headers=False, round_trip=None, error=None):
+        def test(encoded_cookies, cookies, *, headers=False, round_trip=None, error_re=None):
             def _test():
                 ydl.cookiejar.clear()
-                ydl._load_cookies(encoded_cookies, from_headers=headers)
+                ydl._load_cookies(encoded_cookies, autoscope=headers)
                 if headers:
                     ydl._apply_header_cookies(_test_url)
                 data = {'url': _test_url}
@@ -1245,14 +1247,14 @@ def _test():
                 ydl.__dict__['_YoutubeDL__header_cookies'] = []
 
             with self.subTest(msg=encoded_cookies):
-                if not error:
+                if not error_re:
                     _test()
                     return
-                with self.assertRaisesRegex(Exception, error):
+                with self.assertRaisesRegex(Exception, error_re):
                     _test()
 
         test('test=value; Domain=.yt.dlp', [cookie('test', 'value', domain='.yt.dlp')])
-        test('test=value', [cookie('test', 'value')], error='Unscoped cookies are not allowed')
+        test('test=value', [cookie('test', 'value')], error_re=r'Unscoped cookies are not allowed')
         test('cookie1=value1; Domain=.yt.dlp; Path=/test; cookie2=value2; Domain=.yt.dlp; Path=/', [
             cookie('cookie1', 'value1', domain='.yt.dlp', path='/test'),
             cookie('cookie2', 'value2', domain='.yt.dlp', path='/')])
@@ -1265,9 +1267,76 @@ def _test():
              round_trip='name=""; Domain=.yt.dlp')
 
         test('test=value', [cookie('test', 'value', domain='.yt.dlp')], headers=True)
-        test('cookie1=value; Domain=.yt.dlp; cookie2=value', [], headers=True, error='Invalid syntax')
+        test('cookie1=value; Domain=.yt.dlp; cookie2=value', [], headers=True, error_re=r'Invalid syntax')
         ydl.deprecated_feature = ydl.report_error
-        test('test=value', [], headers=True, error='Passing cookies as a header is a potential security risk')
+        test('test=value', [], headers=True, error_re=r'Passing cookies as a header is a potential security risk')
+
+    def test_infojson_cookies(self):
+        TEST_FILE = 'test_infojson_cookies.info.json'
+        TEST_URL = 'https://example.com/example.mp4'
+        COOKIES = 'a=b; Domain=.example.com; c=d; Domain=.example.com'
+        COOKIE_HEADER = {'Cookie': 'a=b; c=d'}
+
+        ydl = FakeYDL()
+        ydl.process_info = lambda x: ydl._write_info_json('test', x, TEST_FILE)
+
+        def make_info(info_header_cookies=False, fmts_header_cookies=False, cookies_field=False):
+            fmt = {'url': TEST_URL}
+            if fmts_header_cookies:
+                fmt['http_headers'] = COOKIE_HEADER
+            if cookies_field:
+                fmt['cookies'] = COOKIES
+            return _make_result([fmt], http_headers=COOKIE_HEADER if info_header_cookies else None)
+
+        def test(initial_info, note):
+            result = {}
+            result['processed'] = ydl.process_ie_result(initial_info)
+            self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
+                            msg=f'No cookies set in cookiejar after initial process when {note}')
+            ydl.cookiejar.clear()
+            with open(TEST_FILE) as infojson:
+                result['loaded'] = ydl.sanitize_info(json.load(infojson), True)
+            result['final'] = ydl.process_ie_result(result['loaded'].copy(), download=False)
+            self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
+                            msg=f'No cookies set in cookiejar after final process when {note}')
+            ydl.cookiejar.clear()
+            for key in ('processed', 'loaded', 'final'):
+                info = result[key]
+                self.assertIsNone(
+                    traverse_obj(info, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False),
+                    msg=f'Cookie header not removed in {key} result when {note}')
+                self.assertEqual(
+                    traverse_obj(info, ((None, ('formats', 0)), 'cookies'), get_all=False), COOKIES,
+                    msg=f'No cookies field found in {key} result when {note}')
+
+        test({'url': TEST_URL, 'http_headers': COOKIE_HEADER, 'id': '1', 'title': 'x'}, 'no formats field')
+        test(make_info(info_header_cookies=True), 'info_dict header cokies')
+        test(make_info(fmts_header_cookies=True), 'format header cookies')
+        test(make_info(info_header_cookies=True, fmts_header_cookies=True), 'info_dict and format header cookies')
+        test(make_info(info_header_cookies=True, fmts_header_cookies=True, cookies_field=True), 'all cookies fields')
+        test(make_info(cookies_field=True), 'cookies format field')
+        test({'url': TEST_URL, 'cookies': COOKIES, 'id': '1', 'title': 'x'}, 'info_dict cookies field only')
+
+        try_rm(TEST_FILE)
+
+    def test_add_headers_cookie(self):
+        def check_for_cookie_header(result):
+            return traverse_obj(result, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False)
+
+        ydl = FakeYDL({'http_headers': {'Cookie': 'a=b'}})
+        ydl._apply_header_cookies(_make_result([])['webpage_url'])  # Scope to input webpage URL: .example.com
+
+        fmt = {'url': 'https://example.com/video.mp4'}
+        result = ydl.process_ie_result(_make_result([fmt]), download=False)
+        self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies in result info_dict')
+        self.assertEqual(result.get('cookies'), 'a=b; Domain=.example.com', msg='No cookies were set in cookies field')
+        self.assertIn('a=b', ydl.cookiejar.get_cookie_header(fmt['url']), msg='No cookies were set in cookiejar')
+
+        fmt = {'url': 'https://wrong.com/video.mp4'}
+        result = ydl.process_ie_result(_make_result([fmt]), download=False)
+        self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies for wrong domain')
+        self.assertFalse(result.get('cookies'), msg='Cookies set in cookies field for wrong domain')
+        self.assertFalse(ydl.cookiejar.get_cookie_header(fmt['url']), msg='Cookies set in cookiejar for wrong domain')
 
 
 if __name__ == '__main__':
diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py
index 2c73d7d85..0b7a0acdb 100644
--- a/test/test_YoutubeDLCookieJar.py
+++ b/test/test_YoutubeDLCookieJar.py
@@ -53,6 +53,14 @@ def test_get_cookie_header(self):
         header = cookiejar.get_cookie_header('https://www.foobar.foobar')
         self.assertIn('HTTPONLY_COOKIE', header)
 
+    def test_get_cookies_for_url(self):
+        cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt')
+        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        cookies = cookiejar.get_cookies_for_url('https://www.foobar.foobar/')
+        self.assertEqual(len(cookies), 2)
+        cookies = cookiejar.get_cookies_for_url('https://foobar.foobar/')
+        self.assertFalse(cookies)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index c49960782..1a2f42fe9 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -680,14 +680,15 @@ def process_color_policy(stream):
 
         self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
         self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers'))
+        self.__header_cookies = []
+        self._load_cookies(self.params['http_headers'].get('Cookie'))  # compat
+        self.params['http_headers'].pop('Cookie', None)
+
         self._request_director = self.build_request_director(
             sorted(_REQUEST_HANDLERS.values(), key=lambda rh: rh.RH_NAME.lower()))
         if auto_init and auto_init != 'no_verbose_header':
             self.print_debug_header()
 
-        self.__header_cookies = []
-        self._load_cookies(traverse_obj(self.params.get('http_headers'), 'cookie', casesense=False))  # compat
-
         def check_deprecated(param, option, suggestion):
             if self.params.get(param) is not None:
                 self.report_warning(f'{option} is deprecated. Use {suggestion} instead')
@@ -1645,18 +1646,19 @@ def progress(msg):
                 self.to_screen('')
             raise
 
-    def _load_cookies(self, data, *, from_headers=True):
+    def _load_cookies(self, data, *, autoscope=True):
         """Loads cookies from a `Cookie` header
 
         This tries to work around the security vulnerability of passing cookies to every domain.
         See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
-        The unscoped cookies are saved for later to be stored in the jar with a limited scope.
 
         @param data         The Cookie header as string to load the cookies from
-        @param from_headers If `False`, allows Set-Cookie syntax in the cookie string (at least a domain will be required)
+        @param autoscope    If `False`, scope cookies using Set-Cookie syntax and error for cookie without domains
+                            If `True`, save cookies for later to be stored in the jar with a limited scope
+                            If a URL, save cookies in the jar with the domain of the URL
         """
         for cookie in LenientSimpleCookie(data).values():
-            if from_headers and any(cookie.values()):
+            if autoscope and any(cookie.values()):
                 raise ValueError('Invalid syntax in Cookie Header')
 
             domain = cookie.get('domain') or ''
@@ -1670,17 +1672,23 @@ def _load_cookies(self, data, *, from_headers=True):
 
             if domain:
                 self.cookiejar.set_cookie(prepared_cookie)
-            elif from_headers:
+            elif autoscope is True:
                 self.deprecated_feature(
                     'Passing cookies as a header is a potential security risk; '
                     'they will be scoped to the domain of the downloaded urls. '
                     'Please consider loading cookies from a file or browser instead.')
                 self.__header_cookies.append(prepared_cookie)
+            elif autoscope:
+                self.report_warning(
+                    'The extractor result contains an unscoped cookie as an HTTP header. '
+                    f'If you are using yt-dlp with an input URL{bug_reports_message(before=",")}',
+                    only_once=True)
+                self._apply_header_cookies(autoscope, [prepared_cookie])
             else:
                 self.report_error('Unscoped cookies are not allowed; please specify some sort of scoping',
                                   tb=False, is_error=False)
 
-    def _apply_header_cookies(self, url):
+    def _apply_header_cookies(self, url, cookies=None):
         """Applies stray header cookies to the provided url
 
         This loads header cookies and scopes them to the domain provided in `url`.
@@ -1691,7 +1699,7 @@ def _apply_header_cookies(self, url):
         if not parsed.hostname:
             return
 
-        for cookie in map(copy.copy, self.__header_cookies):
+        for cookie in map(copy.copy, cookies or self.__header_cookies):
             cookie.domain = f'.{parsed.hostname}'
             self.cookiejar.set_cookie(cookie)
 
@@ -2481,9 +2489,16 @@ def restore_last_token(self):
         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
         return _build_selector_function(parsed_selector)
 
-    def _calc_headers(self, info_dict):
+    def _calc_headers(self, info_dict, load_cookies=False):
         res = HTTPHeaderDict(self.params['http_headers'], info_dict.get('http_headers'))
         clean_headers(res)
+
+        if load_cookies:  # For --load-info-json
+            self._load_cookies(res.get('Cookie'), autoscope=info_dict['url'])  # compat
+            self._load_cookies(info_dict.get('cookies'), autoscope=False)
+        # The `Cookie` header is removed to prevent leaks and unscoped cookies.
+        # See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
+        res.pop('Cookie', None)
         cookies = self.cookiejar.get_cookies_for_url(info_dict['url'])
         if cookies:
             encoder = LenientSimpleCookie()
@@ -2762,7 +2777,12 @@ def is_wellformed(f):
                     and info_dict.get('duration') and format.get('tbr')
                     and not format.get('filesize') and not format.get('filesize_approx')):
                 format['filesize_approx'] = int(info_dict['duration'] * format['tbr'] * (1024 / 8))
-            format['http_headers'] = self._calc_headers(collections.ChainMap(format, info_dict))
+            format['http_headers'] = self._calc_headers(collections.ChainMap(format, info_dict), load_cookies=True)
+
+        # Safeguard against old/insecure infojson when using --load-info-json
+        if info_dict.get('http_headers'):
+            info_dict['http_headers'] = HTTPHeaderDict(info_dict['http_headers'])
+            info_dict['http_headers'].pop('Cookie', None)
 
         # This is copied to http_headers by the above _calc_headers and can now be removed
         if '__x_forwarded_for_ip' in info_dict:
@@ -3508,8 +3528,6 @@ def download_with_info_file(self, info_filename):
             infos = [self.sanitize_info(info, self.params.get('clean_infojson', True))
                      for info in variadic(json.loads('\n'.join(f)))]
         for info in infos:
-            self._load_cookies(info.get('cookies'), from_headers=False)
-            self._load_cookies(traverse_obj(info.get('http_headers'), 'Cookie', casesense=False))  # compat
             try:
                 self.__download_wrapper(self.process_ie_result)(info, download=True)
             except (DownloadError, EntryNotInPlaylist, ReExtractInfo) as e:
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index 2c404ee90..b71d7ee8f 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -32,7 +32,6 @@
     timetuple_from_msec,
     try_call,
 )
-from ..utils.traversal import traverse_obj
 
 
 class FileDownloader:
@@ -453,11 +452,6 @@ def download(self, filename, info_dict, subtitle=False):
             self.to_screen(f'[download] Sleeping {sleep_interval:.2f} seconds ...')
             time.sleep(sleep_interval)
 
-        # Filter the `Cookie` header from the info_dict to prevent leaks.
-        # See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
-        info_dict['http_headers'] = dict(traverse_obj(info_dict, (
-            'http_headers', {dict.items}, lambda _, pair: pair[0].lower() != 'cookie'))) or None
-
         ret = self.real_download(filename, info_dict)
         self._finish_multiline_status()
         return ret, True
diff --git a/yt_dlp/extractor/mailru.py b/yt_dlp/extractor/mailru.py
index 387d211fe..0f0550c92 100644
--- a/yt_dlp/extractor/mailru.py
+++ b/yt_dlp/extractor/mailru.py
@@ -1,6 +1,7 @@
 import itertools
 import json
 import re
+import urllib.parse
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
@@ -140,17 +141,15 @@ def _real_extract(self, url):
                 'http://api.video.mail.ru/videos/%s.json?new=1' % video_id,
                 video_id, 'Downloading video JSON')
 
-        headers = {}
-
         video_key = self._get_cookies('https://my.mail.ru').get('video_key')
-        if video_key:
-            headers['Cookie'] = 'video_key=%s' % video_key.value
 
         formats = []
         for f in video_data['videos']:
             video_url = f.get('url')
             if not video_url:
                 continue
+            if video_key:
+                self._set_cookie(urllib.parse.urlparse(video_url).hostname, 'video_key', video_key.value)
             format_id = f.get('key')
             height = int_or_none(self._search_regex(
                 r'^(\d+)[pP]$', format_id, 'height', default=None)) if format_id else None
@@ -158,7 +157,6 @@ def _real_extract(self, url):
                 'url': video_url,
                 'format_id': format_id,
                 'height': height,
-                'http_headers': headers,
             })
 
         meta_data = video_data['meta']

From 42ded0a429c20ec13dc006825e1508d9a02f0ad4 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 15 Jul 2023 15:18:25 -0500
Subject: [PATCH 452/871] [fd/external] Fixes to cookie handling

- Fix bug in `axel` Cookie header arg
- Pass cookies to `curl` as strings
- Write session cookies for `aria2c` and `wget`

Closes #7539
Authored by: bashonly
---
 test/test_downloader_external.py | 9 +++++----
 yt_dlp/downloader/external.py    | 9 +++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/test/test_downloader_external.py b/test/test_downloader_external.py
index e5b02ba5a..d3d74df04 100644
--- a/test/test_downloader_external.py
+++ b/test/test_downloader_external.py
@@ -68,7 +68,7 @@ def test_make_cmd(self):
             ydl.cookiejar.set_cookie(http.cookiejar.Cookie(**TEST_COOKIE))
             self.assertEqual(
                 downloader._make_cmd('test', TEST_INFO),
-                ['axel', '-o', 'test', 'Cookie: test=ytdlp', '--max-redirect=0', '--', 'http://www.example.com/'])
+                ['axel', '-o', 'test', '-H', 'Cookie: test=ytdlp', '--max-redirect=0', '--', 'http://www.example.com/'])
 
 
 class TestWgetFD(unittest.TestCase):
@@ -85,10 +85,11 @@ class TestCurlFD(unittest.TestCase):
     def test_make_cmd(self):
         with FakeYDL() as ydl:
             downloader = CurlFD(ydl, {})
-            self.assertNotIn('--cookie-jar', downloader._make_cmd('test', TEST_INFO))
-            # Test cookiejar tempfile arg is added
+            self.assertNotIn('--cookie', downloader._make_cmd('test', TEST_INFO))
+            # Test cookie header is added
             ydl.cookiejar.set_cookie(http.cookiejar.Cookie(**TEST_COOKIE))
-            self.assertIn('--cookie-jar', downloader._make_cmd('test', TEST_INFO))
+            self.assertIn('--cookie', downloader._make_cmd('test', TEST_INFO))
+            self.assertIn('test=ytdlp', downloader._make_cmd('test', TEST_INFO))
 
 
 class TestAria2cFD(unittest.TestCase):
diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index e307502db..4f52f6e8d 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -137,7 +137,7 @@ def _write_cookies(self):
             self._cookies_tempfile = tmp_cookies.name
             self.to_screen(f'[download] Writing temporary cookies file to "{self._cookies_tempfile}"')
         # real_download resets _cookies_tempfile; if it's None then save() will write to cookiejar.filename
-        self.ydl.cookiejar.save(self._cookies_tempfile)
+        self.ydl.cookiejar.save(self._cookies_tempfile, ignore_discard=True, ignore_expires=True)
         return self.ydl.cookiejar.filename or self._cookies_tempfile
 
     def _call_downloader(self, tmpfilename, info_dict):
@@ -199,8 +199,9 @@ class CurlFD(ExternalFD):
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '--location', '-o', tmpfilename, '--compressed']
-        if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
-            cmd += ['--cookie-jar', self._write_cookies()]
+        cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
+        if cookie_header:
+            cmd += ['--cookie', cookie_header]
         if info_dict.get('http_headers') is not None:
             for key, val in info_dict['http_headers'].items():
                 cmd += ['--header', f'{key}: {val}']
@@ -233,7 +234,7 @@ def _make_cmd(self, tmpfilename, info_dict):
                 cmd += ['-H', f'{key}: {val}']
         cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
         if cookie_header:
-            cmd += [f'Cookie: {cookie_header}', '--max-redirect=0']
+            cmd += ['-H', f'Cookie: {cookie_header}', '--max-redirect=0']
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd

From 1d3d579c2142f69831b6ae140e1d8e824e07fa0e Mon Sep 17 00:00:00 2001
From: zhong-yiyu <53254770+zhong-yiyu@users.noreply.github.com>
Date: Sat, 15 Jul 2023 21:54:19 +0100
Subject: [PATCH 453/871] [ie/pornhub] Update access cookies for UK (#7591)

Closes #7590
Authored by: zhong-yiyu
---
 yt_dlp/extractor/pornhub.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/yt_dlp/extractor/pornhub.py b/yt_dlp/extractor/pornhub.py
index f08414030..999d038d4 100644
--- a/yt_dlp/extractor/pornhub.py
+++ b/yt_dlp/extractor/pornhub.py
@@ -62,6 +62,7 @@ def _real_initialize(self):
     def _set_age_cookies(self, host):
         self._set_cookie(host, 'age_verified', '1')
         self._set_cookie(host, 'accessAgeDisclaimerPH', '1')
+        self._set_cookie(host, 'accessAgeDisclaimerUK', '1')
         self._set_cookie(host, 'accessPH', '1')
 
     def _login(self, host):

From bb5d84c9d2f1e978c3eddfb5ccbe138036682a36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C4=83n=20Anh?=
 <65241526+demon071@users.noreply.github.com>
Date: Sun, 16 Jul 2023 04:03:23 +0700
Subject: [PATCH 454/871] [ie/facebook:reel] Fix extraction (#7564)

Closes #7469
Authored by: demon071, bashonly
---
 yt_dlp/extractor/facebook.py | 83 +++++++++++++++++++++++-------------
 1 file changed, 54 insertions(+), 29 deletions(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 9f4d3fb78..574f8e8c9 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -24,6 +24,7 @@
     parse_count,
     parse_qs,
     qualities,
+    str_or_none,
     traverse_obj,
     try_get,
     url_or_none,
@@ -90,16 +91,16 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '274175099429670',
             'ext': 'mp4',
-            'title': 'Asif Nawab Butt',
-            'description': 'Asif Nawab Butt',
+            'title': 'Asif',
+            'description': '',
             'uploader': 'Asif Nawab Butt',
             'upload_date': '20140506',
             'timestamp': 1399398998,
             'thumbnail': r're:^https?://.*',
+            'uploader_id': 'pfbid04scW44U4P9iTyLZAGy8y8W3pR3i2VugvHCimiRudUAVbN3MPp9eXBaYFcgVworZwl',
+            'duration': 131.03,
+            'concurrent_view_count': int,
         },
-        'expected_warnings': [
-            'title'
-        ]
     }, {
         'note': 'Video with DASH manifest',
         'url': 'https://www.facebook.com/video.php?v=957955867617029',
@@ -151,7 +152,7 @@ class FacebookIE(InfoExtractor):
         # have 1080P, but only up to 720p in swf params
         # data.video.story.attachments[].media
         'url': 'https://www.facebook.com/cnn/videos/10155529876156509/',
-        'md5': '3f3798adb2b73423263e59376f1f5eb7',
+        'md5': 'ca63897a90c9452efee5f8c40d080e25',
         'info_dict': {
             'id': '10155529876156509',
             'ext': 'mp4',
@@ -162,6 +163,9 @@ class FacebookIE(InfoExtractor):
             'uploader': 'CNN',
             'thumbnail': r're:^https?://.*',
             'view_count': int,
+            'uploader_id': '100059479812265',
+            'concurrent_view_count': int,
+            'duration': 44.478,
         },
     }, {
         # bigPipe.onPageletArrive ... onPageletArrive pagelet_group_mall
@@ -170,12 +174,16 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '1417995061575415',
             'ext': 'mp4',
-            'title': 'Ukrainian Scientists Worldwide | Довгоочікуване відео',
+            'title': 'Довгоочікуване відео | By Yaroslav - Facebook',
             'description': 'Довгоочікуване відео',
-            'timestamp': 1486648771,
+            'timestamp': 1486648217,
             'upload_date': '20170209',
             'uploader': 'Yaroslav Korpan',
-            'uploader_id': '100000948048708',
+            'uploader_id': 'pfbid029y8j22EwH3ikeqgH3SEP9G3CAi9kmWKgXJJG9s5geV7mo3J2bvURqHCdgucRgAyhl',
+            'concurrent_view_count': int,
+            'thumbnail': r're:^https?://.*',
+            'view_count': int,
+            'duration': 11736.446,
         },
         'params': {
             'skip_download': True,
@@ -192,9 +200,7 @@ class FacebookIE(InfoExtractor):
             'uploader': 'La Guía Del Varón',
             'thumbnail': r're:^https?://.*',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'skip': 'Requires logging in',
     }, {
         # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/',
@@ -208,9 +214,7 @@ class FacebookIE(InfoExtractor):
             'uploader': 'Elisabeth Ahtn',
             'uploader_id': '100013949973717',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'skip': 'Requires logging in',
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -252,7 +256,11 @@ class FacebookIE(InfoExtractor):
             'timestamp': 1527084179,
             'upload_date': '20180523',
             'uploader': 'ESL One Dota 2',
-            'uploader_id': '234218833769558',
+            'uploader_id': '100066514874195',
+            'duration': 4524.212,
+            'view_count': int,
+            'thumbnail': r're:^https?://.*',
+            'concurrent_view_count': int,
         },
         'params': {
             'skip_download': True,
@@ -262,8 +270,17 @@ class FacebookIE(InfoExtractor):
         'url': 'https://www.facebook.com/100033620354545/videos/106560053808006/',
         'info_dict': {
             'id': '106560053808006',
+            'ext': 'mp4',
+            'title': 'Josef',
+            'thumbnail': r're:^https?://.*',
+            'concurrent_view_count': int,
+            'uploader_id': 'pfbid02gXHbDwxumkaKJQaTGUf3znYfYzTuidGEWawiramNx4YamSj2afwYSRkpcjtHtMRJl',
+            'timestamp': 1549275572,
+            'duration': 3.413,
+            'uploader': 'Josef Novak',
+            'description': '',
+            'upload_date': '20190204',
         },
-        'playlist_count': 2,
     }, {
         # data.video.story.attachments[].media
         'url': 'https://www.facebook.com/watch/?v=647537299265662',
@@ -276,6 +293,7 @@ class FacebookIE(InfoExtractor):
             'id': '10157667649866271',
         },
         'playlist_count': 3,
+        'skip': 'Requires logging in',
     }, {
         # data.nodes[].comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://m.facebook.com/Alliance.Police.Department/posts/4048563708499330',
@@ -497,6 +515,13 @@ def extract_relay_prefetched_data(_filter):
                 entries = []
 
                 def parse_graphql_video(video):
+                    v_id = video.get('videoId') or video.get('id') or video_id
+                    reel_info = traverse_obj(
+                        video, ('creation_story', 'short_form_video_context', 'playback_video', {dict}))
+                    if reel_info:
+                        video = video['creation_story']
+                        video['owner'] = traverse_obj(video, ('short_form_video_context', 'video_owner'))
+                        video.update(reel_info)
                     formats = []
                     q = qualities(['sd', 'hd'])
                     for key, format_id in (('playable_url', 'sd'), ('playable_url_quality_hd', 'hd'),
@@ -513,15 +538,15 @@ def parse_graphql_video(video):
                                 'url': playable_url,
                             })
                     extract_dash_manifest(video, formats)
-                    v_id = video.get('videoId') or video.get('id') or video_id
                     info = {
                         'id': v_id,
                         'formats': formats,
                         'thumbnail': traverse_obj(
                             video, ('thumbnailImage', 'uri'), ('preferred_thumbnail', 'image', 'uri')),
-                        'uploader_id': try_get(video, lambda x: x['owner']['id']),
-                        'timestamp': int_or_none(video.get('publish_time')),
-                        'duration': float_or_none(video.get('playable_duration_in_ms'), 1000),
+                        'uploader_id': traverse_obj(video, ('owner', 'id', {str_or_none})),
+                        'timestamp': traverse_obj(video, 'publish_time', 'creation_time', expected_type=int_or_none),
+                        'duration': (float_or_none(video.get('playable_duration_in_ms'), 1000)
+                                     or float_or_none(video.get('length_in_second'))),
                     }
                     process_formats(info)
                     description = try_get(video, lambda x: x['savable_description']['text'])
@@ -782,18 +807,18 @@ class FacebookReelIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://www.facebook.com/reel/1195289147628387',
-        'md5': 'c4ff9a7182ff9ff7d6f7a83603bae831',
+        'md5': 'f13dd37f2633595982db5ed8765474d3',
         'info_dict': {
             'id': '1195289147628387',
             'ext': 'mp4',
-            'title': 'md5:9f5b142921b2dc57004fa13f76005f87',
-            'description': 'md5:24ea7ef062215d295bdde64e778f5474',
-            'uploader': 'Beast Camp Training',
-            'uploader_id': '1738535909799870',
-            'duration': 9.536,
-            'thumbnail': r're:^https?://.*',
+            'title': 'md5:b05800b5b1ad56c0ca78bd3807b6a61e',
+            'description': 'md5:22f03309b216ac84720183961441d8db',
+            'uploader': 'md5:723e6cb3091241160f20b3c5dc282af1',
+            'uploader_id': '100040874179269',
+            'duration': 9.579,
+            'timestamp': 1637502609,
             'upload_date': '20211121',
-            'timestamp': 1637502604,
+            'thumbnail': r're:^https?://.*',
         }
     }]
 

From 613dbce177d34ffc31053e8e01acf4bb107bcd1e Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 15 Jul 2023 16:10:12 -0500
Subject: [PATCH 455/871] [ie/twitter:spaces] Fix format protocol (#7550)

Closes #7536
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 38 ++++++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 4015277a8..9d87dbc4b 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1499,6 +1499,38 @@ class TwitterSpacesIE(TwitterBaseIE):
             'release_date': '20220807',
         },
         'params': {'skip_download': 'm3u8'},
+    }, {
+        # post_live/TimedOut but downloadable
+        'url': 'https://twitter.com/i/spaces/1vAxRAVQWONJl',
+        'info_dict': {
+            'id': '1vAxRAVQWONJl',
+            'ext': 'm4a',
+            'title': 'Framing Up FinOps: Billing Tools',
+            'description': 'Twitter Space participated by rupa, Alfonso Hernandez',
+            'uploader': 'Google Cloud',
+            'uploader_id': 'googlecloud',
+            'live_status': 'post_live',
+            'timestamp': 1681409554,
+            'upload_date': '20230413',
+            'release_timestamp': 1681839000,
+            'release_date': '20230418',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        # Needs ffmpeg as downloader, see: https://github.com/yt-dlp/yt-dlp/issues/7536
+        'url': 'https://twitter.com/i/spaces/1eaKbrQbjoRKX',
+        'info_dict': {
+            'id': '1eaKbrQbjoRKX',
+            'ext': 'm4a',
+            'title': 'あ',
+            'description': 'Twitter Space participated by nobody yet',
+            'uploader': '息根とめる🔪Twitchで復活',
+            'uploader_id': 'tomeru_ikinone',
+            'live_status': 'was_live',
+            'timestamp': 1685617198,
+            'upload_date': '20230601',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     SPACE_STATUS = {
@@ -1555,9 +1587,9 @@ def _real_extract(self, url):
             source = traverse_obj(
                 self._call_api(f'live_video_stream/status/{metadata["media_key"]}', metadata['media_key']),
                 ('source', ('noRedirectPlaybackUrl', 'location'), {url_or_none}), get_all=False)
-            formats = self._extract_m3u8_formats(
-                source, metadata['media_key'], 'm4a', live=is_live, fatal=False,
-                headers={'Referer': 'https://twitter.com/'}) if source else []
+            formats = self._extract_m3u8_formats(  # XXX: Some Spaces need ffmpeg as downloader
+                source, metadata['media_key'], 'm4a', entry_protocol='m3u8', live=is_live,
+                headers={'Referer': 'https://twitter.com/'}, fatal=False) if source else []
             for fmt in formats:
                 fmt.update({'vcodec': 'none', 'acodec': 'aac'})
                 if not is_live:

From 71baa490ebd3655746430f208a9b605d120cd315 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 20 Jul 2023 08:23:30 -0500
Subject: [PATCH 456/871] [networking] Fix POST requests with zero-length
 payloads (#7648)

Bugfix for 227bf1a33be7b89cd7d44ad046844c4ccba104f4

Authored by: bashonly
---
 test/test_networking.py      | 11 +++++++++++
 yt_dlp/extractor/ettutv.py   |  2 +-
 yt_dlp/networking/_urllib.py |  2 +-
 yt_dlp/networking/common.py  |  2 +-
 4 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index b60ed283b..3cf587a63 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -1280,6 +1280,17 @@ def test_content_type_header(self):
         req.data = b'test3'
         assert req.headers.get('Content-Type') == 'application/x-www-form-urlencoded'
 
+    def test_update_req(self):
+        req = Request('http://example.com')
+        assert req.data is None
+        assert req.method == 'GET'
+        assert 'Content-Type' not in req.headers
+        # Test that zero-byte payloads will be sent
+        req.update(data=b'')
+        assert req.data == b''
+        assert req.method == 'POST'
+        assert req.headers.get('Content-Type') == 'application/x-www-form-urlencoded'
+
     def test_proxies(self):
         req = Request(url='http://example.com', proxies={'http': 'http://127.0.0.1:8080'})
         assert req.proxies == {'http': 'http://127.0.0.1:8080'}
diff --git a/yt_dlp/extractor/ettutv.py b/yt_dlp/extractor/ettutv.py
index 46d725543..133b52555 100644
--- a/yt_dlp/extractor/ettutv.py
+++ b/yt_dlp/extractor/ettutv.py
@@ -41,7 +41,7 @@ def _real_extract(self, url):
                 'device': 'desktop',
             })
 
-        stream_response = self._download_json(player_settings['streamAccess'], video_id, data={})
+        stream_response = self._download_json(player_settings['streamAccess'], video_id, data=b'')
 
         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
             stream_response['data']['stream'], video_id, 'mp4')
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index 2c5f09872..8a76676d9 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -315,7 +315,7 @@ def get_method(self):
 def update_Request(req, url=None, data=None, headers=None, query=None):
     req_headers = req.headers.copy()
     req_headers.update(headers or {})
-    req_data = data or req.data
+    req_data = data if data is not None else req.data
     req_url = update_url_query(url or req.get_full_url(), query)
     req_get_method = req.get_method()
     if req_get_method == 'HEAD':
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
index 458eca39f..61196406d 100644
--- a/yt_dlp/networking/common.py
+++ b/yt_dlp/networking/common.py
@@ -425,7 +425,7 @@ def headers(self, new_headers: Mapping):
             raise TypeError('headers must be a mapping')
 
     def update(self, url=None, data=None, headers=None, query=None):
-        self.data = data or self.data
+        self.data = data if data is not None else self.data
         self.headers.update(headers or {})
         self.url = update_url_query(url or self.url, query or {})
 

From 75dc8e673b481a82d0688aeec30f6c65d82bb359 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 20 Jul 2023 08:31:17 -0500
Subject: [PATCH 457/871] [networking] Fix `--legacy-server-connect` (#7645)

Bugfix for 227bf1a33be7b89cd7d44ad046844c4ccba104f4

Authored by: bashonly
---
 test/test_networking.py | 2 +-
 yt_dlp/YoutubeDL.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index 3cf587a63..d4eba2a5d 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -1152,7 +1152,7 @@ def test_build_handler_params(self):
             'debug_printtraffic': True,
             'compat_opts': ['no-certifi'],
             'nocheckcertificate': True,
-            'legacy_server_connect': True,
+            'legacyserverconnect': True,
         }) as ydl:
             rh = self.build_handler(ydl)
             assert rh.headers.get('test') == 'testtest'
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 1a2f42fe9..324f9e99c 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -4097,7 +4097,7 @@ def build_request_director(self, handlers):
                     'verbose': 'debug_printtraffic',
                     'source_address': 'source_address',
                     'timeout': 'socket_timeout',
-                    'legacy_ssl_support': 'legacy_server_connect',
+                    'legacy_ssl_support': 'legacyserverconnect',
                     'enable_file_urls': 'enable_file_urls',
                     'client_cert': {
                         'client_certificate': 'client_certificate',

From af86873218c24c3859ccf575a87f2b00a73b49d0 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Thu, 20 Jul 2023 08:40:31 -0500
Subject: [PATCH 458/871] [utils] Improve `parse_duration`

Authored by: bashonly
---
 test/test_utils.py     | 2 ++
 yt_dlp/utils/_utils.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 768edfd0c..b36bc04c2 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -655,6 +655,8 @@ def test_parse_duration(self):
         self.assertEqual(parse_duration('P0Y0M0DT0H4M20.880S'), 260.88)
         self.assertEqual(parse_duration('01:02:03:050'), 3723.05)
         self.assertEqual(parse_duration('103:050'), 103.05)
+        self.assertEqual(parse_duration('1HR 3MIN'), 3780)
+        self.assertEqual(parse_duration('2hrs 3mins'), 7380)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 2e619f9ea..abae0f17e 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -2021,7 +2021,7 @@ def parse_duration(s):
                 )?
                 T)?
                 (?:
-                    (?P<hours>[0-9]+)\s*h(?:ours?)?,?\s*
+                    (?P<hours>[0-9]+)\s*h(?:(?:ou)?rs?)?,?\s*
                 )?
                 (?:
                     (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?,?\s*

From f4ea501551526ebcb54d19b84cf0ebe798583a85 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 20 Jul 2023 09:02:50 -0500
Subject: [PATCH 459/871] [ie/MagellanTV] Add extractor (#7616)

Closes #7529
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/magellantv.py  | 50 +++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 yt_dlp/extractor/magellantv.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 2af99b3da..bcd8dbe00 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1012,6 +1012,7 @@
     LyndaCourseIE
 )
 from .m6 import M6IE
+from .magellantv import MagellanTVIE
 from .magentamusik360 import MagentaMusik360IE
 from .mailru import (
     MailRuIE,
diff --git a/yt_dlp/extractor/magellantv.py b/yt_dlp/extractor/magellantv.py
new file mode 100644
index 000000000..0947a450a
--- /dev/null
+++ b/yt_dlp/extractor/magellantv.py
@@ -0,0 +1,50 @@
+from .common import InfoExtractor
+from ..utils import parse_age_limit, parse_duration, traverse_obj
+
+
+class MagellanTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?magellantv\.com/(?:watch|video)/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.magellantv.com/watch/my-dads-on-death-row?type=v',
+        'info_dict': {
+            'id': 'my-dads-on-death-row',
+            'ext': 'mp4',
+            'title': 'My Dad\'s On Death Row',
+            'description': 'md5:33ba23b9f0651fc4537ed19b1d5b0d7a',
+            'duration': 3780.0,
+            'age_limit': 14,
+            'tags': ['Justice', 'Reality', 'United States', 'True Crime'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.magellantv.com/video/james-bulger-the-new-revelations',
+        'info_dict': {
+            'id': 'james-bulger-the-new-revelations',
+            'ext': 'mp4',
+            'title': 'James Bulger: The New Revelations',
+            'description': 'md5:7b97922038bad1d0fe8d0470d8a189f2',
+            'duration': 2640.0,
+            'age_limit': 0,
+            'tags': ['Investigation', 'True Crime', 'Justice', 'Europe'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['reactContext']['video']['detail']
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(data['jwpVideoUrl'], video_id)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('metadata', 'description', {str}),
+                'duration': ('duration', {parse_duration}),
+                'age_limit': ('ratingCategory', {parse_age_limit}),
+                'tags': ('tags', ..., {str}),
+            }),
+        }

From 65cfa2b057d7946fbe322155a778fe206556d0c6 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 20 Jul 2023 09:15:21 -0500
Subject: [PATCH 460/871] [ie/MuseAI] Add extractor (#7614)

Closes #7543
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/museai.py      | 112 ++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 yt_dlp/extractor/museai.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index bcd8dbe00..ae73a9f96 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1142,6 +1142,7 @@
 )
 from .muenchentv import MuenchenTVIE
 from .murrtube import MurrtubeIE, MurrtubeUserIE
+from .museai import MuseAIIE
 from .musescore import MuseScoreIE
 from .musicdex import (
     MusicdexSongIE,
diff --git a/yt_dlp/extractor/museai.py b/yt_dlp/extractor/museai.py
new file mode 100644
index 000000000..7f66928c7
--- /dev/null
+++ b/yt_dlp/extractor/museai.py
@@ -0,0 +1,112 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    float_or_none,
+    int_or_none,
+    js_to_json,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class MuseAIIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?muse\.ai/(?:v|embed)/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://muse.ai/embed/YdTWvUW',
+        'md5': 'f994f9a38be1c3aaf9e37cbd7d76fe7c',
+        'info_dict': {
+            'id': 'YdTWvUW',
+            'ext': 'mp4',
+            'title': '2023-05-28-Grabien-1941111 (1)',
+            'description': '',
+            'uploader': 'Today News Africa',
+            'uploader_id': 'TodayNewsAfrica',
+            'upload_date': '20230528',
+            'timestamp': 1685285044,
+            'duration': 1291.3,
+            'view_count': int,
+            'availability': 'public',
+        },
+    }, {
+        'url': 'https://muse.ai/v/gQ4gGAA-0756',
+        'md5': '52dbfc78e865e56dc19a1715badc35e8',
+        'info_dict': {
+            'id': 'gQ4gGAA',
+            'ext': 'mp4',
+            'title': '0756',
+            'description': 'md5:0ca1483f9aac423e9a96ad00bb3a0785',
+            'uploader': 'Aerial.ie',
+            'uploader_id': 'aerial',
+            'upload_date': '20210306',
+            'timestamp': 1615072842,
+            'duration': 21.4,
+            'view_count': int,
+            'availability': 'public',
+        },
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://muse.ai/docs',
+        'playlist_mincount': 4,
+        'info_dict': {
+            'id': 'docs',
+            'title': 'muse.ai | docs',
+            'description': 'md5:6c0293431481582739c82ee8902687fa',
+            'age_limit': 0,
+            'thumbnail': 'https://muse.ai/static/imgs/poster-img-docs.png',
+        },
+        'params': {'allowed_extractors': ['all', '-html5']},
+    }]
+    _EMBED_REGEX = [r'<iframe[^>]*\bsrc=["\'](?P<url>https://muse\.ai/embed/\w+)']
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        yield from super()._extract_embed_urls(url, webpage)
+        for embed_id in re.findall(r'<script>[^<]*\bMusePlayer\(\{[^}<]*\bvideo:\s*["\'](\w+)["\']', webpage):
+            yield f'https://muse.ai/embed/{embed_id}'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(f'https://muse.ai/embed/{video_id}', video_id)
+        data = self._search_json(
+            r'player\.setData\(', webpage, 'player data', video_id, transform_source=js_to_json)
+
+        source_url = data['url']
+        if not url_or_none(source_url):
+            raise ExtractorError('Unable to extract video URL')
+
+        formats = [{
+            'url': source_url,
+            'format_id': 'source',
+            'quality': 1,
+            **traverse_obj(data, {
+                'ext': ('filename', {determine_ext}),
+                'width': ('width', {int_or_none}),
+                'height': ('height', {int_or_none}),
+                'filesize': ('size', {int_or_none}),
+            }),
+        }]
+        if source_url.endswith('/data'):
+            base_url = f'{source_url[:-5]}/videos'
+            formats.extend(self._extract_m3u8_formats(
+                f'{base_url}/hls.m3u8', video_id, m3u8_id='hls', fatal=False))
+            formats.extend(self._extract_mpd_formats(
+                f'{base_url}/dash.mpd', video_id, mpd_id='dash', fatal=False))
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'duration': ('duration', {float_or_none}),
+                'timestamp': ('tcreated', {int_or_none}),
+                'uploader': ('owner_name', {str}),
+                'uploader_id': ('owner_username', {str}),
+                'view_count': ('views', {int_or_none}),
+                'age_limit': ('mature', {lambda x: 18 if x else None}),
+                'availability': ('visibility', {lambda x: x if x in ('private', 'unlisted') else 'public'}),
+            }),
+        }

From 9b16762f48914de9ac914601769c76668e433325 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Thu, 20 Jul 2023 22:09:52 +0200
Subject: [PATCH 461/871] [ie/crunchyroll] Remove initial state extraction
 (#7632)

Authored by: Grub4K
---
 yt_dlp/extractor/crunchyroll.py | 73 +++++++++++++++++++--------------
 1 file changed, 43 insertions(+), 30 deletions(-)

diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py
index adb3d5dcf..ee34aced5 100644
--- a/yt_dlp/extractor/crunchyroll.py
+++ b/yt_dlp/extractor/crunchyroll.py
@@ -27,11 +27,24 @@ class CrunchyrollBaseIE(InfoExtractor):
     _AUTH_HEADERS = None
     _API_ENDPOINT = None
     _BASIC_AUTH = None
-    _QUERY = {}
+    _CLIENT_ID = ('cr_web', 'noaihdevm_6iyg0a8l0q')
+    _LOCALE_LOOKUP = {
+        'ar': 'ar-SA',
+        'de': 'de-DE',
+        '': 'en-US',
+        'es': 'es-419',
+        'es-es': 'es-ES',
+        'fr': 'fr-FR',
+        'it': 'it-IT',
+        'pt-br': 'pt-BR',
+        'pt-pt': 'pt-PT',
+        'ru': 'ru-RU',
+        'hi': 'hi-IN',
+    }
 
     @property
     def is_logged_in(self):
-        return self._get_cookies(self._BASE_URL).get('etp_rt')
+        return bool(self._get_cookies(self._BASE_URL).get('etp_rt'))
 
     def _perform_login(self, username, password):
         if self.is_logged_in:
@@ -62,49 +75,49 @@ def _perform_login(self, username, password):
         if not self.is_logged_in:
             raise ExtractorError('Login succeeded but did not set etp_rt cookie')
 
-    def _update_query(self, lang):
-        if lang in CrunchyrollBaseIE._QUERY:
-            return
-
-        webpage = self._download_webpage(
-            f'{self._BASE_URL}/{lang}', None, note=f'Retrieving main page (lang={lang or None})')
-
-        initial_state = self._search_json(r'__INITIAL_STATE__\s*=', webpage, 'initial state', None)
-        CrunchyrollBaseIE._QUERY[lang] = traverse_obj(initial_state, {
-            'locale': ('localization', 'locale'),
-        }) or None
-
-        if CrunchyrollBaseIE._BASIC_AUTH:
-            return
-
-        app_config = self._search_json(r'__APP_CONFIG__\s*=', webpage, 'app config', None)
-        cx_api_param = app_config['cxApiParams']['accountAuthClientId' if self.is_logged_in else 'anonClientId']
-        self.write_debug(f'Using cxApiParam={cx_api_param}')
-        CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode()
-
     def _update_auth(self):
         if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_REFRESH > time_seconds():
             return
 
-        assert CrunchyrollBaseIE._BASIC_AUTH, '_update_query needs to be called at least one time beforehand'
+        if not CrunchyrollBaseIE._BASIC_AUTH:
+            cx_api_param = self._CLIENT_ID[self.is_logged_in]
+            self.write_debug(f'Using cxApiParam={cx_api_param}')
+            CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode()
+
         grant_type = 'etp_rt_cookie' if self.is_logged_in else 'client_id'
-        auth_response = self._download_json(
-            f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
-            headers={'Authorization': CrunchyrollBaseIE._BASIC_AUTH}, data=f'grant_type={grant_type}'.encode())
+        try:
+            auth_response = self._download_json(
+                f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
+                headers={'Authorization': CrunchyrollBaseIE._BASIC_AUTH}, data=f'grant_type={grant_type}'.encode())
+        except ExtractorError as error:
+            if isinstance(error.cause, HTTPError) and error.cause.status == 403:
+                raise ExtractorError(
+                    'Request blocked by Cloudflare; navigate to Crunchyroll in your browser, '
+                    'then pass the fresh cookies (with --cookies-from-browser or --cookies) '
+                    'and your browser\'s User-Agent (with --user-agent)', expected=True)
+            raise
 
         CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']}
         CrunchyrollBaseIE._AUTH_REFRESH = time_seconds(seconds=traverse_obj(auth_response, ('expires_in', {float_or_none}), default=300) - 10)
 
+    def _locale_from_language(self, language):
+        config_locale = self._configuration_arg('metadata', ie_key=CrunchyrollBetaIE, casesense=True)
+        return config_locale[0] if config_locale else self._LOCALE_LOOKUP.get(language)
+
     def _call_base_api(self, endpoint, internal_id, lang, note=None, query={}):
-        self._update_query(lang)
         self._update_auth()
 
         if not endpoint.startswith('/'):
             endpoint = f'/{endpoint}'
 
+        query = query.copy()
+        locale = self._locale_from_language(lang)
+        if locale:
+            query['locale'] = locale
+
         return self._download_json(
             f'{self._BASE_URL}{endpoint}', internal_id, note or f'Calling API: {endpoint}',
-            headers=CrunchyrollBaseIE._AUTH_HEADERS, query={**CrunchyrollBaseIE._QUERY[lang], **query})
+            headers=CrunchyrollBaseIE._AUTH_HEADERS, query=query)
 
     def _call_api(self, path, internal_id, lang, note='api', query={}):
         if not path.startswith(f'/content/v2/{self._API_ENDPOINT}/'):
@@ -206,7 +219,7 @@ class CrunchyrollBetaIE(CrunchyrollCmsBaseIE):
     IE_NAME = 'crunchyroll'
     _VALID_URL = r'''(?x)
         https?://(?:beta\.|www\.)?crunchyroll\.com/
-        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        (?:(?P<lang>\w{2}(?:-\w{2})?)/)?
         watch/(?!concert|musicvideo)(?P<id>\w+)'''
     _TESTS = [{
         # Premium only
@@ -304,7 +317,7 @@ class CrunchyrollBetaIE(CrunchyrollCmsBaseIE):
         },
         'playlist_mincount': 5,
     }, {
-        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y',
+        'url': 'https://www.crunchyroll.com/de/watch/GY2P1Q98Y',
         'only_matching': True,
     }, {
         'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',

From e57eb98222d29cc4c09ee975d3c492274a6e5be3 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 21 Jul 2023 21:32:49 -0500
Subject: [PATCH 462/871] [fd/external] Fix ffmpeg input from stdin (#7655)

Bugfix for 1ceb657bdd254ad961489e5060f2ccc7d556b729

Authored by: bashonly
---
 test/test_downloader_external.py | 5 +++++
 yt_dlp/downloader/external.py    | 5 +++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/test_downloader_external.py b/test/test_downloader_external.py
index d3d74df04..62f7d45d4 100644
--- a/test/test_downloader_external.py
+++ b/test/test_downloader_external.py
@@ -129,6 +129,11 @@ def test_make_cmd(self):
                 'ffmpeg', '-y', '-hide_banner', '-cookies', 'test=ytdlp; path=/; domain=.example.com;\r\n',
                 '-i', 'http://www.example.com/', '-c', 'copy', '-f', 'mp4', 'file:test'])
 
+            # Test with non-url input (ffmpeg reads from stdin '-' for websockets)
+            downloader._call_downloader('test', {'url': 'x', 'ext': 'mp4'})
+            self.assertEqual(self._args, [
+                'ffmpeg', '-y', '-hide_banner', '-i', 'x', '-c', 'copy', '-f', 'mp4', 'file:test'])
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index 4f52f6e8d..d3c3eba88 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -559,12 +559,13 @@ def _call_downloader(self, tmpfilename, info_dict):
 
         selected_formats = info_dict.get('requested_formats') or [info_dict]
         for i, fmt in enumerate(selected_formats):
-            cookies = self.ydl.cookiejar.get_cookies_for_url(fmt['url'])
+            is_http = re.match(r'^https?://', fmt['url'])
+            cookies = self.ydl.cookiejar.get_cookies_for_url(fmt['url']) if is_http else []
             if cookies:
                 args.extend(['-cookies', ''.join(
                     f'{cookie.name}={cookie.value}; path={cookie.path}; domain={cookie.domain};\r\n'
                     for cookie in cookies)])
-            if fmt.get('http_headers') and re.match(r'^https?://', fmt['url']):
+            if fmt.get('http_headers') and is_http:
                 # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
                 # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
                 args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in fmt['http_headers'].items())])

From 9f66247289b9f8ecf931833b3f5f127274dd2161 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 19 Jul 2023 07:11:52 +0530
Subject: [PATCH 463/871] [ie/abematv] Temporary fix for protocol handler

Closes #7622
---
 yt_dlp/extractor/abematv.py | 79 +++++--------------------------------
 1 file changed, 10 insertions(+), 69 deletions(-)

diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index 98ece8da7..163b83c6d 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -27,74 +27,18 @@
     update_url_query,
 )
 
-# NOTE: network handler related code is temporary thing until network stack overhaul PRs are merged (#2861/#2862)
 
-
-def add_opener(ydl, handler):
-    ''' Add a handler for opening URLs, like _download_webpage '''
+def add_opener(ydl, handler):  # FIXME: Create proper API in .networking
+    """Add a handler for opening URLs, like _download_webpage"""
     # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426
     # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605
-    assert isinstance(ydl._opener, urllib.request.OpenerDirector)
-    ydl._opener.add_handler(handler)
-
-
-def remove_opener(ydl, handler):
-    '''
-    Remove handler(s) for opening URLs
-    @param handler Either handler object itself or handler type.
-    Specifying handler type will remove all handler which isinstance returns True.
-    '''
-    # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426
-    # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605
-    opener = ydl._opener
-    assert isinstance(ydl._opener, urllib.request.OpenerDirector)
-    if isinstance(handler, (type, tuple)):
-        find_cp = lambda x: isinstance(x, handler)
-    else:
-        find_cp = lambda x: x is handler
-
-    removed = []
-    for meth in dir(handler):
-        if meth in ["redirect_request", "do_open", "proxy_open"]:
-            # oops, coincidental match
-            continue
-
-        i = meth.find("_")
-        protocol = meth[:i]
-        condition = meth[i + 1:]
-
-        if condition.startswith("error"):
-            j = condition.find("_") + i + 1
-            kind = meth[j + 1:]
-            try:
-                kind = int(kind)
-            except ValueError:
-                pass
-            lookup = opener.handle_error.get(protocol, {})
-            opener.handle_error[protocol] = lookup
-        elif condition == "open":
-            kind = protocol
-            lookup = opener.handle_open
-        elif condition == "response":
-            kind = protocol
-            lookup = opener.process_response
-        elif condition == "request":
-            kind = protocol
-            lookup = opener.process_request
-        else:
-            continue
-
-        handlers = lookup.setdefault(kind, [])
-        if handlers:
-            handlers[:] = [x for x in handlers if not find_cp(x)]
-
-        removed.append(x for x in handlers if find_cp(x))
-
-    if removed:
-        for x in opener.handlers:
-            if find_cp(x):
-                x.add_parent(None)
-        opener.handlers[:] = [x for x in opener.handlers if not find_cp(x)]
+    rh = ydl._request_director.handlers['Urllib']
+    if 'abematv-license' in rh._SUPPORTED_URL_SCHEMES:
+        return
+    opener = rh._get_instance(cookiejar=ydl.cookiejar, proxies=ydl.proxies)
+    assert isinstance(opener, urllib.request.OpenerDirector)
+    opener.add_handler(handler)
+    rh._SUPPORTED_URL_SCHEMES = (*rh._SUPPORTED_URL_SCHEMES, 'abematv-license')
 
 
 class AbemaLicenseHandler(urllib.request.BaseHandler):
@@ -140,7 +84,7 @@ def abematv_license_open(self, url):
         ticket = urllib.parse.urlparse(url).netloc
         response_data = self._get_videokey_from_ticket(ticket)
         return urllib.response.addinfourl(io.BytesIO(response_data), headers={
-            'Content-Length': len(response_data),
+            'Content-Length': str(len(response_data)),
         }, url=url, code=200)
 
 
@@ -212,10 +156,7 @@ def _get_device_token(self):
             })
         AbemaTVBaseIE._USERTOKEN = user_data['token']
 
-        # don't allow adding it 2 times or more, though it's guarded
-        remove_opener(self._downloader, AbemaLicenseHandler)
         add_opener(self._downloader, AbemaLicenseHandler(self))
-
         return self._USERTOKEN
 
     def _get_media_token(self, invalidate=False, to_show=True):

From a264433c9fba147ecae2420091614186cfeeb895 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 22 Jul 2023 07:52:55 +0530
Subject: [PATCH 464/871] [outtmpl] Fix replacement for `playlist_index`

---
 test/test_YoutubeDL.py | 1 +
 yt_dlp/YoutubeDL.py    | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index b4f770ca5..c54c3ea5c 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -687,6 +687,7 @@ def test(tmpl, expected, *, info=None, **params):
         test('%(duration_string)s', ('27:46:40', '27-46-40'))
         test('%(resolution)s', '1080p')
         test('%(playlist_index|)s', '001')
+        test('%(playlist_index&{}!)s', '001!')
         test('%(playlist_autonumber)s', '02')
         test('%(autonumber)s', '00001')
         test('%(autonumber+2)03d', '005', autonumber_start=3)
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 324f9e99c..dae29d9f9 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1300,16 +1300,16 @@ def create_key(outer_mobj):
                 else:
                     break
 
-            fmt = outer_mobj.group('format')
-            if fmt == 's' and value is not None and last_field in field_size_compat_map.keys():
-                fmt = f'0{field_size_compat_map[last_field]:d}d'
-
             if None not in (value, replacement):
                 try:
                     value = replacement_formatter.format(replacement, value)
                 except ValueError:
                     value, default = None, na
 
+            fmt = outer_mobj.group('format')
+            if fmt == 's' and last_field in field_size_compat_map.keys() and isinstance(value, int):
+                fmt = f'0{field_size_compat_map[last_field]:d}d'
+
             flags = outer_mobj.group('conversion') or ''
             str_fmt = f'{fmt[:-1]}s'
             if value is None:

From 994f7ef8e6003f4b7b258528755d0b6adcc31714 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 22 Jul 2023 07:54:25 +0530
Subject: [PATCH 465/871] [ie/generic] Fix generic title for embeds

Closes #7067
---
 yt_dlp/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 8fa4c6221..f5c59a093 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -2562,7 +2562,7 @@ def _real_extract(self, url):
         self._downloader.write_debug('Looking for embeds')
         embeds = list(self._extract_embeds(original_url, webpage, urlh=full_response, info_dict=info_dict))
         if len(embeds) == 1:
-            return {**info_dict, **embeds[0]}
+            return merge_dicts(embeds[0], info_dict)
         elif embeds:
             return self.playlist_result(embeds, **info_dict)
         raise UnsupportedError(url)

From 81b4712bca608b9015aa68a4d96661d56e9cb894 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 22 Jul 2023 08:28:38 +0530
Subject: [PATCH 466/871] [extractor] Fix `--load-pages`

---
 yt_dlp/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index d44918776..64a280dc0 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -1042,7 +1042,7 @@ def download_content(self, url_or_request, video_id, note=note, errnote=errnote,
                              fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
             if self.get_param('load_pages'):
                 url_or_request = self._create_request(url_or_request, data, headers, query)
-                filename = self._request_dump_filename(url_or_request.full_url, video_id)
+                filename = self._request_dump_filename(url_or_request.url, video_id)
                 self.to_screen(f'Loading request from {filename}')
                 try:
                     with open(filename, 'rb') as dumpf:

From e0c4db04dc82a699bdabd9821ddc239ebe17d30a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 22 Jul 2023 08:56:45 +0530
Subject: [PATCH 467/871] [compat] Add `types.NoneType`

---
 yt_dlp/compat/types.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 yt_dlp/compat/types.py

diff --git a/yt_dlp/compat/types.py b/yt_dlp/compat/types.py
new file mode 100644
index 000000000..ae7024564
--- /dev/null
+++ b/yt_dlp/compat/types.py
@@ -0,0 +1,12 @@
+# flake8: noqa: F405
+from types import *  # noqa: F403
+
+from .compat_utils import passthrough_module
+
+passthrough_module(__name__, 'types')
+del passthrough_module
+
+try:
+    NoneType  # >= 3.10
+except NameError:
+    NoneType = type(None)

From 62b5c94cadaa5f596dc1a7083db9db12efe357be Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 22 Jul 2023 09:08:12 +0530
Subject: [PATCH 468/871] [cleanup] Misc fixes

Closes #7528
---
 Changelog.md                       |  4 ++--
 README.md                          |  4 ++--
 devscripts/changelog_override.json |  7 ++++++-
 devscripts/make_changelog.py       |  4 ++--
 test/test_YoutubeDL.py             |  3 +--
 test/test_YoutubeDLCookieJar.py    | 12 ++++++------
 yt_dlp/YoutubeDL.py                | 13 +++++++------
 yt_dlp/compat/_legacy.py           |  4 ++--
 yt_dlp/cookies.py                  | 10 +++++-----
 yt_dlp/downloader/external.py      |  2 +-
 yt_dlp/networking/_urllib.py       |  4 ++--
 yt_dlp/networking/common.py        |  2 +-
 12 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/Changelog.md b/Changelog.md
index 622ae68b9..32cdaca2a 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -10,7 +10,7 @@ #### Important changes
 - Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)
     - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains
     - Cookies are scoped when passed to external downloaders
-    - Add `cookie` field to info.json and deprecate `http_headers.Cookie`
+    - Add `cookies` field to info.json and deprecate `http_headers.Cookie`
 
 #### Core changes
 - [Allow extractors to mark formats as potentially DRM](https://github.com/yt-dlp/yt-dlp/commit/bc344cd456380999c1ee74554dfd432a38f32ec7) ([#7396](https://github.com/yt-dlp/yt-dlp/issues/7396)) by [pukkandan](https://github.com/pukkandan)
@@ -51,7 +51,7 @@ #### Downloader changes
 - **http**: [Avoid infinite loop when no data is received](https://github.com/yt-dlp/yt-dlp/commit/662ef1e910b72e57957f06589925b2332ba52821) by [pukkandan](https://github.com/pukkandan)
 
 #### Misc. changes
-- [Add CodeQL workflow](https://github.com/yt-dlp/yt-dlp/commit/6355b5f1e1e8e7f4ef866d71d51e03baf0e82f17) ([#7497](https://github.com/yt-dlp/yt-dlp/issues/7497)) by [pukkandan](https://github.com/pukkandan)
+- [Add CodeQL workflow](https://github.com/yt-dlp/yt-dlp/commit/6355b5f1e1e8e7f4ef866d71d51e03baf0e82f17) ([#7497](https://github.com/yt-dlp/yt-dlp/issues/7497)) by [jorgectf](https://github.com/jorgectf)
 - **cleanup**: Miscellaneous: [337734d](https://github.com/yt-dlp/yt-dlp/commit/337734d4a8a6500bc65434843db346b5cbd05e81) by [pukkandan](https://github.com/pukkandan)
 - **docs**: [Minor fixes](https://github.com/yt-dlp/yt-dlp/commit/b532a3481046e1eabb6232ee8196fb696c356ff6) by [pukkandan](https://github.com/pukkandan)
 - **make_changelog**: [Skip reverted commits](https://github.com/yt-dlp/yt-dlp/commit/fa44802809d189fca0f4782263d48d6533384503) by [pukkandan](https://github.com/pukkandan)
diff --git a/README.md b/README.md
index 655cd41f5..ff88f817c 100644
--- a/README.md
+++ b/README.md
@@ -1569,7 +1569,7 @@ ## Sorting Formats
  - `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other). If `--prefer-free-formats` is used, the order changes to `ogg` > `opus` > `webm` > `mp3` > `m4a` > `aac`
  - `ext`: Equivalent to `vext,aext`
  - `filesize`: Exact filesize, if known in advance
- - `fs_approx`: Approximate filesize calculated from the manifests
+ - `fs_approx`: Approximate filesize
  - `size`: Exact filesize if available, otherwise approximate filesize
  - `height`: Height of video
  - `width`: Width of video
@@ -1580,7 +1580,7 @@ ## Sorting Formats
  - `tbr`: Total average bitrate in KBit/s
  - `vbr`: Average video bitrate in KBit/s
  - `abr`: Average audio bitrate in KBit/s
- - `br`: Equivalent to using `tbr,vbr,abr`
+ - `br`: Average bitrate in KBit/s, `tbr`/`vbr`/`abr`
  - `asr`: Audio sample rate in Hz
  
 **Deprecation warning**: Many of these fields have (currently undocumented) aliases, that may be removed in a future version. It is recommended to use only the documented field names.
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index f573a7463..d03db3f23 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -63,6 +63,11 @@
     {
         "action": "add",
         "when": "1ceb657bdd254ad961489e5060f2ccc7d556b729",
-        "short": "[priority] Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)\n    - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains\n    - Cookies are scoped when passed to external downloaders\n    - Add `cookie` field to info.json and deprecate `http_headers.Cookie`"
+        "short": "[priority] Security: [[CVE-2023-35934](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-35934)] Fix [Cookie leak](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj)\n    - `--add-header Cookie:` is deprecated and auto-scoped to input URL domains\n    - Cookies are scoped when passed to external downloaders\n    - Add `cookies` field to info.json and deprecate `http_headers.Cookie`"
+    },
+    {
+        "action": "change",
+        "when": "b03fa7834579a01cc5fba48c0e73488a16683d48",
+        "short": "[ie/twitter] Revert 92315c03774cfabb3a921884326beb4b981f786b"
     }
 ]
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index 157c66126..84f72d52f 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -53,10 +53,10 @@ def commit_lookup(cls):
                     'cookies',
                     'core',
                     'dependencies',
+                    'formats',
                     'jsinterp',
                     'networking',
                     'outtmpl',
-                    'formats',
                     'plugins',
                     'update',
                     'upstream',
@@ -254,7 +254,7 @@ class CommitRange:
         (?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
         ''', re.VERBOSE | re.DOTALL)
     EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE)
-    REVERT_RE = re.compile(r'(?i:Revert)\s+([\da-f]{40})')
+    REVERT_RE = re.compile(r'(?:\[[^\]]+\]\s+)?(?i:Revert)\s+([\da-f]{40})')
     FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})')
     UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)')
 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index c54c3ea5c..ab1250848 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -26,7 +26,6 @@
 )
 from yt_dlp.utils.traversal import traverse_obj
 
-
 TEST_URL = 'http://localhost/sample.mp4'
 
 
@@ -687,7 +686,7 @@ def test(tmpl, expected, *, info=None, **params):
         test('%(duration_string)s', ('27:46:40', '27-46-40'))
         test('%(resolution)s', '1080p')
         test('%(playlist_index|)s', '001')
-        test('%(playlist_index&{}!)s', '001!')
+        test('%(playlist_index&{}!)s', '1!')
         test('%(playlist_autonumber)s', '02')
         test('%(autonumber)s', '00001')
         test('%(autonumber+2)03d', '005', autonumber_start=3)
diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py
index 0b7a0acdb..fdb9baee5 100644
--- a/test/test_YoutubeDLCookieJar.py
+++ b/test/test_YoutubeDLCookieJar.py
@@ -17,10 +17,10 @@
 class TestYoutubeDLCookieJar(unittest.TestCase):
     def test_keep_session_cookies(self):
         cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt')
-        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        cookiejar.load()
         tf = tempfile.NamedTemporaryFile(delete=False)
         try:
-            cookiejar.save(filename=tf.name, ignore_discard=True, ignore_expires=True)
+            cookiejar.save(filename=tf.name)
             temp = tf.read().decode()
             self.assertTrue(re.search(
                 r'www\.foobar\.foobar\s+FALSE\s+/\s+TRUE\s+0\s+YoutubeDLExpiresEmpty\s+YoutubeDLExpiresEmptyValue', temp))
@@ -32,7 +32,7 @@ def test_keep_session_cookies(self):
 
     def test_strip_httponly_prefix(self):
         cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt')
-        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        cookiejar.load()
 
         def assert_cookie_has_value(key):
             self.assertEqual(cookiejar._cookies['www.foobar.foobar']['/'][key].value, key + '_VALUE')
@@ -42,20 +42,20 @@ def assert_cookie_has_value(key):
 
     def test_malformed_cookies(self):
         cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/malformed_cookies.txt')
-        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        cookiejar.load()
         # Cookies should be empty since all malformed cookie file entries
         # will be ignored
         self.assertFalse(cookiejar._cookies)
 
     def test_get_cookie_header(self):
         cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt')
-        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        cookiejar.load()
         header = cookiejar.get_cookie_header('https://www.foobar.foobar')
         self.assertIn('HTTPONLY_COOKIE', header)
 
     def test_get_cookies_for_url(self):
         cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt')
-        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        cookiejar.load()
         cookies = cookiejar.get_cookies_for_url('https://www.foobar.foobar/')
         self.assertEqual(len(cookies), 2)
         cookies = cookiejar.get_cookies_for_url('https://foobar.foobar/')
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index dae29d9f9..c9cf07e53 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -572,7 +572,7 @@ class YoutubeDL:
         'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels',
         'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns',
         'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start',
-        'preference', 'language', 'language_preference', 'quality', 'source_preference',
+        'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies',
         'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'hls_aes', 'downloader_options',
         'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time'
     }
@@ -621,7 +621,8 @@ def __init__(self, params=None, auto_init=True):
 
         if self.params.get('no_color'):
             if self.params.get('color') is not None:
-                self.report_warning('Overwriting params from "color" with "no_color"')
+                self.params.setdefault('_warnings', []).append(
+                    'Overwriting params from "color" with "no_color"')
             self.params['color'] = 'no_color'
 
         term_allow_color = os.environ.get('TERM', '').lower() != 'dumb'
@@ -949,7 +950,7 @@ def __enter__(self):
 
     def save_cookies(self):
         if self.params.get('cookiefile') is not None:
-            self.cookiejar.save(ignore_discard=True, ignore_expires=True)
+            self.cookiejar.save()
 
     def __exit__(self, *args):
         self.restore_console_title()
@@ -3290,7 +3291,7 @@ def existing_video_file(*filepaths):
                 fd, success = None, True
                 if info_dict.get('protocol') or info_dict.get('url'):
                     fd = get_suitable_downloader(info_dict, self.params, to_stdout=temp_filename == '-')
-                    if fd is not FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and (
+                    if fd != FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and (
                             info_dict.get('section_start') or info_dict.get('section_end')):
                         msg = ('This format cannot be partially downloaded' if FFmpegFD.available()
                                else 'You have requested downloading the video partially, but ffmpeg is not installed')
@@ -3451,7 +3452,7 @@ def ffmpeg_fixup(cndn, msg, cls):
                     postprocessed_by_ffmpeg = info_dict.get('requested_formats') or any((
                         isinstance(pp, FFmpegVideoConvertorPP)
                         and resolve_recode_mapping(ext, pp.mapping)[0] not in (ext, None)
-                    ) for pp in self._pps['post_process'])
+                    ) for pp in self._pps['post_process']) or fd == FFmpegFD
 
                     if not postprocessed_by_ffmpeg:
                         ffmpeg_fixup(ext == 'm4a' and info_dict.get('container') == 'm4a_dash',
@@ -4031,7 +4032,7 @@ def _opener(self):
         """
         Get a urllib OpenerDirector from the Urllib handler (deprecated).
         """
-        self.deprecation_warning('YoutubeDL._opener() is deprecated, use YoutubeDL.urlopen()')
+        self.deprecation_warning('YoutubeDL._opener is deprecated, use YoutubeDL.urlopen()')
         handler = self._request_director.handlers['Urllib']
         return handler._get_instance(cookiejar=self.cookiejar, proxies=self.proxies)
 
diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py
index 912907a02..90ccf0f14 100644
--- a/yt_dlp/compat/_legacy.py
+++ b/yt_dlp/compat/_legacy.py
@@ -16,12 +16,12 @@
 import shutil
 import socket
 import struct
+import subprocess
 import tokenize
 import urllib.error
 import urllib.parse
 import urllib.request
 import xml.etree.ElementTree as etree
-from subprocess import DEVNULL
 
 # isort: split
 import asyncio  # noqa: F401
@@ -85,7 +85,7 @@ def compat_setenv(key, value, env=os.environ):
 compat_Struct = struct.Struct
 compat_struct_pack = struct.pack
 compat_struct_unpack = struct.unpack
-compat_subprocess_get_DEVNULL = lambda: DEVNULL
+compat_subprocess_get_DEVNULL = lambda: subprocess.DEVNULL
 compat_tokenize_tokenize = tokenize.tokenize
 compat_urllib_error = urllib.error
 compat_urllib_HTTPError = urllib.error.HTTPError
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index 16f1918e6..80428c747 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -97,7 +97,7 @@ def load_cookies(cookie_file, browser_specification, ydl):
 
         jar = YoutubeDLCookieJar(cookie_file)
         if not is_filename or os.access(cookie_file, os.R_OK):
-            jar.load(ignore_discard=True, ignore_expires=True)
+            jar.load()
         cookie_jars.append(jar)
 
     return _merge_cookie_jars(cookie_jars)
@@ -1213,7 +1213,7 @@ def open(self, file, *, write=False):
                 file.truncate(0)
             yield file
 
-    def _really_save(self, f, ignore_discard=False, ignore_expires=False):
+    def _really_save(self, f, ignore_discard, ignore_expires):
         now = time.time()
         for cookie in self:
             if (not ignore_discard and cookie.discard
@@ -1234,7 +1234,7 @@ def _really_save(self, f, ignore_discard=False, ignore_expires=False):
                 name, value
             )))
 
-    def save(self, filename=None, *args, **kwargs):
+    def save(self, filename=None, ignore_discard=True, ignore_expires=True):
         """
         Save cookies to a file.
         Code is taken from CPython 3.6
@@ -1253,9 +1253,9 @@ def save(self, filename=None, *args, **kwargs):
 
         with self.open(filename, write=True) as f:
             f.write(self._HEADER)
-            self._really_save(f, *args, **kwargs)
+            self._really_save(f, ignore_discard, ignore_expires)
 
-    def load(self, filename=None, ignore_discard=False, ignore_expires=False):
+    def load(self, filename=None, ignore_discard=True, ignore_expires=True):
         """Load cookies from a file."""
         if filename is None:
             if self.filename is not None:
diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index d3c3eba88..4ce8a3bf7 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -137,7 +137,7 @@ def _write_cookies(self):
             self._cookies_tempfile = tmp_cookies.name
             self.to_screen(f'[download] Writing temporary cookies file to "{self._cookies_tempfile}"')
         # real_download resets _cookies_tempfile; if it's None then save() will write to cookiejar.filename
-        self.ydl.cookiejar.save(self._cookies_tempfile, ignore_discard=True, ignore_expires=True)
+        self.ydl.cookiejar.save(self._cookies_tempfile)
         return self.ydl.cookiejar.filename or self._cookies_tempfile
 
     def _call_downloader(self, tmpfilename, info_dict):
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index 8a76676d9..ff3a22c8c 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -28,7 +28,7 @@
     make_socks_proxy_opts,
     select_proxy,
 )
-from .common import Features, RequestHandler, Response, register
+from .common import Features, RequestHandler, Response, register_rh
 from .exceptions import (
     CertificateVerifyError,
     HTTPError,
@@ -372,7 +372,7 @@ def handle_response_read_exceptions(e):
         raise TransportError(cause=e) from e
 
 
-@register
+@register_rh
 class UrllibRH(RequestHandler, InstanceStoreMixin):
     _SUPPORTED_URL_SCHEMES = ('http', 'https', 'data', 'ftp')
     _SUPPORTED_PROXY_SCHEMES = ('http', 'socks4', 'socks4a', 'socks5', 'socks5h')
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
index 61196406d..7f7457978 100644
--- a/yt_dlp/networking/common.py
+++ b/yt_dlp/networking/common.py
@@ -105,7 +105,7 @@ def send(self, request: Request) -> Response:
 _REQUEST_HANDLERS = {}
 
 
-def register(handler):
+def register_rh(handler):
     """Register a RequestHandler class"""
     assert issubclass(handler, RequestHandler), f'{handler} must be a subclass of RequestHandler'
     assert handler.RH_KEY not in _REQUEST_HANDLERS, f'RequestHandler {handler.RH_KEY} already registered'

From e7057383380d7d53815f8feaf90ca3dcbde88983 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 22 Jul 2023 09:43:51 +0530
Subject: [PATCH 469/871] [ie/unsupported] List more sites with DRM

Closes #7323, #3072, #5740, #5767, #6125
---
 yt_dlp/extractor/unsupported.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/yt_dlp/extractor/unsupported.py b/yt_dlp/extractor/unsupported.py
index 1bc49786f..bbcbf3acb 100644
--- a/yt_dlp/extractor/unsupported.py
+++ b/yt_dlp/extractor/unsupported.py
@@ -42,6 +42,12 @@ class KnownDRMIE(UnsupportedInfoExtractor):
         r'vootkids\.com',
         r'nowtv\.it/watch',
         r'tv\.apple\.com',
+        r'primevideo\.com',
+        r'hulu\.com',
+        r'resource\.inkryptvideos\.com',
+        r'joyn\.de',
+        r'amazon\.(?:\w{2}\.)?\w+/gp/video',
+        r'music\.amazon\.(?:\w{2}\.)?\w+',
     )
 
     _TESTS = [{
@@ -111,6 +117,30 @@ class KnownDRMIE(UnsupportedInfoExtractor):
         # https://github.com/yt-dlp/yt-dlp/issues/5557
         'url': 'https://tv.apple.com/it/show/loot---una-fortuna/umc.cmc.5erbujil1mpazuerhr1udnk45?ctx_brand=tvs.sbd.4000',
         'only_matching': True,
+    }, {
+        # https://github.com/yt-dlp/yt-dlp/issues/3072
+        'url': 'https://www.joyn.de/play/serien/clannad/1-1-wo-die-kirschblueten-fallen',
+        'only_matching': True,
+    }, {
+        # https://github.com/yt-dlp/yt-dlp/issues/7323
+        'url': 'https://music.amazon.co.jp/albums/B088Y368TK',
+        'only_matching': True,
+    }, {
+        # https://github.com/yt-dlp/yt-dlp/issues/7323
+        'url': 'https://www.amazon.co.jp/gp/video/detail/B09X5HBYRS/',
+        'only_matching': True,
+    }, {
+        # https://github.com/yt-dlp/yt-dlp/issues/6125
+        'url': 'https://www.primevideo.com/region/eu/detail/0H3DDB4KBJFNDCKKLHNRLRLVKQ/ref=atv_br_def_r_br_c_unkc_1_10',
+        'only_matching': True,
+    }, {
+        # https://github.com/yt-dlp/yt-dlp/issues/5740
+        'url': 'https://resource.inkryptvideos.com/v2-a83ns52/iframe/index.html#video_id=7999ea0f6e03439eb40d056258c2d736&otp=xxx',
+        'only_matching': True,
+    }, {
+        # https://github.com/yt-dlp/yt-dlp/issues/5767
+        'url': 'https://www.hulu.com/movie/anthem-6b25fac9-da2b-45a3-8e09-e4156b0471cc',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 25b6e8f94679b4458550702b46e61249b875a4fd Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 22 Jul 2023 10:17:36 +0530
Subject: [PATCH 470/871] Fix e0c4db04dc82a699bdabd9821ddc239ebe17d30a for pypy

---
 yt_dlp/compat/types.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/compat/types.py b/yt_dlp/compat/types.py
index ae7024564..4aa3b0efd 100644
--- a/yt_dlp/compat/types.py
+++ b/yt_dlp/compat/types.py
@@ -7,6 +7,7 @@
 del passthrough_module
 
 try:
-    NoneType  # >= 3.10
-except NameError:
+    # NB: pypy has builtin NoneType, so checking NameError won't work
+    from types import NoneType  # >= 3.10
+except ImportError:
     NoneType = type(None)

From a250b247334ce9f641e709cbb64974da6034a2b3 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 22 Jul 2023 17:56:53 +0530
Subject: [PATCH 471/871] [compat] Ensure submodules are imported correctly

Closes #7663
---
 test/test_compat.py              | 6 +++---
 yt_dlp/compat/__init__.py        | 7 ++-----
 yt_dlp/compat/_deprecated.py     | 8 ++++++++
 yt_dlp/compat/urllib/__init__.py | 3 +++
 yt_dlp/utils/__init__.py         | 6 ++----
 5 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 003a97abf..71ca7f99f 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -9,15 +9,16 @@
 
 
 import struct
-import urllib.parse
 
 from yt_dlp import compat
+from yt_dlp.compat import urllib  # isort: split
 from yt_dlp.compat import (
     compat_etree_fromstring,
     compat_expanduser,
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlencode,
 )
+from yt_dlp.compat.urllib.request import getproxies
 
 
 class TestCompat(unittest.TestCase):
@@ -28,8 +29,7 @@ def test_compat_passthrough(self):
         with self.assertWarns(DeprecationWarning):
             compat.WINDOWS_VT_MODE
 
-        # TODO: Test submodule
-        # compat.asyncio.events  # Must not raise error
+        self.assertEqual(urllib.request.getproxies, getproxies)
 
         with self.assertWarns(DeprecationWarning):
             compat.compat_pycrypto_AES  # Must not raise error
diff --git a/yt_dlp/compat/__init__.py b/yt_dlp/compat/__init__.py
index a41a80ebb..832a9138d 100644
--- a/yt_dlp/compat/__init__.py
+++ b/yt_dlp/compat/__init__.py
@@ -1,14 +1,11 @@
 import os
 import sys
-import warnings
 import xml.etree.ElementTree as etree
 
-from ._deprecated import *  # noqa: F401, F403
 from .compat_utils import passthrough_module
 
-# XXX: Implement this the same way as other DeprecationWarnings without circular import
-passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
-    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=5))
+passthrough_module(__name__, '._deprecated')
+del passthrough_module
 
 
 # HTMLParseError has been deprecated in Python 3.3 and removed in
diff --git a/yt_dlp/compat/_deprecated.py b/yt_dlp/compat/_deprecated.py
index 14d37b236..607bae999 100644
--- a/yt_dlp/compat/_deprecated.py
+++ b/yt_dlp/compat/_deprecated.py
@@ -1,4 +1,12 @@
 """Deprecated - New code should avoid these"""
+import warnings
+
+from .compat_utils import passthrough_module
+
+# XXX: Implement this the same way as other DeprecationWarnings without circular import
+passthrough_module(__name__, '.._legacy', callback=lambda attr: warnings.warn(
+    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=6))
+del passthrough_module
 
 import base64
 import urllib.error
diff --git a/yt_dlp/compat/urllib/__init__.py b/yt_dlp/compat/urllib/__init__.py
index 6b6b8e103..b27cc6133 100644
--- a/yt_dlp/compat/urllib/__init__.py
+++ b/yt_dlp/compat/urllib/__init__.py
@@ -1,6 +1,9 @@
 # flake8: noqa: F405
 from urllib import *  # noqa: F403
 
+del request
+from . import request  # noqa: F401
+
 from ..compat_utils import passthrough_module
 
 passthrough_module(__name__, 'urllib')
diff --git a/yt_dlp/utils/__init__.py b/yt_dlp/utils/__init__.py
index 0b00adddb..c267e326f 100644
--- a/yt_dlp/utils/__init__.py
+++ b/yt_dlp/utils/__init__.py
@@ -1,6 +1,4 @@
-# flake8: noqa: F401, F403
-import warnings
-
+# flake8: noqa: F403
 from ..compat.compat_utils import passthrough_module
 
 passthrough_module(__name__, '._deprecated')
@@ -9,4 +7,4 @@
 # isort: off
 from .traversal import *
 from ._utils import *
-from ._utils import _configuration_args, _get_exe_version_output
+from ._utils import _configuration_args, _get_exe_version_output  # noqa: F401

From 11de6fec9c9b8d34d1f90c8e6218ec58a3471b58 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 22 Jul 2023 08:10:25 -0500
Subject: [PATCH 472/871] [ie/PatreonCampaign] Fix extraction (#7664)

Authored by: bashonly
---
 yt_dlp/extractor/patreon.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/patreon.py b/yt_dlp/extractor/patreon.py
index 447087436..9316789df 100644
--- a/yt_dlp/extractor/patreon.py
+++ b/yt_dlp/extractor/patreon.py
@@ -2,21 +2,21 @@
 
 from .common import InfoExtractor
 from .vimeo import VimeoIE
-
 from ..compat import compat_urllib_parse_unquote
 from ..networking.exceptions import HTTPError
 from ..utils import (
+    KNOWN_EXTENSIONS,
+    ExtractorError,
     clean_html,
     determine_ext,
-    ExtractorError,
     int_or_none,
-    KNOWN_EXTENSIONS,
     mimetype2ext,
     parse_iso8601,
     str_or_none,
     traverse_obj,
     try_get,
     url_or_none,
+    urljoin,
 )
 
 
@@ -404,8 +404,8 @@ def _entries(self, campaign_id):
             posts_json = self._call_api('posts', campaign_id, query=params, note='Downloading posts page %d' % page)
 
             cursor = traverse_obj(posts_json, ('meta', 'pagination', 'cursors', 'next'))
-            for post in posts_json.get('data') or []:
-                yield self.url_result(url_or_none(traverse_obj(post, ('attributes', 'patreon_url'))), 'Patreon')
+            for post_url in traverse_obj(posts_json, ('data', ..., 'attributes', 'patreon_url')):
+                yield self.url_result(urljoin('https://www.patreon.com/', post_url), PatreonIE)
 
             if cursor is None:
                 break

From 86aea0d3a213da3be1da638b9b828e6f0ee1d59f Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sun, 23 Jul 2023 17:17:15 +1200
Subject: [PATCH 473/871] [networking] Add strict Request extension checking
 (#7604)

Authored by: coletdjnz
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
---
 test/test_networking.py      | 54 ++++++++++++++++++++++++------------
 yt_dlp/networking/_urllib.py |  5 ++++
 yt_dlp/networking/common.py  | 34 +++++++++++------------
 3 files changed, 58 insertions(+), 35 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index d4eba2a5d..1bd6afc88 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -804,10 +804,10 @@ def test_httplib_validation_errors(self, handler):
             assert not isinstance(exc_info.value, TransportError)
 
 
-def run_validation(handler, fail, req, **handler_kwargs):
+def run_validation(handler, error, req, **handler_kwargs):
     with handler(**handler_kwargs) as rh:
-        if fail:
-            with pytest.raises(UnsupportedRequest):
+        if error:
+            with pytest.raises(error):
                 rh.validate(req)
         else:
             rh.validate(req)
@@ -824,6 +824,9 @@ class NoCheckRH(ValidationRH):
         _SUPPORTED_PROXY_SCHEMES = None
         _SUPPORTED_URL_SCHEMES = None
 
+        def _check_extensions(self, extensions):
+            extensions.clear()
+
     class HTTPSupportedRH(ValidationRH):
         _SUPPORTED_URL_SCHEMES = ('http',)
 
@@ -834,26 +837,26 @@ class HTTPSupportedRH(ValidationRH):
             ('https', False, {}),
             ('data', False, {}),
             ('ftp', False, {}),
-            ('file', True, {}),
+            ('file', UnsupportedRequest, {}),
             ('file', False, {'enable_file_urls': True}),
         ]),
         (NoCheckRH, [('http', False, {})]),
-        (ValidationRH, [('http', True, {})])
+        (ValidationRH, [('http', UnsupportedRequest, {})])
     ]
 
     PROXY_SCHEME_TESTS = [
         # scheme, expected to fail
         ('Urllib', [
             ('http', False),
-            ('https', True),
+            ('https', UnsupportedRequest),
             ('socks4', False),
             ('socks4a', False),
             ('socks5', False),
             ('socks5h', False),
-            ('socks', True),
+            ('socks', UnsupportedRequest),
         ]),
         (NoCheckRH, [('http', False)]),
-        (HTTPSupportedRH, [('http', True)]),
+        (HTTPSupportedRH, [('http', UnsupportedRequest)]),
     ]
 
     PROXY_KEY_TESTS = [
@@ -863,8 +866,22 @@ class HTTPSupportedRH(ValidationRH):
             ('unrelated', False),
         ]),
         (NoCheckRH, [('all', False)]),
-        (HTTPSupportedRH, [('all', True)]),
-        (HTTPSupportedRH, [('no', True)]),
+        (HTTPSupportedRH, [('all', UnsupportedRequest)]),
+        (HTTPSupportedRH, [('no', UnsupportedRequest)]),
+    ]
+
+    EXTENSION_TESTS = [
+        ('Urllib', [
+            ({'cookiejar': 'notacookiejar'}, AssertionError),
+            ({'cookiejar': CookieJar()}, False),
+            ({'timeout': 1}, False),
+            ({'timeout': 'notatimeout'}, AssertionError),
+            ({'unsupported': 'value'}, UnsupportedRequest),
+        ]),
+        (NoCheckRH, [
+            ({'cookiejar': 'notacookiejar'}, False),
+            ({'somerandom': 'test'}, False),  # but any extension is allowed through
+        ]),
     ]
 
     @pytest.mark.parametrize('handler,scheme,fail,handler_kwargs', [
@@ -907,15 +924,16 @@ def test_empty_proxy(self, handler):
     @pytest.mark.parametrize('proxy_url', ['//example.com', 'example.com', '127.0.0.1'])
     @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
     def test_missing_proxy_scheme(self, handler, proxy_url):
-        run_validation(handler, True, Request('http://', proxies={'http': 'example.com'}))
+        run_validation(handler, UnsupportedRequest, Request('http://', proxies={'http': 'example.com'}))
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
-    def test_cookiejar_extension(self, handler):
-        run_validation(handler, True, Request('http://', extensions={'cookiejar': 'notacookiejar'}))
-
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
-    def test_timeout_extension(self, handler):
-        run_validation(handler, True, Request('http://', extensions={'timeout': 'notavalidtimeout'}))
+    @pytest.mark.parametrize('handler,extensions,fail', [
+        (handler_tests[0], extensions, fail)
+        for handler_tests in EXTENSION_TESTS
+        for extensions, fail in handler_tests[1]
+    ], indirect=['handler'])
+    def test_extension(self, handler, extensions, fail):
+        run_validation(
+            handler, fail, Request('http://', extensions=extensions))
 
     def test_invalid_request_type(self):
         rh = self.ValidationRH(logger=FakeLogger())
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index ff3a22c8c..3fe5fa52e 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -385,6 +385,11 @@ def __init__(self, *, enable_file_urls: bool = False, **kwargs):
         if self.enable_file_urls:
             self._SUPPORTED_URL_SCHEMES = (*self._SUPPORTED_URL_SCHEMES, 'file')
 
+    def _check_extensions(self, extensions):
+        super()._check_extensions(extensions)
+        extensions.pop('cookiejar', None)
+        extensions.pop('timeout', None)
+
     def _create_instance(self, proxies, cookiejar):
         opener = urllib.request.OpenerDirector()
         handlers = [
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
index 7f7457978..ab26a0628 100644
--- a/yt_dlp/networking/common.py
+++ b/yt_dlp/networking/common.py
@@ -21,6 +21,7 @@
     TransportError,
     UnsupportedRequest,
 )
+from ..compat.types import NoneType
 from ..utils import (
     bug_reports_message,
     classproperty,
@@ -147,6 +148,7 @@ class RequestHandler(abc.ABC):
         a proxy url with an url scheme not in this list will raise an UnsupportedRequest.
 
     - `_SUPPORTED_FEATURES`: a tuple of supported features, as defined in Features enum.
+
     The above may be set to None to disable the checks.
 
     Parameters:
@@ -169,9 +171,14 @@ class RequestHandler(abc.ABC):
     Requests may have additional optional parameters defined as extensions.
      RequestHandler subclasses may choose to support custom extensions.
 
+    If an extension is supported, subclasses should extend _check_extensions(extensions)
+    to pop and validate the extension.
+    - Extensions left in `extensions` are treated as unsupported and UnsupportedRequest will be raised.
+
     The following extensions are defined for RequestHandler:
-    - `cookiejar`: Cookiejar to use for this request
-    - `timeout`: socket timeout to use for this request
+    - `cookiejar`: Cookiejar to use for this request.
+    - `timeout`: socket timeout to use for this request.
+    To enable these, add extensions.pop('<extension>', None) to _check_extensions
 
     Apart from the url protocol, proxies dict may contain the following keys:
     - `all`: proxy to use for all protocols. Used as a fallback if no proxy is set for a specific protocol.
@@ -263,26 +270,19 @@ def _check_proxies(self, proxies):
             if scheme not in self._SUPPORTED_PROXY_SCHEMES:
                 raise UnsupportedRequest(f'Unsupported proxy type: "{scheme}"')
 
-    def _check_cookiejar_extension(self, extensions):
-        if not extensions.get('cookiejar'):
-            return
-        if not isinstance(extensions['cookiejar'], CookieJar):
-            raise UnsupportedRequest('cookiejar is not a CookieJar')
-
-    def _check_timeout_extension(self, extensions):
-        if extensions.get('timeout') is None:
-            return
-        if not isinstance(extensions['timeout'], (float, int)):
-            raise UnsupportedRequest('timeout is not a float or int')
-
     def _check_extensions(self, extensions):
-        self._check_cookiejar_extension(extensions)
-        self._check_timeout_extension(extensions)
+        """Check extensions for unsupported extensions. Subclasses should extend this."""
+        assert isinstance(extensions.get('cookiejar'), (CookieJar, NoneType))
+        assert isinstance(extensions.get('timeout'), (float, int, NoneType))
 
     def _validate(self, request):
         self._check_url_scheme(request)
         self._check_proxies(request.proxies or self.proxies)
-        self._check_extensions(request.extensions)
+        extensions = request.extensions.copy()
+        self._check_extensions(extensions)
+        if extensions:
+            # TODO: add support for optional extensions
+            raise UnsupportedRequest(f'Unsupported extensions: {", ".join(extensions.keys())}')
 
     @wrap_request_errors
     def validate(self, request: Request):

From 39837ae3199aa934299badbd0d63243ed639e6c8 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 23 Jul 2023 18:29:45 -0500
Subject: [PATCH 474/871] [ie/triller] Fix unlisted video extraction (#7670)

Authored by: bashonly
---
 yt_dlp/extractor/triller.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/extractor/triller.py b/yt_dlp/extractor/triller.py
index c5d01c827..56e51fea8 100644
--- a/yt_dlp/extractor/triller.py
+++ b/yt_dlp/extractor/triller.py
@@ -66,13 +66,6 @@ def _get_comments(self, video_id, limit=15):
             'timestamp': ('timestamp', {unified_timestamp}),
         }))
 
-    def _check_user_info(self, user_info):
-        if user_info.get('private') and not user_info.get('followed_by_me'):
-            raise ExtractorError('This video is private', expected=True)
-        elif traverse_obj(user_info, 'blocked_by_user', 'blocking_user'):
-            raise ExtractorError('The author of the video is blocked', expected=True)
-        return user_info
-
     def _parse_video_info(self, video_info, username, user_id, display_id=None):
         video_id = str(video_info['id'])
         display_id = display_id or video_info.get('video_uuid')
@@ -231,8 +224,6 @@ def _real_extract(self, url):
             f'{self._API_BASE_URL}/api/videos/{display_id}', display_id,
             headers=self._API_HEADERS)['videos'][0]
 
-        self._check_user_info(video_info.get('user') or {})
-
         return self._parse_video_info(video_info, username, None, display_id)
 
 
@@ -287,9 +278,14 @@ def _entries(self, username, user_id, limit=6):
     def _real_extract(self, url):
         username = self._match_id(url)
 
-        user_info = self._check_user_info(self._download_json(
+        user_info = traverse_obj(self._download_json(
             f'{self._API_BASE_URL}/api/users/by_username/{username}',
-            username, note='Downloading user info', headers=self._API_HEADERS)['user'])
+            username, note='Downloading user info', headers=self._API_HEADERS), ('user', {dict})) or {}
+
+        if user_info.get('private') and user_info.get('followed_by_me') not in (True, 'true'):
+            raise ExtractorError('This user profile is private', expected=True)
+        elif traverse_obj(user_info, (('blocked_by_user', 'blocking_user'), {bool}), get_all=False):
+            raise ExtractorError('The author of the video is blocked', expected=True)
 
         user_id = str_or_none(user_info.get('user_id'))
         if not user_id:

From 550e65410a7a1b105923494ac44460a4dc1a15d9 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 23 Jul 2023 19:09:52 -0500
Subject: [PATCH 475/871] [ie] Extract subtitles from SMIL manifests (#7667)

Authored by: bashonly, pukkandan
---
 yt_dlp/extractor/common.py      | 46 +++++++++++++++++++--------------
 yt_dlp/extractor/livestream.py  |  5 ++--
 yt_dlp/extractor/mediaset.py    |  6 +++--
 yt_dlp/extractor/nbc.py         |  1 -
 yt_dlp/extractor/theplatform.py |  4 +--
 5 files changed, 34 insertions(+), 28 deletions(-)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 64a280dc0..b69ac1d65 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -2248,18 +2248,10 @@ def _extract_smil_formats_and_subtitles(self, smil_url, video_id, fatal=True, f4
         if res is False:
             assert not fatal
             return [], {}
-
         smil, urlh = res
-        smil_url = urlh.url
 
-        namespace = self._parse_smil_namespace(smil)
-
-        fmts = self._parse_smil_formats(
-            smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
-        subs = self._parse_smil_subtitles(
-            smil, namespace=namespace)
-
-        return fmts, subs
+        return self._parse_smil_formats_and_subtitles(smil, urlh.url, video_id, f4m_params=f4m_params,
+                                                      namespace=self._parse_smil_namespace(smil))
 
     def _extract_smil_formats(self, *args, **kwargs):
         fmts, subs = self._extract_smil_formats_and_subtitles(*args, **kwargs)
@@ -2285,9 +2277,8 @@ def _download_smil(self, smil_url, video_id, fatal=True, transform_source=None):
     def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
         namespace = self._parse_smil_namespace(smil)
 
-        formats = self._parse_smil_formats(
+        formats, subtitles = self._parse_smil_formats_and_subtitles(
             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
-        subtitles = self._parse_smil_subtitles(smil, namespace=namespace)
 
         video_id = os.path.splitext(url_basename(smil_url))[0]
         title = None
@@ -2326,7 +2317,14 @@ def _parse_smil_namespace(self, smil):
         return self._search_regex(
             r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
 
-    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
+    def _parse_smil_formats(self, *args, **kwargs):
+        fmts, subs = self._parse_smil_formats_and_subtitles(*args, **kwargs)
+        if subs:
+            self._report_ignoring_subs('SMIL')
+        return fmts
+
+    def _parse_smil_formats_and_subtitles(
+            self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
         base = smil_url
         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
             b = meta.get('base') or meta.get('httpBase')
@@ -2334,7 +2332,7 @@ def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_para
                 base = b
                 break
 
-        formats = []
+        formats, subtitles = [], {}
         rtmp_count = 0
         http_count = 0
         m3u8_count = 0
@@ -2382,8 +2380,9 @@ def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_para
             src_url = src_url.strip()
 
             if proto == 'm3u8' or src_ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
+                m3u8_formats, m3u8_subs = self._extract_m3u8_formats_and_subtitles(
                     src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
+                self._merge_subtitles(m3u8_subs, target=subtitles)
                 if len(m3u8_formats) == 1:
                     m3u8_count += 1
                     m3u8_formats[0].update({
@@ -2404,11 +2403,15 @@ def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_para
                 f4m_url += urllib.parse.urlencode(f4m_params)
                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
             elif src_ext == 'mpd':
-                formats.extend(self._extract_mpd_formats(
-                    src_url, video_id, mpd_id='dash', fatal=False))
+                mpd_formats, mpd_subs = self._extract_mpd_formats_and_subtitles(
+                    src_url, video_id, mpd_id='dash', fatal=False)
+                formats.extend(mpd_formats)
+                self._merge_subtitles(mpd_subs, target=subtitles)
             elif re.search(r'\.ism/[Mm]anifest', src_url):
-                formats.extend(self._extract_ism_formats(
-                    src_url, video_id, ism_id='mss', fatal=False))
+                ism_formats, ism_subs = self._extract_ism_formats_and_subtitles(
+                    src_url, video_id, ism_id='mss', fatal=False)
+                formats.extend(ism_formats)
+                self._merge_subtitles(ism_subs, target=subtitles)
             elif src_url.startswith('http') and self._is_valid_url(src, video_id):
                 http_count += 1
                 formats.append({
@@ -2439,7 +2442,10 @@ def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_para
                 'format_note': 'SMIL storyboards',
             })
 
-        return formats
+        smil_subs = self._parse_smil_subtitles(smil, namespace=namespace)
+        self._merge_subtitles(smil_subs, target=subtitles)
+
+        return formats, subtitles
 
     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
         urls = []
diff --git a/yt_dlp/extractor/livestream.py b/yt_dlp/extractor/livestream.py
index 692d6ab3a..a05a0fa9e 100644
--- a/yt_dlp/extractor/livestream.py
+++ b/yt_dlp/extractor/livestream.py
@@ -80,7 +80,8 @@ class LivestreamIE(InfoExtractor):
     }]
     _API_URL_TEMPLATE = 'http://livestream.com/api/accounts/%s/events/%s'
 
-    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
+    def _parse_smil_formats_and_subtitles(
+            self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
         base_ele = find_xpath_attr(
             smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
         base = base_ele.get('content') if base_ele is not None else 'http://livestreamvod-f.akamaihd.net/'
@@ -104,7 +105,7 @@ def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_para
                 'tbr': tbr,
                 'preference': -1000,  # Strictly inferior than all other formats?
             })
-        return formats
+        return formats, {}
 
     def _extract_video_info(self, video_data):
         video_id = compat_str(video_data['id'])
diff --git a/yt_dlp/extractor/mediaset.py b/yt_dlp/extractor/mediaset.py
index 1fa529914..e3b728dca 100644
--- a/yt_dlp/extractor/mediaset.py
+++ b/yt_dlp/extractor/mediaset.py
@@ -154,10 +154,12 @@ class MediasetIE(ThePlatformBaseIE):
         }
     }]
 
-    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
+    def _parse_smil_formats_and_subtitles(
+            self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
         for video in smil.findall(self._xpath_ns('.//video', namespace)):
             video.attrib['src'] = re.sub(r'(https?://vod05)t(-mediaset-it\.akamaized\.net/.+?.mpd)\?.+', r'\1\2', video.attrib['src'])
-        return super(MediasetIE, self)._parse_smil_formats(smil, smil_url, video_id, namespace, f4m_params, transform_rtmp_url)
+        return super(MediasetIE, self)._parse_smil_formats_and_subtitles(
+            smil, smil_url, video_id, namespace, f4m_params, transform_rtmp_url)
 
     def _check_drm_formats(self, tp_formats, video_id):
         has_nondrm, drm_manifest = False, ''
diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index 299b05174..b3c28ab55 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -131,7 +131,6 @@ class NBCIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'age_limit': 0,
                 'thumbnail': r're:https?://.+\.jpg',
             },
-            'expected_warnings': ['Ignoring subtitle tracks'],
             'params': {
                 'skip_download': 'm3u8',
             },
diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py
index 537f6f6cd..8307b912d 100644
--- a/yt_dlp/extractor/theplatform.py
+++ b/yt_dlp/extractor/theplatform.py
@@ -45,7 +45,7 @@ def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL d
                     raise ExtractorError(
                         error_element.attrib['abstract'], expected=True)
 
-        smil_formats = self._parse_smil_formats(
+        smil_formats, subtitles = self._parse_smil_formats_and_subtitles(
             meta, smil_url, video_id, namespace=default_ns,
             # the parameters are from syfy.com, other sites may use others,
             # they also work for nbc.com
@@ -65,8 +65,6 @@ def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL d
 
                 formats.append(_format)
 
-        subtitles = self._parse_smil_subtitles(meta, default_ns)
-
         return formats, subtitles
 
     def _download_theplatform_metadata(self, path, video_id):

From 95abea9a03289da1384e5bda3d590223ccc0a238 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Tue, 25 Jul 2023 07:18:52 +1200
Subject: [PATCH 476/871] [test] Fix `httplib_validation_errors` test for old
 Python versions (#7677)

Fixes https://github.com/yt-dlp/yt-dlp/issues/7674

Authored by: coletdjnz
---
 test/test_networking.py | 39 ++++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index 1bd6afc88..dbe28359b 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -785,22 +785,31 @@ def test_verify_cert_error_text(self, handler):
                 validate_and_send(rh, Request(f'https://127.0.0.1:{self.https_port}/headers'))
 
     @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
-    def test_httplib_validation_errors(self, handler):
+    @pytest.mark.parametrize('req,match,version_check', [
+        # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1256
+        # bpo-39603: Check implemented in 3.7.9+, 3.8.5+
+        (
+            Request('http://127.0.0.1', method='GET\n'),
+            'method can\'t contain control characters',
+            lambda v: v < (3, 7, 9) or (3, 8, 0) <= v < (3, 8, 5)
+        ),
+        # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1265
+        # bpo-38576: Check implemented in 3.7.8+, 3.8.3+
+        (
+            Request('http://127.0.0. 1', method='GET'),
+            'URL can\'t contain control characters',
+            lambda v: v < (3, 7, 8) or (3, 8, 0) <= v < (3, 8, 3)
+        ),
+        # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1288C31-L1288C50
+        (Request('http://127.0.0.1', headers={'foo\n': 'bar'}), 'Invalid header name', None),
+    ])
+    def test_httplib_validation_errors(self, handler, req, match, version_check):
+        if version_check and version_check(sys.version_info):
+            pytest.skip(f'Python {sys.version} version does not have the required validation for this test.')
+
         with handler() as rh:
-
-            # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1256
-            with pytest.raises(RequestError, match='method can\'t contain control characters') as exc_info:
-                validate_and_send(rh, Request('http://127.0.0.1', method='GET\n'))
-            assert not isinstance(exc_info.value, TransportError)
-
-            # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1265
-            with pytest.raises(RequestError, match='URL can\'t contain control characters') as exc_info:
-                validate_and_send(rh, Request('http://127.0.0. 1', method='GET\n'))
-            assert not isinstance(exc_info.value, TransportError)
-
-            # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1288C31-L1288C50
-            with pytest.raises(RequestError, match='Invalid header name') as exc_info:
-                validate_and_send(rh, Request('http://127.0.0.1', headers={'foo\n': 'bar'}))
+            with pytest.raises(RequestError, match=match) as exc_info:
+                validate_and_send(rh, req)
             assert not isinstance(exc_info.value, TransportError)
 
 

From dae349da97cafe7357106a8f3187fd48a2ad1210 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Thu, 27 Jul 2023 09:53:22 -0500
Subject: [PATCH 477/871] [ie/WrestleUniversePPV] Fix HLS AES key extraction

Fix bug in ef8fb7f029b816dfc95600727d84400591a3b5c5

Closes #7708
Authored by: bashonly
---
 yt_dlp/extractor/wrestleuniverse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/wrestleuniverse.py b/yt_dlp/extractor/wrestleuniverse.py
index 99a8f0120..dd12804db 100644
--- a/yt_dlp/extractor/wrestleuniverse.py
+++ b/yt_dlp/extractor/wrestleuniverse.py
@@ -300,7 +300,7 @@ def _real_extract(self, url):
             info['hls_aes'] = {
                 'key': hls_aes_key,
                 'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
-            },
+            }
         elif traverse_obj(video_data, ('hls', 'encryptType', {int})):
             self.report_warning('HLS AES-128 key was not found in API response')
 

From bbeacff7fcaa3b521066088a5ccbf34ef5070d1d Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Fri, 28 Jul 2023 02:56:02 +1200
Subject: [PATCH 478/871] [networking] Ignore invalid proxies in env (#7704)

Authored by: coletdjnz
---
 test/test_networking.py     |  8 +++++---
 yt_dlp/networking/common.py | 10 +++++++---
 yt_dlp/utils/networking.py  |  8 +++++++-
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index dbe28359b..f0938ab91 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -930,10 +930,10 @@ def test_empty_proxy(self, handler):
         run_validation(handler, False, Request('http://', proxies={'http': None}))
         run_validation(handler, False, Request('http://'), proxies={'http': None})
 
-    @pytest.mark.parametrize('proxy_url', ['//example.com', 'example.com', '127.0.0.1'])
+    @pytest.mark.parametrize('proxy_url', ['//example.com', 'example.com', '127.0.0.1', '/a/b/c'])
     @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
-    def test_missing_proxy_scheme(self, handler, proxy_url):
-        run_validation(handler, UnsupportedRequest, Request('http://', proxies={'http': 'example.com'}))
+    def test_invalid_proxy_url(self, handler, proxy_url):
+        run_validation(handler, UnsupportedRequest, Request('http://', proxies={'http': proxy_url}))
 
     @pytest.mark.parametrize('handler,extensions,fail', [
         (handler_tests[0], extensions, fail)
@@ -1126,9 +1126,11 @@ def test_legacy_server_connect_error(self):
         ('http', '__noproxy__', None),
         ('no', '127.0.0.1,foo.bar', '127.0.0.1,foo.bar'),
         ('https', 'example.com', 'http://example.com'),
+        ('https', '//example.com', 'http://example.com'),
         ('https', 'socks5://example.com', 'socks5h://example.com'),
         ('http', 'socks://example.com', 'socks4://example.com'),
         ('http', 'socks4://example.com', 'socks4://example.com'),
+        ('unrelated', '/bad/proxy', '/bad/proxy'),  # clean_proxies should ignore bad proxies
     ])
     def test_clean_proxy(self, proxy_key, proxy_url, expected):
         # proxies should be cleaned in urlopen()
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
index ab26a0628..3164df49b 100644
--- a/yt_dlp/networking/common.py
+++ b/yt_dlp/networking/common.py
@@ -262,9 +262,13 @@ def _check_proxies(self, proxies):
                 # Skip proxy scheme checks
                 continue
 
-            # Scheme-less proxies are not supported
-            if urllib.request._parse_proxy(proxy_url)[0] is None:
-                raise UnsupportedRequest(f'Proxy "{proxy_url}" missing scheme')
+            try:
+                if urllib.request._parse_proxy(proxy_url)[0] is None:
+                    # Scheme-less proxies are not supported
+                    raise UnsupportedRequest(f'Proxy "{proxy_url}" missing scheme')
+            except ValueError as e:
+                # parse_proxy may raise on some invalid proxy urls such as "/a/b/c"
+                raise UnsupportedRequest(f'Invalid proxy url "{proxy_url}": {e}')
 
             scheme = urllib.parse.urlparse(proxy_url).scheme.lower()
             if scheme not in self._SUPPORTED_PROXY_SCHEMES:
diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py
index ac355ddc8..e6515ec8e 100644
--- a/yt_dlp/utils/networking.py
+++ b/yt_dlp/utils/networking.py
@@ -98,7 +98,13 @@ def clean_proxies(proxies: dict, headers: HTTPHeaderDict):
             continue
         if proxy_url is not None:
             # Ensure proxies without a scheme are http.
-            proxy_scheme = urllib.request._parse_proxy(proxy_url)[0]
+            try:
+                proxy_scheme = urllib.request._parse_proxy(proxy_url)[0]
+            except ValueError:
+                # Ignore invalid proxy URLs. Sometimes these may be introduced through environment
+                # variables unrelated to proxy settings - e.g. Colab `COLAB_LANGUAGE_SERVER_PROXY`.
+                # If the proxy is going to be used, the Request Handler proxy validation will handle it.
+                continue
             if proxy_scheme is None:
                 proxies[proxy_key] = 'http://' + remove_start(proxy_url, '//')
 

From c03a58ec9933e4a42c2d8fa80b8a0ddb2cde64e6 Mon Sep 17 00:00:00 2001
From: Amirreza Aflakparast <84932095+AmirAflak@users.noreply.github.com>
Date: Fri, 28 Jul 2023 22:21:16 +0330
Subject: [PATCH 479/871] [ie/MotorTrendOnDemand] Update `_VALID_URL` (#7683)

Closes #7680
Authored by: AmirAflak
---
 yt_dlp/extractor/dplay.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/dplay.py b/yt_dlp/extractor/dplay.py
index 6404752f7..363b4bec9 100644
--- a/yt_dlp/extractor/dplay.py
+++ b/yt_dlp/extractor/dplay.py
@@ -746,7 +746,7 @@ class MotorTrendIE(DiscoveryPlusBaseIE):
 
 
 class MotorTrendOnDemandIE(DiscoveryPlusBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?motortrendondemand\.com/detail' + DPlayBaseIE._PATH_REGEX
+    _VALID_URL = r'https?://(?:www\.)?motortrend(?:ondemand\.com|\.com/plus)/detail' + DPlayBaseIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://www.motortrendondemand.com/detail/wheelstanding-dump-truck-stubby-bobs-comeback/37699/784',
         'info_dict': {
@@ -767,6 +767,25 @@ class MotorTrendOnDemandIE(DiscoveryPlusBaseIE):
             'upload_date': '20140101',
             'tags': [],
         },
+    }, {
+        'url': 'https://www.motortrend.com/plus/detail/roadworthy-rescues-teaser-trailer/4922860/',
+        'info_dict': {
+            'id': '4922860',
+            'ext': 'mp4',
+            'title': 'Roadworthy Rescues | Teaser Trailer',
+            'description': 'Derek Bieri helps Freiburger and Finnegan with their \'68 big-block Dart.',
+            'display_id': 'roadworthy-rescues-teaser-trailer/4922860',
+            'creator': 'Originals',
+            'series': 'Roadworthy Rescues',
+            'thumbnail': r're:^https?://.+\.jpe?g$',
+            'upload_date': '20220907',
+            'timestamp': 1662523200,
+            'duration': 1066.356,
+            'tags': [],
+        },
+    }, {
+        'url': 'https://www.motortrend.com/plus/detail/ugly-duckling/2450033/12439',
+        'only_matching': True,
     }]
 
     _PRODUCT = 'MTOD'

From a15fcd299e767a510debd8dc1646fe863b96ce0e Mon Sep 17 00:00:00 2001
From: nnoboa <90611593+nnoboa@users.noreply.github.com>
Date: Fri, 28 Jul 2023 14:52:07 -0400
Subject: [PATCH 480/871] [ie/Wimbledon] Add extractor (#7551)

Closes #7462
Authored by: nnoboa
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/wimbledon.py   | 61 +++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 yt_dlp/extractor/wimbledon.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ae73a9f96..9d935a7d1 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2363,6 +2363,7 @@
 from .whyp import WhypIE
 from .wikimedia import WikimediaIE
 from .willow import WillowIE
+from .wimbledon import WimbledonIE
 from .wimtv import WimTVIE
 from .whowatch import WhoWatchIE
 from .wistia import (
diff --git a/yt_dlp/extractor/wimbledon.py b/yt_dlp/extractor/wimbledon.py
new file mode 100644
index 000000000..ee4872e88
--- /dev/null
+++ b/yt_dlp/extractor/wimbledon.py
@@ -0,0 +1,61 @@
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    traverse_obj,
+)
+
+
+class WimbledonIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?wimbledon\.com/\w+/video/media/(?P<id>\d+).html'
+    _TESTS = [{
+        'url': 'https://www.wimbledon.com/en_GB/video/media/6330247525112.html',
+        'info_dict': {
+            'id': '6330247525112',
+            'ext': 'mp4',
+            'timestamp': 1687972186,
+            'description': '',
+            'thumbnail': r're:^https://[\w.-]+\.prod\.boltdns\.net/[^?#]+/image\.jpg',
+            'upload_date': '20230628',
+            'title': 'Coco Gauff | My Wimbledon Inspiration',
+            'tags': ['features', 'trending', 'homepage'],
+            'uploader_id': '3506358525001',
+            'duration': 163072.0,
+        },
+    }, {
+        'url': 'https://www.wimbledon.com/en_GB/video/media/6308703111112.html',
+        'info_dict': {
+            'id': '6308703111112',
+            'ext': 'mp4',
+            'thumbnail': r're:^https://[\w.-]+\.prod\.boltdns\.net/[^?#]+/image\.jpg',
+            'description': 'null',
+            'upload_date': '20220629',
+            'uploader_id': '3506358525001',
+            'title': 'Roblox | WimbleWorld ',
+            'duration': 101440.0,
+            'tags': ['features', 'kids'],
+            'timestamp': 1656500867,
+        },
+    }, {
+        'url': 'https://www.wimbledon.com/en_US/video/media/6309327106112.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.wimbledon.com/es_Es/video/media/6308377909112.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        metadata = self._download_json(
+            f'https://www.wimbledon.com/relatedcontent/rest/v2/wim_v1/en/content/wim_v1_{video_id}_en', video_id)
+
+        return {
+            '_type': 'url_transparent',
+            'url': f'http://players.brightcove.net/3506358525001/default_default/index.html?videoId={video_id}',
+            'ie_key': 'BrightcoveNew',
+            'id': video_id,
+            **traverse_obj(metadata, {
+                'title': 'title',
+                'description': 'description',
+                'duration': ('metadata', 'duration', {parse_duration}),
+            }),
+        }

From 4bf912282a34b58b6b35d8f7e6be535770c89c76 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 29 Jul 2023 10:40:20 +1200
Subject: [PATCH 481/871] [networking] Remove dot segments during URL
 normalization (#7662)

This implements RFC3986 5.2.4 remove_dot_segments during the URL normalization process.

Closes #3355, #6526

Authored by: coletdjnz
---
 test/test_networking.py      | 21 ++++++++++++++++++
 test/test_utils.py           | 42 ++++++++++++++++++++++++++++--------
 yt_dlp/cookies.py            |  6 +++---
 yt_dlp/networking/_urllib.py |  7 +++---
 yt_dlp/networking/common.py  |  5 ++---
 yt_dlp/utils/_legacy.py      |  4 +++-
 yt_dlp/utils/_utils.py       | 17 ---------------
 yt_dlp/utils/networking.py   | 38 ++++++++++++++++++++++++++++++++
 8 files changed, 104 insertions(+), 36 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index f0938ab91..684bf5f96 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -173,6 +173,12 @@ def do_GET(self):
             self.send_header('Location', self.path)
             self.send_header('Content-Length', '0')
             self.end_headers()
+        elif self.path == '/redirect_dotsegments':
+            self.send_response(301)
+            # redirect to /headers but with dot segments before
+            self.send_header('Location', '/a/b/./../../headers')
+            self.send_header('Content-Length', '0')
+            self.end_headers()
         elif self.path.startswith('/redirect_'):
             self._redirect()
         elif self.path.startswith('/method'):
@@ -355,6 +361,21 @@ def test_percent_encode(self, handler):
             assert res.status == 200
             res.close()
 
+    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    def test_remove_dot_segments(self, handler):
+        with handler() as rh:
+            # This isn't a comprehensive test,
+            # but it should be enough to check whether the handler is removing dot segments
+            res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/a/b/./../../headers'))
+            assert res.status == 200
+            assert res.url == f'http://127.0.0.1:{self.http_port}/headers'
+            res.close()
+
+            res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/redirect_dotsegments'))
+            assert res.status == 200
+            assert res.url == f'http://127.0.0.1:{self.http_port}/headers'
+            res.close()
+
     @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
     def test_unicode_path_redirection(self, handler):
         with handler() as rh:
diff --git a/test/test_utils.py b/test/test_utils.py
index b36bc04c2..453a01a1c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -47,8 +47,6 @@
     encode_base_n,
     encode_compat_str,
     encodeFilename,
-    escape_rfc3986,
-    escape_url,
     expand_path,
     extract_attributes,
     extract_basic_auth,
@@ -132,7 +130,12 @@
     xpath_text,
     xpath_with_ns,
 )
-from yt_dlp.utils.networking import HTTPHeaderDict
+from yt_dlp.utils.networking import (
+    HTTPHeaderDict,
+    escape_rfc3986,
+    normalize_url,
+    remove_dot_segments,
+)
 
 
 class TestUtil(unittest.TestCase):
@@ -933,24 +936,45 @@ def test_escape_rfc3986(self):
         self.assertEqual(escape_rfc3986('foo bar'), 'foo%20bar')
         self.assertEqual(escape_rfc3986('foo%20bar'), 'foo%20bar')
 
-    def test_escape_url(self):
+    def test_normalize_url(self):
         self.assertEqual(
-            escape_url('http://wowza.imust.org/srv/vod/telemb/new/UPLOAD/UPLOAD/20224_IncendieHavré_FD.mp4'),
+            normalize_url('http://wowza.imust.org/srv/vod/telemb/new/UPLOAD/UPLOAD/20224_IncendieHavré_FD.mp4'),
             'http://wowza.imust.org/srv/vod/telemb/new/UPLOAD/UPLOAD/20224_IncendieHavre%CC%81_FD.mp4'
         )
         self.assertEqual(
-            escape_url('http://www.ardmediathek.de/tv/Sturm-der-Liebe/Folge-2036-Zu-Mann-und-Frau-erklärt/Das-Erste/Video?documentId=22673108&bcastId=5290'),
+            normalize_url('http://www.ardmediathek.de/tv/Sturm-der-Liebe/Folge-2036-Zu-Mann-und-Frau-erklärt/Das-Erste/Video?documentId=22673108&bcastId=5290'),
             'http://www.ardmediathek.de/tv/Sturm-der-Liebe/Folge-2036-Zu-Mann-und-Frau-erkl%C3%A4rt/Das-Erste/Video?documentId=22673108&bcastId=5290'
         )
         self.assertEqual(
-            escape_url('http://тест.рф/фрагмент'),
+            normalize_url('http://тест.рф/фрагмент'),
             'http://xn--e1aybc.xn--p1ai/%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82'
         )
         self.assertEqual(
-            escape_url('http://тест.рф/абв?абв=абв#абв'),
+            normalize_url('http://тест.рф/абв?абв=абв#абв'),
             'http://xn--e1aybc.xn--p1ai/%D0%B0%D0%B1%D0%B2?%D0%B0%D0%B1%D0%B2=%D0%B0%D0%B1%D0%B2#%D0%B0%D0%B1%D0%B2'
         )
-        self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
+        self.assertEqual(normalize_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
+
+        self.assertEqual(normalize_url('http://www.example.com/../a/b/../c/./d.html'), 'http://www.example.com/a/c/d.html')
+
+    def test_remove_dot_segments(self):
+        self.assertEqual(remove_dot_segments('/a/b/c/./../../g'), '/a/g')
+        self.assertEqual(remove_dot_segments('mid/content=5/../6'), 'mid/6')
+        self.assertEqual(remove_dot_segments('/ad/../cd'), '/cd')
+        self.assertEqual(remove_dot_segments('/ad/../cd/'), '/cd/')
+        self.assertEqual(remove_dot_segments('/..'), '/')
+        self.assertEqual(remove_dot_segments('/./'), '/')
+        self.assertEqual(remove_dot_segments('/./a'), '/a')
+        self.assertEqual(remove_dot_segments('/abc/./.././d/././e/.././f/./../../ghi'), '/ghi')
+        self.assertEqual(remove_dot_segments('/'), '/')
+        self.assertEqual(remove_dot_segments('/t'), '/t')
+        self.assertEqual(remove_dot_segments('t'), 't')
+        self.assertEqual(remove_dot_segments(''), '')
+        self.assertEqual(remove_dot_segments('/../a/b/c'), '/a/b/c')
+        self.assertEqual(remove_dot_segments('../a'), 'a')
+        self.assertEqual(remove_dot_segments('./a'), 'a')
+        self.assertEqual(remove_dot_segments('.'), '')
+        self.assertEqual(remove_dot_segments('////'), '////')
 
     def test_js_to_json_vars_strings(self):
         self.assertDictEqual(
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index 80428c747..157f5b0c2 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -33,7 +33,6 @@
 from .utils import (
     Popen,
     error_to_str,
-    escape_url,
     expand_path,
     is_path_like,
     sanitize_url,
@@ -42,6 +41,7 @@
     write_string,
 )
 from .utils._utils import _YDLLogger
+from .utils.networking import normalize_url
 
 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
 SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
@@ -1308,7 +1308,7 @@ def prepare_line(line):
 
     def get_cookie_header(self, url):
         """Generate a Cookie HTTP header for a given url"""
-        cookie_req = urllib.request.Request(escape_url(sanitize_url(url)))
+        cookie_req = urllib.request.Request(normalize_url(sanitize_url(url)))
         self.add_cookie_header(cookie_req)
         return cookie_req.get_header('Cookie')
 
@@ -1317,7 +1317,7 @@ def get_cookies_for_url(self, url):
         # Policy `_now` attribute must be set before calling `_cookies_for_request`
         # Ref: https://github.com/python/cpython/blob/3.7/Lib/http/cookiejar.py#L1360
         self._policy._now = self._now = int(time.time())
-        return self._cookies_for_request(urllib.request.Request(escape_url(sanitize_url(url))))
+        return self._cookies_for_request(urllib.request.Request(normalize_url(sanitize_url(url))))
 
     def clear(self, *args, **kwargs):
         with contextlib.suppress(KeyError):
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index 3fe5fa52e..0c4794954 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -41,7 +41,8 @@
 from ..dependencies import brotli
 from ..socks import ProxyError as SocksProxyError
 from ..socks import sockssocket
-from ..utils import escape_url, update_url_query
+from ..utils import update_url_query
+from ..utils.networking import normalize_url
 
 SUPPORTED_ENCODINGS = ['gzip', 'deflate']
 CONTENT_DECODE_ERRORS = [zlib.error, OSError]
@@ -179,7 +180,7 @@ def http_request(self, req):
         # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
         # the code of this workaround has been moved here from YoutubeDL.urlopen()
         url = req.get_full_url()
-        url_escaped = escape_url(url)
+        url_escaped = normalize_url(url)
 
         # Substitute URL if any change after escaping
         if url != url_escaped:
@@ -212,7 +213,7 @@ def http_response(self, req, resp):
             if location:
                 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
                 location = location.encode('iso-8859-1').decode()
-                location_escaped = escape_url(location)
+                location_escaped = normalize_url(location)
                 if location != location_escaped:
                     del resp.headers['Location']
                     resp.headers['Location'] = location_escaped
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
index 3164df49b..792e062fd 100644
--- a/yt_dlp/networking/common.py
+++ b/yt_dlp/networking/common.py
@@ -27,10 +27,9 @@
     classproperty,
     deprecation_warning,
     error_to_str,
-    escape_url,
     update_url_query,
 )
-from ..utils.networking import HTTPHeaderDict
+from ..utils.networking import HTTPHeaderDict, normalize_url
 
 if typing.TYPE_CHECKING:
     RequestData = bytes | Iterable[bytes] | typing.IO | None
@@ -372,7 +371,7 @@ def url(self, url):
             raise TypeError('url must be a string')
         elif url.startswith('//'):
             url = 'http:' + url
-        self._url = escape_url(url)
+        self._url = normalize_url(url)
 
     @property
     def method(self):
diff --git a/yt_dlp/utils/_legacy.py b/yt_dlp/utils/_legacy.py
index 077000971..dde02092c 100644
--- a/yt_dlp/utils/_legacy.py
+++ b/yt_dlp/utils/_legacy.py
@@ -8,6 +8,8 @@
 import zlib
 
 from ._utils import Popen, decode_base_n, preferredencoding
+from .networking import escape_rfc3986  # noqa: F401
+from .networking import normalize_url as escape_url  # noqa: F401
 from .traversal import traverse_obj
 from ..dependencies import certifi, websockets
 from ..networking._helper import make_ssl_context
@@ -197,7 +199,7 @@ def request_to_url(req):
 
 
 def sanitized_Request(url, *args, **kwargs):
-    from ..utils import escape_url, extract_basic_auth, sanitize_url
+    from ..utils import extract_basic_auth, sanitize_url
     url, auth_header = extract_basic_auth(escape_url(sanitize_url(url)))
     if auth_header is not None:
         headers = args[1] if len(args) >= 2 else kwargs.setdefault('headers', {})
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index abae0f17e..f5552ce80 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -2464,23 +2464,6 @@ def lowercase_escape(s):
         s)
 
 
-def escape_rfc3986(s):
-    """Escape non-ASCII characters as suggested by RFC 3986"""
-    return urllib.parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
-
-
-def escape_url(url):
-    """Escape URL as suggested by RFC 3986"""
-    url_parsed = urllib.parse.urlparse(url)
-    return url_parsed._replace(
-        netloc=url_parsed.netloc.encode('idna').decode('ascii'),
-        path=escape_rfc3986(url_parsed.path),
-        params=escape_rfc3986(url_parsed.params),
-        query=escape_rfc3986(url_parsed.query),
-        fragment=escape_rfc3986(url_parsed.fragment)
-    ).geturl()
-
-
 def parse_qs(url, **kwargs):
     return urllib.parse.parse_qs(urllib.parse.urlparse(url).query, **kwargs)
 
diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py
index e6515ec8e..bbcea84d2 100644
--- a/yt_dlp/utils/networking.py
+++ b/yt_dlp/utils/networking.py
@@ -121,3 +121,41 @@ def clean_headers(headers: HTTPHeaderDict):
     if 'Youtubedl-No-Compression' in headers:  # compat
         del headers['Youtubedl-No-Compression']
         headers['Accept-Encoding'] = 'identity'
+
+
+def remove_dot_segments(path):
+    # Implements RFC3986 5.2.4 remote_dot_segments
+    # Pseudo-code: https://tools.ietf.org/html/rfc3986#section-5.2.4
+    # https://github.com/urllib3/urllib3/blob/ba49f5c4e19e6bca6827282feb77a3c9f937e64b/src/urllib3/util/url.py#L263
+    output = []
+    segments = path.split('/')
+    for s in segments:
+        if s == '.':
+            continue
+        elif s == '..':
+            if output:
+                output.pop()
+        else:
+            output.append(s)
+    if not segments[0] and (not output or output[0]):
+        output.insert(0, '')
+    if segments[-1] in ('.', '..'):
+        output.append('')
+    return '/'.join(output)
+
+
+def escape_rfc3986(s):
+    """Escape non-ASCII characters as suggested by RFC 3986"""
+    return urllib.parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
+
+
+def normalize_url(url):
+    """Normalize URL as suggested by RFC 3986"""
+    url_parsed = urllib.parse.urlparse(url)
+    return url_parsed._replace(
+        netloc=url_parsed.netloc.encode('idna').decode('ascii'),
+        path=escape_rfc3986(remove_dot_segments(url_parsed.path)),
+        params=escape_rfc3986(url_parsed.params),
+        query=escape_rfc3986(url_parsed.query),
+        fragment=escape_rfc3986(url_parsed.fragment)
+    ).geturl()

From ba06d77a316650ff057347d224b5afa8b203ad65 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 29 Jul 2023 18:20:42 +1200
Subject: [PATCH 482/871] [ie/youtube] Add `player_params` extractor arg
 (#7719)

Authored by: coletdjnz
---
 README.md                   |  1 +
 yt_dlp/extractor/youtube.py | 11 ++++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index ff88f817c..b82d92a6e 100644
--- a/README.md
+++ b/README.md
@@ -1802,6 +1802,7 @@ #### youtube
 * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
 * `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb` and `tv_embedded` (agegate bypass) with no variants. By default, `ios,android,web` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly, the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
 * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
+* `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp.
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
 * `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all`
     * E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 2b3776aa1..940a4995b 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3583,8 +3583,6 @@ def _is_agegated(player_response):
     def _is_unplayable(player_response):
         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
 
-    _PLAYER_PARAMS = 'CgIQBg=='
-
     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
 
         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
@@ -3597,7 +3595,11 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
             'videoId': video_id,
         }
         if _split_innertube_client(client)[0] == 'android':
-            yt_query['params'] = self._PLAYER_PARAMS
+            yt_query['params'] = 'CgIQBg=='
+
+        pp_arg = self._configuration_arg('player_params', [None])[0]
+        if pp_arg:
+            yt_query['params'] = pp_arg
 
         yt_query.update(self._generate_player_context(sts))
         return self._extract_response(
@@ -4016,6 +4018,9 @@ def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
         webpage = None
         if 'webpage' not in self._configuration_arg('player_skip'):
             query = {'bpctr': '9999999999', 'has_verified': '1'}
+            pp = self._configuration_arg('player_params', [None])[0]
+            if pp:
+                query['pp'] = pp
             webpage = self._download_webpage(
                 webpage_url, video_id, fatal=False, query=query)
 

From 9a04113dfbb69b904e4e2bea736da293505786b8 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Fri, 28 Jul 2023 17:21:45 -0500
Subject: [PATCH 483/871] [ie/Reddit] Fix thumbnail extraction

Authored by: bashonly
---
 yt_dlp/extractor/reddit.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/yt_dlp/extractor/reddit.py b/yt_dlp/extractor/reddit.py
index 13615e82f..813e62874 100644
--- a/yt_dlp/extractor/reddit.py
+++ b/yt_dlp/extractor/reddit.py
@@ -240,6 +240,7 @@ def add_thumbnail(src):
                 'url': unescapeHTML(thumbnail_url),
                 'width': int_or_none(src.get('width')),
                 'height': int_or_none(src.get('height')),
+                'http_headers': {'Accept': '*/*'},
             })
 
         for image in try_get(data, lambda x: x['preview']['images']) or []:

From 86eeb044c2342d68c6ef177577f87852e6badd85 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 29 Jul 2023 10:47:43 -0500
Subject: [PATCH 484/871] [ie/hotstar] Support `/clips/` URLs (#7710)

Closes #7699
Authored by: bashonly
---
 yt_dlp/extractor/hotstar.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/hotstar.py b/yt_dlp/extractor/hotstar.py
index 324e9f51d..cdd937941 100644
--- a/yt_dlp/extractor/hotstar.py
+++ b/yt_dlp/extractor/hotstar.py
@@ -84,7 +84,7 @@ class HotStarIE(HotStarBaseIE):
     _VALID_URL = r'''(?x)
         https?://(?:www\.)?hotstar\.com(?:/in)?/(?!in/)
         (?:
-            (?P<type>movies|sports|episode|(?P<tv>tv|shows))/
+            (?P<type>movies|sports|clips|episode|(?P<tv>tv|shows))/
             (?(tv)(?:[^/?#]+/){2}|[^?#]*)
         )?
         [^/?#]+/
@@ -142,6 +142,18 @@ class HotStarIE(HotStarBaseIE):
             'duration': 1272,
             'channel_id': 3,
         },
+    }, {
+        'url': 'https://www.hotstar.com/in/clips/e3-sairat-kahani-pyaar-ki/1000262286',
+        'info_dict': {
+            'id': '1000262286',
+            'ext': 'mp4',
+            'title': 'E3 - SaiRat, Kahani Pyaar Ki',
+            'description': 'md5:e3b4b3203bc0c5396fe7d0e4948a6385',
+            'episode': 'E3 - SaiRat, Kahani Pyaar Ki',
+            'upload_date': '20210606',
+            'timestamp': 1622943900,
+            'duration': 5395,
+        },
     }, {
         'url': 'https://www.hotstar.com/movies/radha-gopalam/1000057157',
         'only_matching': True,
@@ -160,6 +172,7 @@ class HotStarIE(HotStarBaseIE):
         'episode': 'episode',
         'tv': 'episode',
         'shows': 'episode',
+        'clips': 'content',
         None: 'content',
     }
 

From 127a22460658ac39cbe5c4b3fb88d578363e0dfa Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 29 Jul 2023 11:01:43 -0500
Subject: [PATCH 485/871] [ie/LBRY] Fix original format extraction (#7711)

Authored by: bashonly
---
 yt_dlp/extractor/lbry.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py
index 6af64f0df..7dd3a4861 100644
--- a/yt_dlp/extractor/lbry.py
+++ b/yt_dlp/extractor/lbry.py
@@ -1,5 +1,6 @@
 import functools
 import json
+import re
 import urllib.parse
 
 from .common import InfoExtractor
@@ -83,7 +84,7 @@ class LBRYIE(LBRYBaseIE):
     _TESTS = [{
         # Video
         'url': 'https://lbry.tv/@Mantega:1/First-day-LBRY:1',
-        'md5': 'fffd15d76062e9a985c22c7c7f2f4805',
+        'md5': '65bd7ec1f6744ada55da8e4c48a2edf9',
         'info_dict': {
             'id': '17f983b61f53091fb8ea58a9c56804e4ff8cff4d',
             'ext': 'mp4',
@@ -132,9 +133,8 @@ class LBRYIE(LBRYBaseIE):
             'license': 'None',
         }
     }, {
-        # HLS
         'url': 'https://odysee.com/@gardeningincanada:b/plants-i-will-never-grow-again.-the:e',
-        'md5': '25049011f3c8bc2f8b60ad88a031837e',
+        'md5': 'c35fac796f62a14274b4dc2addb5d0ba',
         'info_dict': {
             'id': 'e51671357333fe22ae88aad320bde2f6f96b1410',
             'ext': 'mp4',
@@ -246,12 +246,13 @@ def _real_extract(self, url):
             streaming_url = self._call_api_proxy(
                 'get', claim_id, {'uri': uri}, 'streaming url')['streaming_url']
 
-            # GET request returns original video/audio file if available
+            # GET request to v3 API returns original video/audio file if available
+            direct_url = re.sub(r'/api/v\d+/', '/api/v3/', streaming_url)
             ext = urlhandle_detect_ext(self._request_webpage(
-                streaming_url, display_id, 'Checking for original quality', headers=headers))
+                direct_url, display_id, 'Checking for original quality', headers=headers))
             if ext != 'm3u8':
                 formats.append({
-                    'url': streaming_url,
+                    'url': direct_url,
                     'format_id': 'original',
                     'quality': 1,
                     **traverse_obj(result, ('value', {

From b09bd0c19648f60c59fb980cd454cb0069959fb9 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 29 Jul 2023 11:14:16 -0500
Subject: [PATCH 486/871] [ie/tiktok] Fix audio-only format extraction (#7712)

Closes #6608
Authored by: bashonly
---
 yt_dlp/extractor/tiktok.py | 43 +++++++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index 48de61f93..f14c4f9d6 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -205,15 +205,16 @@ def parse_url_key(url_key):
 
         known_resolutions = {}
 
-        def mp3_meta(url):
+        def audio_meta(url):
+            ext = determine_ext(url, default_ext='m4a')
             return {
                 'format_note': 'Music track',
-                'ext': 'mp3',
-                'acodec': 'mp3',
+                'ext': ext,
+                'acodec': 'aac' if ext == 'm4a' else ext,
                 'vcodec': 'none',
                 'width': None,
                 'height': None,
-            } if determine_ext(url) == 'mp3' else {}
+            } if ext == 'mp3' or '-music-' in url else {}
 
         def extract_addr(addr, add_meta={}):
             parsed_meta, res = parse_url_key(addr.get('url_key', ''))
@@ -231,7 +232,7 @@ def extract_addr(addr, add_meta={}):
                 **add_meta, **parsed_meta,
                 'format_note': join_nonempty(
                     add_meta.get('format_note'), '(API)' if 'aweme/v1' in url else None, delim=' '),
-                **mp3_meta(url),
+                **audio_meta(url),
             } for url in addr.get('url_list') or []]
 
         # Hack: Add direct video links first to prioritize them when removing duplicate formats
@@ -527,6 +528,7 @@ class TikTokIE(TikTokBaseIE):
             'repost_count': int,
             'comment_count': int,
         },
+        'params': {'skip_download': True},  # XXX: unable to download video data: HTTP Error 403: Forbidden
     }, {
         # Video without title and description
         'url': 'https://www.tiktok.com/@pokemonlife22/video/7059698374567611694',
@@ -600,7 +602,7 @@ class TikTokIE(TikTokBaseIE):
     }, {
         # only available via web
         'url': 'https://www.tiktok.com/@moxypatch/video/7206382937372134662',
-        'md5': '8d8c0be14127020cd9f5def4a2e6b411',
+        'md5': '6aba7fad816e8709ff2c149679ace165',
         'info_dict': {
             'id': '7206382937372134662',
             'ext': 'mp4',
@@ -637,8 +639,8 @@ class TikTokIE(TikTokBaseIE):
             'uploader_id': '86328792343818240',
             'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAA-0bQT0CqebTRr6I4IkYvMDMKSRSJHLNPBo5HrSklJwyA2psXLSZG5FP-LMNpHnJd',
             'channel_id': 'MS4wLjABAAAA-0bQT0CqebTRr6I4IkYvMDMKSRSJHLNPBo5HrSklJwyA2psXLSZG5FP-LMNpHnJd',
-            'creator': 't8',
-            'artist': 't8',
+            'creator': 'tate mcrae',
+            'artist': 'tate mcrae',
             'track': 'original sound',
             'upload_date': '20220609',
             'timestamp': 1654805899,
@@ -650,6 +652,31 @@ class TikTokIE(TikTokBaseIE):
             'thumbnail': r're:^https://.+\.webp',
         },
         'params': {'format': 'bytevc1_1080p_808907-0'},
+    }, {
+        # Slideshow, audio-only m4a format
+        'url': 'https://www.tiktok.com/@hara_yoimiya/video/7253412088251534594',
+        'md5': '2ff8fe0174db2dbf49c597a7bef4e47d',
+        'info_dict': {
+            'id': '7253412088251534594',
+            'ext': 'm4a',
+            'title': 'я ред флаг простите #переписка #щитпост #тревожныйтиппривязанности #рекомендации ',
+            'description': 'я ред флаг простите #переписка #щитпост #тревожныйтиппривязанности #рекомендации ',
+            'uploader': 'hara_yoimiya',
+            'uploader_id': '6582536342634676230',
+            'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAIAlDxriiPWLE-p8p1R_0Bx8qWKfi-7zwmGhzU8Mv25W8sNxjfIKrol31qTczzuLB',
+            'channel_id': 'MS4wLjABAAAAIAlDxriiPWLE-p8p1R_0Bx8qWKfi-7zwmGhzU8Mv25W8sNxjfIKrol31qTczzuLB',
+            'creator': 'лампочка',
+            'artist': 'Øneheart',
+            'album': 'watching the stars',
+            'track': 'watching the stars',
+            'upload_date': '20230708',
+            'timestamp': 1688816612,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
+            'thumbnail': r're:^https://.+\.webp',
+        },
     }, {
         # Auto-captions available
         'url': 'https://www.tiktok.com/@hankgreen1/video/7047596209028074758',

From de20687ee6b742646128a7629b57096631a20619 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 28 Jul 2023 13:48:17 +0530
Subject: [PATCH 487/871] [test] Fix `test_load_certifi`

Closes #7688, #7675
---
 test/test_networking_utils.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/test/test_networking_utils.py b/test/test_networking_utils.py
index ef46f79ed..dbf656090 100644
--- a/test/test_networking_utils.py
+++ b/test/test_networking_utils.py
@@ -95,17 +95,20 @@ def test_make_socks_proxy_unknown(self):
 
     @pytest.mark.skipif(not certifi, reason='certifi is not installed')
     def test_load_certifi(self):
+        context_certifi = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        context_certifi.load_verify_locations(cafile=certifi.where())
         context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        context2 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
         ssl_load_certs(context, use_certifi=True)
-        context2.load_verify_locations(cafile=certifi.where())
-        assert context.get_ca_certs() == context2.get_ca_certs()
+        assert context.get_ca_certs() == context_certifi.get_ca_certs()
 
-        # Test load normal certs
-        # XXX: could there be a case where system certs are the same as certifi?
-        context3 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        ssl_load_certs(context3, use_certifi=False)
-        assert context3.get_ca_certs() != context.get_ca_certs()
+        context_default = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        context_default.load_default_certs()
+        context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        ssl_load_certs(context, use_certifi=False)
+        assert context.get_ca_certs() == context_default.get_ca_certs()
+
+        if context_default.get_ca_certs() == context_certifi.get_ca_certs():
+            pytest.skip('System uses certifi as default. The test is not valid')
 
     @pytest.mark.parametrize('method,status,expected', [
         ('GET', 303, 'GET'),

From 3f7965105d8d2048359e67c1e8b8ebd51588143b Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 30 Jul 2023 03:18:10 +0530
Subject: [PATCH 488/871] [utils] HTTPHeaderDict: Handle byte values

---
 test/test_utils.py         | 2 ++
 yt_dlp/utils/networking.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 453a01a1c..91e3ffd39 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2344,6 +2344,8 @@ def test_traverse_obj(self):
 
     def test_http_header_dict(self):
         headers = HTTPHeaderDict()
+        headers['ytdl-test'] = b'0'
+        self.assertEqual(list(headers.items()), [('Ytdl-Test', '0')])
         headers['ytdl-test'] = 1
         self.assertEqual(list(headers.items()), [('Ytdl-Test', '1')])
         headers['Ytdl-test'] = '2'
diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py
index bbcea84d2..ba0493cc2 100644
--- a/yt_dlp/utils/networking.py
+++ b/yt_dlp/utils/networking.py
@@ -65,6 +65,8 @@ def __init__(self, *args, **kwargs):
         self.update(kwargs)
 
     def __setitem__(self, key, value):
+        if isinstance(value, bytes):
+            value = value.decode('latin-1')
         super().__setitem__(key.title(), str(value))
 
     def __getitem__(self, key):

From 8cb7fc44db010e965d808ee679ef0725cb6e147c Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 30 Jul 2023 03:21:35 +0530
Subject: [PATCH 489/871] Fix `--check-formats`

Bug in bc344cd456380999c1ee74554dfd432a38f32ec7
---
 yt_dlp/YoutubeDL.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index c9cf07e53..6e8be40ba 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2339,13 +2339,13 @@ def _merge(formats_pair):
             return new_dict
 
         def _check_formats(formats):
-            if (self.params.get('check_formats') is not None
+            if self.params.get('check_formats') == 'selected':
+                yield from self._check_formats(formats)
+                return
+            elif (self.params.get('check_formats') is not None
                     or self.params.get('allow_unplayable_formats')):
                 yield from formats
                 return
-            elif self.params.get('check_formats') == 'selected':
-                yield from self._check_formats(formats)
-                return
 
             for f in formats:
                 if f.get('has_drm'):

From 6148833f5ceb7674142ddb8d761ffe03cee7df69 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 30 Jul 2023 03:36:17 +0530
Subject: [PATCH 490/871] [cleanup] Misc

---
 test/test_YoutubeDL.py        |  1 +
 test/test_networking.py       |  8 +++++---
 yt_dlp/YoutubeDL.py           | 14 ++++++--------
 yt_dlp/extractor/common.py    |  2 +-
 yt_dlp/extractor/lbry.py      |  6 +++---
 yt_dlp/extractor/netverse.py  |  4 ++--
 yt_dlp/extractor/ninenow.py   |  2 +-
 yt_dlp/extractor/vk.py        |  4 ++--
 yt_dlp/extractor/wimbledon.py |  2 +-
 yt_dlp/networking/common.py   |  9 +++++----
 10 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index ab1250848..3cfb61fb2 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -831,6 +831,7 @@ def expect_same_infodict(out):
         test('%(id&hi {:>10} {}|)s', 'hi       1234 1234')
         test(R'%(id&{0} {}|)s', 'NA')
         test(R'%(id&{0.1}|)s', 'NA')
+        test('%(height&{:,d})S', '1,080')
 
         # Laziness
         def gen():
diff --git a/test/test_networking.py b/test/test_networking.py
index 684bf5f96..9c33b0d4c 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -29,6 +29,7 @@
 from http.cookiejar import CookieJar
 
 from test.helper import FakeYDL, http_server_port
+from yt_dlp.cookies import YoutubeDLCookieJar
 from yt_dlp.dependencies import brotli
 from yt_dlp.networking import (
     HEADRequest,
@@ -478,7 +479,7 @@ def test_request_cookie_header(self, handler):
             assert 'Cookie: test=test' not in res
 
         # Specified Cookie header should override global cookiejar for that request
-        cookiejar = http.cookiejar.CookieJar()
+        cookiejar = YoutubeDLCookieJar()
         cookiejar.set_cookie(http.cookiejar.Cookie(
             version=0, name='test', value='ytdlp', port=None, port_specified=False,
             domain='127.0.0.1', domain_specified=True, domain_initial_dot=False, path='/',
@@ -505,7 +506,7 @@ def test_incompleteread(self, handler):
 
     @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
     def test_cookies(self, handler):
-        cookiejar = http.cookiejar.CookieJar()
+        cookiejar = YoutubeDLCookieJar()
         cookiejar.set_cookie(http.cookiejar.Cookie(
             0, 'test', 'ytdlp', None, False, '127.0.0.1', True,
             False, '/headers', True, False, None, False, None, None, {}))
@@ -903,7 +904,8 @@ class HTTPSupportedRH(ValidationRH):
     EXTENSION_TESTS = [
         ('Urllib', [
             ({'cookiejar': 'notacookiejar'}, AssertionError),
-            ({'cookiejar': CookieJar()}, False),
+            ({'cookiejar': YoutubeDLCookieJar()}, False),
+            ({'cookiejar': CookieJar()}, AssertionError),
             ({'timeout': 1}, False),
             ({'timeout': 'notatimeout'}, AssertionError),
             ({'unsupported': 'value'}, UnsupportedRequest),
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 6e8be40ba..db5932c44 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -256,8 +256,6 @@ class YoutubeDL:
     overwrites:        Overwrite all video and metadata files if True,
                        overwrite only non-video files if None
                        and don't overwrite any file if False
-                       For compatibility with youtube-dl,
-                       "nooverwrites" may also be used instead
     playlist_items:    Specific indices of playlist to download.
     playlistrandom:    Download playlist items in random order.
     lazy_playlist:     Process playlist entries as they are received.
@@ -553,6 +551,7 @@ class YoutubeDL:
                        You can reduce network I/O by disabling it if you don't
                        care about HLS. (only for youtube)
     no_color:          Same as `color='no_color'`
+    no_overwrites:     Same as `overwrites=False`
     """
 
     _NUMERIC_FIELDS = {
@@ -604,6 +603,7 @@ def __init__(self, params=None, auto_init=True):
         self._playlist_level = 0
         self._playlist_urls = set()
         self.cache = Cache(self)
+        self.__header_cookies = []
 
         stdout = sys.stderr if self.params.get('logtostderr') else sys.stdout
         self._out_files = Namespace(
@@ -632,7 +632,7 @@ def process_color_policy(stream):
             policy = traverse_obj(self.params, ('color', (stream_name, None), {str}), get_all=False)
             if policy in ('auto', None):
                 return term_allow_color and supports_terminal_sequences(stream)
-            assert policy in ('always', 'never', 'no_color')
+            assert policy in ('always', 'never', 'no_color'), policy
             return {'always': True, 'never': False}.get(policy, policy)
 
         self._allow_colors = Namespace(**{
@@ -681,12 +681,10 @@ def process_color_policy(stream):
 
         self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
         self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers'))
-        self.__header_cookies = []
         self._load_cookies(self.params['http_headers'].get('Cookie'))  # compat
         self.params['http_headers'].pop('Cookie', None)
+        self._request_director = self.build_request_director(_REQUEST_HANDLERS.values())
 
-        self._request_director = self.build_request_director(
-            sorted(_REQUEST_HANDLERS.values(), key=lambda rh: rh.RH_NAME.lower()))
         if auto_init and auto_init != 'no_verbose_header':
             self.print_debug_header()
 
@@ -3977,7 +3975,7 @@ def get_encoding(stream):
         })) or 'none'))
 
         write_debug(f'Proxy map: {self.proxies}')
-        # write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers)}')
+        # write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}')
         for plugin_type, plugins in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}.items():
             display_list = ['%s%s' % (
                 klass.__name__, '' if klass.__name__ == name else f' as {name}')
@@ -4080,7 +4078,7 @@ def urlopen(self, req):
 
     def build_request_director(self, handlers):
         logger = _YDLLogger(self)
-        headers = self.params.get('http_headers').copy()
+        headers = self.params['http_headers'].copy()
         proxies = self.proxies.copy()
         clean_headers(headers)
         clean_proxies(proxies, headers)
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index b69ac1d65..7deab995c 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -729,7 +729,7 @@ def extract(self, url):
         except UnsupportedError:
             raise
         except ExtractorError as e:
-            e.video_id = e.video_id or self.get_temp_id(url),
+            e.video_id = e.video_id or self.get_temp_id(url)
             e.ie = e.ie or self.IE_NAME,
             e.traceback = e.traceback or sys.exc_info()[2]
             raise
diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py
index 7dd3a4861..9a9f9256f 100644
--- a/yt_dlp/extractor/lbry.py
+++ b/yt_dlp/extractor/lbry.py
@@ -248,9 +248,9 @@ def _real_extract(self, url):
 
             # GET request to v3 API returns original video/audio file if available
             direct_url = re.sub(r'/api/v\d+/', '/api/v3/', streaming_url)
-            ext = urlhandle_detect_ext(self._request_webpage(
-                direct_url, display_id, 'Checking for original quality', headers=headers))
-            if ext != 'm3u8':
+            urlh = self._request_webpage(
+                direct_url, display_id, 'Checking for original quality', headers=headers, fatal=False)
+            if urlh and urlhandle_detect_ext(urlh) != 'm3u8':
                 formats.append({
                     'url': direct_url,
                     'format_id': 'original',
diff --git a/yt_dlp/extractor/netverse.py b/yt_dlp/extractor/netverse.py
index 398198a1b..ef53e15da 100644
--- a/yt_dlp/extractor/netverse.py
+++ b/yt_dlp/extractor/netverse.py
@@ -160,7 +160,7 @@ class NetverseIE(NetverseBaseIE):
             'uploader': 'Net Prime',
             'comment_count': int,
         },
-        'params':{
+        'params': {
             'getcomments': True
         }
     }, {
@@ -187,7 +187,7 @@ class NetverseIE(NetverseBaseIE):
             'season': 'Season 1',
             'comment_count': int,
         },
-        'params':{
+        'params': {
             'getcomments': True
         }
     }]
diff --git a/yt_dlp/extractor/ninenow.py b/yt_dlp/extractor/ninenow.py
index b970f8ccb..c655b75f4 100644
--- a/yt_dlp/extractor/ninenow.py
+++ b/yt_dlp/extractor/ninenow.py
@@ -53,7 +53,7 @@ class NineNowIE(InfoExtractor):
             'upload_date': '20210421',
         },
         'expected_warnings': ['Ignoring subtitle tracks'],
-        'params':{
+        'params': {
             'skip_download': True,
         }
     }]
diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py
index 6b7379d46..915422817 100644
--- a/yt_dlp/extractor/vk.py
+++ b/yt_dlp/extractor/vk.py
@@ -765,7 +765,7 @@ def _extract_common_meta(self, stream_info):
 
 
 class VKPlayIE(VKPlayBaseIE):
-    _VALID_URL = r'https?://vkplay\.live/(?P<username>[^/]+)/record/(?P<id>[a-f0-9\-]+)'
+    _VALID_URL = r'https?://vkplay\.live/(?P<username>[^/#?]+)/record/(?P<id>[a-f0-9-]+)'
     _TESTS = [{
         'url': 'https://vkplay.live/zitsmann/record/f5e6e3b5-dc52-4d14-965d-0680dd2882da',
         'info_dict': {
@@ -802,7 +802,7 @@ def _real_extract(self, url):
 
 
 class VKPlayLiveIE(VKPlayBaseIE):
-    _VALID_URL = r'https?://vkplay\.live/(?P<id>[^/]+)/?(?:[#?]|$)'
+    _VALID_URL = r'https?://vkplay\.live/(?P<id>[^/#?]+)/?(?:[#?]|$)'
     _TESTS = [{
         'url': 'https://vkplay.live/bayda',
         'info_dict': {
diff --git a/yt_dlp/extractor/wimbledon.py b/yt_dlp/extractor/wimbledon.py
index ee4872e88..0223e54f1 100644
--- a/yt_dlp/extractor/wimbledon.py
+++ b/yt_dlp/extractor/wimbledon.py
@@ -6,7 +6,7 @@
 
 
 class WimbledonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?wimbledon\.com/\w+/video/media/(?P<id>\d+).html'
+    _VALID_URL = r'https?://(?:www\.)?wimbledon\.com/\w+/video/media/(?P<id>\d+)\.html'
     _TESTS = [{
         'url': 'https://www.wimbledon.com/en_GB/video/media/6330247525112.html',
         'info_dict': {
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
index 792e062fd..8fba8c1c5 100644
--- a/yt_dlp/networking/common.py
+++ b/yt_dlp/networking/common.py
@@ -12,7 +12,6 @@
 from collections.abc import Iterable, Mapping
 from email.message import Message
 from http import HTTPStatus
-from http.cookiejar import CookieJar
 
 from ._helper import make_ssl_context, wrap_request_errors
 from .exceptions import (
@@ -22,6 +21,7 @@
     UnsupportedRequest,
 )
 from ..compat.types import NoneType
+from ..cookies import YoutubeDLCookieJar
 from ..utils import (
     bug_reports_message,
     classproperty,
@@ -194,7 +194,7 @@ def __init__(
         self, *,
         logger,  # TODO(Grub4k): default logger
         headers: HTTPHeaderDict = None,
-        cookiejar: CookieJar = None,
+        cookiejar: YoutubeDLCookieJar = None,
         timeout: float | int | None = None,
         proxies: dict = None,
         source_address: str = None,
@@ -208,7 +208,7 @@ def __init__(
 
         self._logger = logger
         self.headers = headers or {}
-        self.cookiejar = cookiejar if cookiejar is not None else CookieJar()
+        self.cookiejar = cookiejar if cookiejar is not None else YoutubeDLCookieJar()
         self.timeout = float(timeout or 20)
         self.proxies = proxies or {}
         self.source_address = source_address
@@ -275,7 +275,7 @@ def _check_proxies(self, proxies):
 
     def _check_extensions(self, extensions):
         """Check extensions for unsupported extensions. Subclasses should extend this."""
-        assert isinstance(extensions.get('cookiejar'), (CookieJar, NoneType))
+        assert isinstance(extensions.get('cookiejar'), (YoutubeDLCookieJar, NoneType))
         assert isinstance(extensions.get('timeout'), (float, int, NoneType))
 
     def _validate(self, request):
@@ -302,6 +302,7 @@ def send(self, request: Request) -> Response:
     @abc.abstractmethod
     def _send(self, request: Request):
         """Handle a request from start to finish. Redefine in subclasses."""
+        pass
 
     def close(self):
         pass

From 546b2c28a106cf8101d481b215b676d1b091d276 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sun, 30 Jul 2023 10:50:25 +1200
Subject: [PATCH 491/871] [ie/youtube] Fix `player_params` arg being converted
 to lowercase

Fix bug in ba06d77a316650ff057347d224b5afa8b203ad65

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 940a4995b..1e16631b1 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3597,7 +3597,7 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
         if _split_innertube_client(client)[0] == 'android':
             yt_query['params'] = 'CgIQBg=='
 
-        pp_arg = self._configuration_arg('player_params', [None])[0]
+        pp_arg = self._configuration_arg('player_params', [None], casesense=True)[0]
         if pp_arg:
             yt_query['params'] = pp_arg
 
@@ -4018,7 +4018,7 @@ def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
         webpage = None
         if 'webpage' not in self._configuration_arg('player_skip'):
             query = {'bpctr': '9999999999', 'has_verified': '1'}
-            pp = self._configuration_arg('player_params', [None])[0]
+            pp = self._configuration_arg('player_params', [None], casesense=True)[0]
             if pp:
                 query['pp'] = pp
             webpage = self._download_webpage(

From f73c11803579889dc8e1c99e25dba9a22fef39d8 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 30 Jul 2023 04:24:38 +0530
Subject: [PATCH 492/871] `FFmpegFixupM3u8PP` may need to run with ffmpeg

Bug in 62b5c94cadaa5f596dc1a7083db9db12efe357be
Closes #7725
---
 yt_dlp/YoutubeDL.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index db5932c44..87bca5bbe 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3450,10 +3450,11 @@ def ffmpeg_fixup(cndn, msg, cls):
                     postprocessed_by_ffmpeg = info_dict.get('requested_formats') or any((
                         isinstance(pp, FFmpegVideoConvertorPP)
                         and resolve_recode_mapping(ext, pp.mapping)[0] not in (ext, None)
-                    ) for pp in self._pps['post_process']) or fd == FFmpegFD
+                    ) for pp in self._pps['post_process'])
 
                     if not postprocessed_by_ffmpeg:
-                        ffmpeg_fixup(ext == 'm4a' and info_dict.get('container') == 'm4a_dash',
+                        ffmpeg_fixup(fd != FFmpegFD and ext == 'm4a'
+                                     and info_dict.get('container') == 'm4a_dash',
                                      'writing DASH m4a. Only some players support this container',
                                      FFmpegFixupM4aPP)
                         ffmpeg_fixup(downloader == 'hlsnative' and not self.params.get('hls_use_mpegts')

From 6014355c6142f68e20c8374e3787e5b5820f19e2 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 29 Jul 2023 18:37:06 -0500
Subject: [PATCH 493/871] [ie/twitter] Add fallback, improve error handling
 (#7621)

Closes #7579, Closes #7625
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 81 +++++++++++++++++++++++++++----------
 1 file changed, 60 insertions(+), 21 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 9d87dbc4b..34b8625c3 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1,3 +1,4 @@
+import functools
 import json
 import re
 
@@ -279,6 +280,12 @@ def input_dict(subtask_id, text):
                     'Submitting confirmation code', headers, data=build_login_json(input_dict(
                         next_subtask, self._get_tfa_info('confirmation code sent to your email or phone'))))
 
+            elif next_subtask == 'ArkoseLogin':
+                self.raise_login_required('Twitter is requiring captcha for this login attempt', method='cookies')
+
+            elif next_subtask == 'DenyLoginSubtask':
+                self.raise_login_required('Twitter rejected this login attempt as suspicious', method='cookies')
+
             elif next_subtask == 'LoginSuccessSubtask':
                 raise ExtractorError('Twitter API did not grant auth token cookie')
 
@@ -304,8 +311,9 @@ def _call_api(self, path, video_id, query={}, graphql=False):
 
         if result.get('errors'):
             errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
-            raise ExtractorError(
-                f'Error(s) while querying API: {errors or "Unknown error"}', expected=True)
+            if errors and 'not authorized' in errors:
+                self.raise_login_required(remove_end(errors, '.'))
+            raise ExtractorError(f'Error(s) while querying API: {errors or "Unknown error"}')
 
         return result
 
@@ -607,7 +615,7 @@ class TwitterIE(TwitterBaseIE):
         # has mp4 formats via mobile API
         'url': 'https://twitter.com/news_al3alm/status/852138619213144067',
         'info_dict': {
-            'id': '852138619213144067',
+            'id': '852077943283097602',
             'ext': 'mp4',
             'title': 'عالم الأخبار - كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة',
             'description': 'كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة   https://t.co/xg6OhpyKfN',
@@ -616,8 +624,16 @@ class TwitterIE(TwitterBaseIE):
             'duration': 277.4,
             'timestamp': 1492000653,
             'upload_date': '20170412',
+            'display_id': '852138619213144067',
+            'age_limit': 0,
+            'uploader_url': 'https://twitter.com/news_al3alm',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'tags': [],
+            'repost_count': int,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
         },
-        'skip': 'Account suspended',
     }, {
         'url': 'https://twitter.com/i/web/status/910031516746514432',
         'info_dict': {
@@ -675,15 +691,15 @@ class TwitterIE(TwitterBaseIE):
             'id': '1087791272830607360',
             'display_id': '1087791357756956680',
             'ext': 'mp4',
-            'title': 'Twitter - A new is coming.  Some of you got an opt-in to try it now. Check out the emoji button, quick keyboard shortcuts, upgraded trends, advanced search, and more. Let us know your thoughts!',
+            'title': 'X - A new is coming.  Some of you got an opt-in to try it now. Check out the emoji button, quick keyboard shortcuts, upgraded trends, advanced search, and more. Let us know your thoughts!',
             'thumbnail': r're:^https?://.*\.jpg',
             'description': 'md5:6dfd341a3310fb97d80d2bf7145df976',
-            'uploader': 'Twitter',
-            'uploader_id': 'Twitter',
+            'uploader': 'X',
+            'uploader_id': 'X',
             'duration': 61.567,
             'timestamp': 1548184644,
             'upload_date': '20190122',
-            'uploader_url': 'https://twitter.com/Twitter',
+            'uploader_url': 'https://twitter.com/X',
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
@@ -991,10 +1007,10 @@ class TwitterIE(TwitterBaseIE):
             'view_count': int,
             'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1600009362759733248/pu/img/XVhFQivj75H_YxxV.jpg?name=orig',
             'age_limit': 0,
-            'uploader': 'Mün The Shinobi',
+            'uploader': 'Mün The Friend Of YWAP',
             'repost_count': int,
             'upload_date': '20221206',
-            'title': 'Mün The Shinobi - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
+            'title': 'Mün The Friend Of YWAP - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
             'comment_count': int,
             'like_count': int,
             'tags': [],
@@ -1024,6 +1040,7 @@ class TwitterIE(TwitterBaseIE):
             'repost_count': int,
         },
         'params': {'extractor_args': {'twitter': {'legacy_api': ['']}}},
+        'skip': 'Protected tweet',
     }, {
         # orig tweet w/ graphql
         'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
@@ -1047,6 +1064,7 @@ class TwitterIE(TwitterBaseIE):
             'repost_count': int,
             'comment_count': int,
         },
+        'skip': 'Protected tweet',
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -1103,6 +1121,8 @@ def _graphql_to_legacy(self, data, twid):
             reason = result.get('reason')
             if reason == 'NsfwLoggedOut':
                 self.raise_login_required('NSFW tweet requires authentication')
+            elif reason == 'Protected':
+                self.raise_login_required('You are not authorized to view this protected tweet')
             raise ExtractorError(reason or 'Requested tweet is unavailable', expected=True)
 
         status = result.get('legacy', {})
@@ -1187,22 +1207,38 @@ def _build_graphql_query(self, media_id):
             }
         }
 
-    def _real_extract(self, url):
-        twid, selected_index = self._match_valid_url(url).group('id', 'index')
-        if not self.is_logged_in and self._configuration_arg('legacy_api'):
-            status = traverse_obj(self._call_api(f'statuses/show/{twid}.json', twid, {
+    def _extract_status(self, twid):
+        if self.is_logged_in:
+            return self._graphql_to_legacy(
+                self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid), twid)
+
+        try:
+            if not self._configuration_arg('legacy_api'):
+                return self._graphql_to_legacy(
+                    self._call_graphql_api('2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId', twid), twid)
+            return traverse_obj(self._call_api(f'statuses/show/{twid}.json', twid, {
                 'cards_platform': 'Web-12',
                 'include_cards': 1,
                 'include_reply_count': 1,
                 'include_user_entities': 0,
                 'tweet_mode': 'extended',
             }), 'retweeted_status', None)
-        elif not self.is_logged_in:
-            status = self._graphql_to_legacy(
-                self._call_graphql_api('2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId', twid), twid)
-        else:
-            status = self._graphql_to_legacy(
-                self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid), twid)
+
+        except ExtractorError as e:
+            if e.expected:
+                raise
+            self.report_warning(
+                f'{e.orig_msg}. Falling back to syndication endpoint; some metadata may be missing', twid)
+
+        status = self._download_json(
+            'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
+            headers={'User-Agent': 'Googlebot'}, query={'id': twid})
+        status['extended_entities'] = {'media': status.get('mediaDetails')}
+        return status
+
+    def _real_extract(self, url):
+        twid, selected_index = self._match_valid_url(url).group('id', 'index')
+        status = self._extract_status(twid)
 
         title = description = traverse_obj(
             status, (('full_text', 'text'), {lambda x: x.replace('\n', ' ')}), get_all=False) or ''
@@ -1230,7 +1266,10 @@ def _real_extract(self, url):
         }
 
         def extract_from_video_info(media):
-            media_id = traverse_obj(media, 'id_str', 'id', expected_type=str_or_none)
+            media_id = traverse_obj(media, 'id_str', 'id', (
+                'video_info', 'variants', ..., 'url',
+                {functools.partial(re.search, r'_video/(\d+)/')}, 1
+            ), get_all=False, expected_type=str_or_none) or twid
             self.write_debug(f'Extracting from video info: {media_id}')
 
             formats = []

From 6d6081dda1290a85bdab6717f239289e3aa74c8e Mon Sep 17 00:00:00 2001
From: Steve <snixon@gmail.com>
Date: Mon, 31 Jul 2023 10:08:37 -0700
Subject: [PATCH 494/871] [extractor/pbs] Add extractor `PBSKidsIE` (#7602)

Authored by: snixon
Fixes #2440
---
 yt_dlp/extractor/_extractors.py |  2 +-
 yt_dlp/extractor/pbs.py         | 59 +++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 9d935a7d1..2ad7e9800 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1420,7 +1420,7 @@
     PatreonIE,
     PatreonCampaignIE
 )
-from .pbs import PBSIE
+from .pbs import PBSIE, PBSKidsIE
 from .pearvideo import PearVideoIE
 from .peekvids import PeekVidsIE, PlayVidsIE
 from .peertube import (
diff --git a/yt_dlp/extractor/pbs.py b/yt_dlp/extractor/pbs.py
index 5bdf561db..2bb2ea9f1 100644
--- a/yt_dlp/extractor/pbs.py
+++ b/yt_dlp/extractor/pbs.py
@@ -11,6 +11,7 @@
     orderedSet,
     strip_jsonp,
     strip_or_none,
+    traverse_obj,
     unified_strdate,
     url_or_none,
     US_RATINGS,
@@ -696,3 +697,61 @@ def extract_redirect_urls(info):
             'subtitles': subtitles,
             'chapters': chapters,
         }
+
+
+class PBSKidsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pbskids\.org/video/[\w-]+/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'https://pbskids.org/video/molly-of-denali/3030407927',
+            'md5': '1ded20a017cc6b53446238f1804ce4c7',
+            'info_dict': {
+                'id': '3030407927',
+                'title': 'Bird in the Hand/Bye-Bye Birdie',
+                'channel': 'molly-of-denali',
+                'duration': 1540,
+                'ext': 'mp4',
+                'series': 'Molly of Denali',
+                'description': 'md5:d006b2211633685d8ebc8d03b6d5611e',
+                'categories': ['Episode'],
+                'upload_date': '20190718',
+            }
+        },
+        {
+            'url': 'https://pbskids.org/video/plum-landing/2365205059',
+            'md5': '92e5d189851a64ae1d0237a965be71f5',
+            'info_dict': {
+                'id': '2365205059',
+                'title': 'Cooper\'s Favorite Place in Nature',
+                'channel': 'plum-landing',
+                'duration': 67,
+                'ext': 'mp4',
+                'series': 'Plum Landing',
+                'description': 'md5:657e5fc4356a84ead1c061eb280ff05d',
+                'categories': ['Episode'],
+                'upload_date': '20140302',
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        meta = self._search_json(r'window\._PBS_KIDS_DEEPLINK\s*=', webpage, 'video info', video_id)
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            traverse_obj(meta, ('video_obj', 'URI', {url_or_none})), video_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(meta, {
+                'categories': ('video_obj', 'video_type', {str}, {lambda x: [x] if x else None}),
+                'channel': ('show_slug', {str}),
+                'description': ('video_obj', 'description', {str}),
+                'duration': ('video_obj', 'duration', {int_or_none}),
+                'series': ('video_obj', 'program_title', {str}),
+                'title': ('video_obj', 'title', {str}),
+                'upload_date': ('video_obj', 'air_date', {unified_strdate}),
+            })
+        }

From 30b29f37159e9226e2f2d5434c9a4096ac4efa2e Mon Sep 17 00:00:00 2001
From: ischmidt20 <ischmidt20@berkeley.edu>
Date: Tue, 1 Aug 2023 03:24:04 -0400
Subject: [PATCH 495/871] [ie/fox] Support foxsports.com (#7724)

Authored by: ischmidt20
---
 yt_dlp/extractor/fox.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/fox.py b/yt_dlp/extractor/fox.py
index 8fb4ada6b..e00e977bd 100644
--- a/yt_dlp/extractor/fox.py
+++ b/yt_dlp/extractor/fox.py
@@ -20,7 +20,7 @@
 
 
 class FOXIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[\da-fA-F]+)'
+    _VALID_URL = r'https?://(?:www\.)?fox(?:sports)?\.com/(?:watch|replay)/(?P<id>[\da-fA-F]+)'
     _TESTS = [{
         # clip
         'url': 'https://www.fox.com/watch/4b765a60490325103ea69888fb2bd4e8/',
@@ -50,6 +50,10 @@ class FOXIE(InfoExtractor):
         # sports event, geo-restricted
         'url': 'https://www.fox.com/watch/b057484dade738d1f373b3e46216fa2c/',
         'only_matching': True,
+    }, {
+        # fox sports replay, geo-restricted
+        'url': 'https://www.foxsports.com/replay/561f3e071347a24e5e877abc56b22e89',
+        'only_matching': True,
     }]
     _GEO_BYPASS = False
     _HOME_PAGE_URL = 'https://www.fox.com/'

From a854fbec56d5004f5147116a41d1dd050632a579 Mon Sep 17 00:00:00 2001
From: ringus1 <ringus1@users.noreply.github.com>
Date: Tue, 1 Aug 2023 16:13:54 +0200
Subject: [PATCH 496/871] [ie/facebook] Add dash manifest URL (#7743)

Fixes #7742
Authored by: ringus1
---
 yt_dlp/extractor/facebook.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 574f8e8c9..4fd17b574 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -481,7 +481,8 @@ def extract_dash_manifest(video, formats):
             dash_manifest = video.get('dash_manifest')
             if dash_manifest:
                 formats.extend(self._parse_mpd_formats(
-                    compat_etree_fromstring(urllib.parse.unquote_plus(dash_manifest))))
+                    compat_etree_fromstring(urllib.parse.unquote_plus(dash_manifest)),
+                    mpd_url=video.get('dash_manifest_url')))
 
         def process_formats(info):
             # Downloads with browser's User-Agent are rate limited. Working around

From b9de629d78ce31699f2de886071dc257830f9676 Mon Sep 17 00:00:00 2001
From: ifan-t <jacifan2000@gmail.com>
Date: Tue, 1 Aug 2023 19:01:59 +0100
Subject: [PATCH 497/871] [ie/S4C] Add extractor (#7730)

Authored by: ifan-t
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/s4c.py         | 62 +++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 yt_dlp/extractor/s4c.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 2ad7e9800..63bb55ea7 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1709,6 +1709,7 @@
     RuvIE,
     RuvSpilaIE
 )
+from .s4c import S4CIE
 from .safari import (
     SafariIE,
     SafariApiIE,
diff --git a/yt_dlp/extractor/s4c.py b/yt_dlp/extractor/s4c.py
new file mode 100644
index 000000000..38a905896
--- /dev/null
+++ b/yt_dlp/extractor/s4c.py
@@ -0,0 +1,62 @@
+from .common import InfoExtractor
+from ..utils import traverse_obj
+
+
+class S4CIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?s4c\.cymru/clic/programme/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.s4c.cymru/clic/programme/861362209',
+        'info_dict': {
+            'id': '861362209',
+            'ext': 'mp4',
+            'title': 'Y Swn',
+            'description': 'md5:f7681a30e4955b250b3224aa9fe70cf0',
+            'duration': 5340
+        },
+    }, {
+        'url': 'https://www.s4c.cymru/clic/programme/856636948',
+        'info_dict': {
+            'id': '856636948',
+            'ext': 'mp4',
+            'title': 'Am Dro',
+            'duration': 2880,
+            'description': 'md5:100d8686fc9a632a0cb2db52a3433ffe',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        details = self._download_json(
+            f'https://www.s4c.cymru/df/full_prog_details?lang=e&programme_id={video_id}',
+            video_id, fatal=False)
+
+        filename = self._download_json(
+            'https://player-api.s4c-cdn.co.uk/player-configuration/prod', video_id, query={
+                'programme_id': video_id,
+                'signed': '0',
+                'lang': 'en',
+                'mode': 'od',
+                'appId': 'clic',
+                'streamName': '',
+            }, note='Downloading player config JSON')['filename']
+        m3u8_url = self._download_json(
+            'https://player-api.s4c-cdn.co.uk/streaming-urls/prod', video_id, query={
+                'mode': 'od',
+                'application': 'clic',
+                'region': 'WW',
+                'extra': 'false',
+                'thirdParty': 'false',
+                'filename': filename,
+            }, note='Downloading streaming urls JSON')['hls']
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(details, ('full_prog_details', 0, {
+                'title': (('programme_title', 'series_title'), {str}),
+                'description': ('full_billing', {str.strip}),
+                'duration': ('duration', {lambda x: int(x) * 60}),
+            }), get_all=False),
+        }

From db9743894071760f994f640a4c24358f749a78c0 Mon Sep 17 00:00:00 2001
From: Franklin Lee <Frankgoji@users.noreply.github.com>
Date: Tue, 1 Aug 2023 11:21:16 -0700
Subject: [PATCH 498/871] [ie/PicartoVod] Fix extractor (#7727)

Closes #2926
Authored by: Frankgoji
---
 yt_dlp/extractor/picarto.py | 54 ++++++++++++++++++++++++++++---------
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/picarto.py b/yt_dlp/extractor/picarto.py
index 36a062def..d415ba28e 100644
--- a/yt_dlp/extractor/picarto.py
+++ b/yt_dlp/extractor/picarto.py
@@ -1,7 +1,10 @@
+import urllib.parse
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    js_to_json,
+    str_or_none,
+    traverse_obj,
 )
 
 
@@ -84,7 +87,7 @@ def _real_extract(self, url):
 
 
 class PicartoVodIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www.)?picarto\.tv/videopopout/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?picarto\.tv/(?:videopopout|\w+/videos)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://picarto.tv/videopopout/ArtofZod_2017.12.12.00.13.23.flv',
         'md5': '3ab45ba4352c52ee841a28fb73f2d9ca',
@@ -94,6 +97,18 @@ class PicartoVodIE(InfoExtractor):
             'title': 'ArtofZod_2017.12.12.00.13.23.flv',
             'thumbnail': r're:^https?://.*\.jpg'
         },
+        'skip': 'The VOD does not exist',
+    }, {
+        'url': 'https://picarto.tv/ArtofZod/videos/772650',
+        'md5': '00067a0889f1f6869cc512e3e79c521b',
+        'info_dict': {
+            'id': '772650',
+            'ext': 'mp4',
+            'title': 'Art of Zod - Drawing and Painting',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'channel': 'ArtofZod',
+            'age_limit': 18,
+        }
     }, {
         'url': 'https://picarto.tv/videopopout/Plague',
         'only_matching': True,
@@ -102,21 +117,36 @@ class PicartoVodIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        data = self._download_json(
+            'https://ptvintern.picarto.tv/ptvapi', video_id, query={
+                'query': f'''{{
+  video(id: "{video_id}") {{
+    id
+    title
+    adult
+    file_name
+    video_recording_image_url
+    channel {{
+      name
+    }}
+  }}
+}}'''
+            })['data']['video']
 
-        vod_info = self._parse_json(
-            self._search_regex(
-                r'(?s)#vod-player["\']\s*,\s*(\{.+?\})\s*\)', webpage,
-                'vod player'),
-            video_id, transform_source=js_to_json)
+        file_name = data['file_name']
+        netloc = urllib.parse.urlparse(data['video_recording_image_url']).netloc
 
         formats = self._extract_m3u8_formats(
-            vod_info['vod'], video_id, 'mp4', entry_protocol='m3u8_native',
-            m3u8_id='hls')
+            f'https://{netloc}/stream/hls/{file_name}/index.m3u8', video_id, 'mp4', m3u8_id='hls')
 
         return {
             'id': video_id,
-            'title': video_id,
-            'thumbnail': vod_info.get('vodThumb'),
+            **traverse_obj(data, {
+                'id': ('id', {str_or_none}),
+                'title': ('title', {str}),
+                'thumbnail': 'video_recording_image_url',
+                'channel': ('channel', 'name', {str}),
+                'age_limit': ('adult', {lambda x: 18 if x else 0}),
+            }),
             'formats': formats,
         }

From db7b054a6111ca387220d0eb87bf342f9c130eb8 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 5 Aug 2023 10:17:48 +1200
Subject: [PATCH 499/871] [networking] Add request handler preference framework
 (#7603)

Preference functions that take a request and a request handler instance can be registered to prioritize different request handlers per request.

Authored by: coletdjnz
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
---
 test/test_networking.py     | 29 +++++++++++++++++++++++----
 yt_dlp/YoutubeDL.py         |  7 ++++---
 yt_dlp/networking/common.py | 40 +++++++++++++++++++++++++++++++++----
 3 files changed, 65 insertions(+), 11 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index 9c33b0d4c..2622d24da 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -1035,17 +1035,17 @@ def test_send(self):
         assert isinstance(director.send(Request('http://')), FakeResponse)
 
     def test_unsupported_handlers(self):
-        director = RequestDirector(logger=FakeLogger())
-        director.add_handler(FakeRH(logger=FakeLogger()))
-
         class SupportedRH(RequestHandler):
             _SUPPORTED_URL_SCHEMES = ['http']
 
             def _send(self, request: Request):
                 return Response(fp=io.BytesIO(b'supported'), headers={}, url=request.url)
 
-        # This handler should by default take preference over FakeRH
+        director = RequestDirector(logger=FakeLogger())
         director.add_handler(SupportedRH(logger=FakeLogger()))
+        director.add_handler(FakeRH(logger=FakeLogger()))
+
+        # First should take preference
         assert director.send(Request('http://')).read() == b'supported'
         assert director.send(Request('any://')).read() == b''
 
@@ -1072,6 +1072,27 @@ def _send(self, request: Request):
         director.add_handler(UnexpectedRH(logger=FakeLogger))
         assert director.send(Request('any://'))
 
+    def test_preference(self):
+        director = RequestDirector(logger=FakeLogger())
+        director.add_handler(FakeRH(logger=FakeLogger()))
+
+        class SomeRH(RequestHandler):
+            _SUPPORTED_URL_SCHEMES = ['http']
+
+            def _send(self, request: Request):
+                return Response(fp=io.BytesIO(b'supported'), headers={}, url=request.url)
+
+        def some_preference(rh, request):
+            return (0 if not isinstance(rh, SomeRH)
+                    else 100 if 'prefer' in request.headers
+                    else -1)
+
+        director.add_handler(SomeRH(logger=FakeLogger()))
+        director.preferences.add(some_preference)
+
+        assert director.send(Request('http://')).read() == b''
+        assert director.send(Request('http://', headers={'prefer': '1'})).read() == b'supported'
+
 
 # XXX: do we want to move this to test_YoutubeDL.py?
 class TestYoutubeDLNetworking:
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 87bca5bbe..666d89b46 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -34,7 +34,7 @@
 from .extractor.openload import PhantomJSwrapper
 from .minicurses import format_text
 from .networking import HEADRequest, Request, RequestDirector
-from .networking.common import _REQUEST_HANDLERS
+from .networking.common import _REQUEST_HANDLERS, _RH_PREFERENCES
 from .networking.exceptions import (
     HTTPError,
     NoSupportingHandlers,
@@ -683,7 +683,7 @@ def process_color_policy(stream):
         self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers'))
         self._load_cookies(self.params['http_headers'].get('Cookie'))  # compat
         self.params['http_headers'].pop('Cookie', None)
-        self._request_director = self.build_request_director(_REQUEST_HANDLERS.values())
+        self._request_director = self.build_request_director(_REQUEST_HANDLERS.values(), _RH_PREFERENCES)
 
         if auto_init and auto_init != 'no_verbose_header':
             self.print_debug_header()
@@ -4077,7 +4077,7 @@ def urlopen(self, req):
         except HTTPError as e:  # TODO: Remove in a future release
             raise _CompatHTTPError(e) from e
 
-    def build_request_director(self, handlers):
+    def build_request_director(self, handlers, preferences=None):
         logger = _YDLLogger(self)
         headers = self.params['http_headers'].copy()
         proxies = self.proxies.copy()
@@ -4106,6 +4106,7 @@ def build_request_director(self, handlers):
                     },
                 }),
             ))
+        director.preferences.update(preferences or [])
         return director
 
     def encode(self, s):
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
index 8fba8c1c5..584c7bb4d 100644
--- a/yt_dlp/networking/common.py
+++ b/yt_dlp/networking/common.py
@@ -31,8 +31,19 @@
 )
 from ..utils.networking import HTTPHeaderDict, normalize_url
 
-if typing.TYPE_CHECKING:
-    RequestData = bytes | Iterable[bytes] | typing.IO | None
+
+def register_preference(*handlers: type[RequestHandler]):
+    assert all(issubclass(handler, RequestHandler) for handler in handlers)
+
+    def outer(preference: Preference):
+        @functools.wraps(preference)
+        def inner(handler, *args, **kwargs):
+            if not handlers or isinstance(handler, handlers):
+                return preference(handler, *args, **kwargs)
+            return 0
+        _RH_PREFERENCES.add(inner)
+        return inner
+    return outer
 
 
 class RequestDirector:
@@ -40,12 +51,17 @@ class RequestDirector:
 
     Helper class that, when given a request, forward it to a RequestHandler that supports it.
 
+    Preference functions in the form of func(handler, request) -> int
+    can be registered into the `preferences` set. These are used to sort handlers
+    in order of preference.
+
     @param logger: Logger instance.
     @param verbose: Print debug request information to stdout.
     """
 
     def __init__(self, logger, verbose=False):
         self.handlers: dict[str, RequestHandler] = {}
+        self.preferences: set[Preference] = set()
         self.logger = logger  # TODO(Grub4k): default logger
         self.verbose = verbose
 
@@ -58,6 +74,16 @@ def add_handler(self, handler: RequestHandler):
         assert isinstance(handler, RequestHandler), 'handler must be a RequestHandler'
         self.handlers[handler.RH_KEY] = handler
 
+    def _get_handlers(self, request: Request) -> list[RequestHandler]:
+        """Sorts handlers by preference, given a request"""
+        preferences = {
+            rh: sum(pref(rh, request) for pref in self.preferences)
+            for rh in self.handlers.values()
+        }
+        self._print_verbose('Handler preferences for this request: %s' % ', '.join(
+            f'{rh.RH_NAME}={pref}' for rh, pref in preferences.items()))
+        return sorted(self.handlers.values(), key=preferences.get, reverse=True)
+
     def _print_verbose(self, msg):
         if self.verbose:
             self.logger.stdout(f'director: {msg}')
@@ -73,8 +99,7 @@ def send(self, request: Request) -> Response:
 
         unexpected_errors = []
         unsupported_errors = []
-        # TODO (future): add a per-request preference system
-        for handler in reversed(list(self.handlers.values())):
+        for handler in self._get_handlers(request):
             self._print_verbose(f'Checking if "{handler.RH_NAME}" supports this request.')
             try:
                 handler.validate(request)
@@ -530,3 +555,10 @@ def info(self):
     def getheader(self, name, default=None):
         deprecation_warning('Response.getheader() is deprecated, use Response.get_header', stacklevel=2)
         return self.get_header(name, default)
+
+
+if typing.TYPE_CHECKING:
+    RequestData = bytes | Iterable[bytes] | typing.IO | None
+    Preference = typing.Callable[[RequestHandler, Request], int]
+
+_RH_PREFERENCES: set[Preference] = set()

From 378ae9f9fb8e8c86e6ac89c4c5b815b48ce93620 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 12 Aug 2023 16:26:08 +1200
Subject: [PATCH 500/871] [ie/youtube] Fix consent cookie (#7774)

Fixes #7594

Authored by: coletdjnz
---
 yt_dlp/extractor/youtube.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 1e16631b1..023d8fd8c 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -496,16 +496,10 @@ def _initialize_consent(self):
         cookies = self._get_cookies('https://www.youtube.com/')
         if cookies.get('__Secure-3PSID'):
             return
-        consent_id = None
-        consent = cookies.get('CONSENT')
-        if consent:
-            if 'YES' in consent.value:
-                return
-            consent_id = self._search_regex(
-                r'PENDING\+(\d+)', consent.value, 'consent', default=None)
-        if not consent_id:
-            consent_id = random.randint(100, 999)
-        self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
+        socs = cookies.get('SOCS')
+        if socs and not socs.value.startswith('CAA'):  # not consented
+            return
+        self._set_cookie('.youtube.com', 'SOCS', 'CAI', secure=True)  # accept all (required for mixes)
 
     def _initialize_pref(self):
         cookies = self._get_cookies('https://www.youtube.com/')

From dab87ca23650fd87184ff5286b53e6985b59f71d Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 12 Aug 2023 16:30:23 -0500
Subject: [PATCH 501/871] [cookies] Containers JSON should be opened as utf-8
 (#7800)

Closes #7797
Authored by: bashonly
---
 yt_dlp/cookies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index 157f5b0c2..a71fbc28b 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -138,7 +138,7 @@ def _extract_firefox_cookies(profile, container, logger):
         containers_path = os.path.join(os.path.dirname(cookie_database_path), 'containers.json')
         if not os.path.isfile(containers_path) or not os.access(containers_path, os.R_OK):
             raise FileNotFoundError(f'could not read containers.json in {search_root}')
-        with open(containers_path) as containers:
+        with open(containers_path, encoding='utf8') as containers:
             identities = json.load(containers).get('identities', [])
         container_id = next((context.get('userContextId') for context in identities if container in (
             context.get('name'),

From 339c339fec095ff4141b20e6aa83629117fb26df Mon Sep 17 00:00:00 2001
From: trainman261 <trainman261@users.noreply.github.com>
Date: Sun, 13 Aug 2023 01:58:55 +0200
Subject: [PATCH 502/871] [ie/CBCPlayer] Extract HLS formats and subtitles
 (#7484)

Authored by: trainman261
---
 yt_dlp/extractor/cbc.py             | 28 +++++++++++++++++++++++++++-
 yt_dlp/extractor/scrippsnetworks.py |  1 +
 yt_dlp/extractor/theplatform.py     | 22 ++++++++++++++++++++--
 3 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index 41e092422..9413281a5 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -161,7 +161,7 @@ class CBCPlayerIE(InfoExtractor):
             'upload_date': '20160210',
             'uploader': 'CBCC-NEW',
         },
-        'skip': 'Geo-restricted to Canada',
+        'skip': 'Geo-restricted to Canada and no longer available',
     }, {
         # Redirected from http://www.cbc.ca/player/AudioMobile/All%20in%20a%20Weekend%20Montreal/ID/2657632011/
         'url': 'http://www.cbc.ca/player/play/2657631896',
@@ -174,6 +174,9 @@ class CBCPlayerIE(InfoExtractor):
             'timestamp': 1425704400,
             'upload_date': '20150307',
             'uploader': 'CBCC-NEW',
+            'thumbnail': 'http://thumbnails.cbc.ca/maven_legacy/thumbnails/sonali-karnick-220.jpg',
+            'chapters': [],
+            'duration': 494.811,
         },
     }, {
         'url': 'http://www.cbc.ca/player/play/2164402062',
@@ -186,6 +189,28 @@ class CBCPlayerIE(InfoExtractor):
             'timestamp': 1320410746,
             'upload_date': '20111104',
             'uploader': 'CBCC-NEW',
+            'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/277/67/cancer_852x480_2164412612.jpg',
+            'chapters': [],
+            'duration': 186.867,
+        },
+    }, {
+        # Has subtitles
+        # These broadcasts expire after ~1 month, can find new test URL here:
+        # https://www.cbc.ca/player/news/TV%20Shows/The%20National/Latest%20Broadcast
+        'url': 'http://www.cbc.ca/player/play/2249992771553',
+        'md5': '2f2fb675dd4f0f8a5bb7588d1b13bacd',
+        'info_dict': {
+            'id': '2249992771553',
+            'ext': 'mp4',
+            'title': 'The National | Women’s soccer pay, Florida seawater, Swift quake',
+            'description': 'md5:adba28011a56cfa47a080ff198dad27a',
+            'timestamp': 1690596000,
+            'duration': 2716.333,
+            'subtitles': {'eng': [{'ext': 'vtt', 'protocol': 'm3u8_native'}]},
+            'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/481/326/thumbnail.jpeg',
+            'uploader': 'CBCC-NEW',
+            'chapters': 'count:5',
+            'upload_date': '20230729',
         },
     }]
 
@@ -199,6 +224,7 @@ def _real_extract(self, url):
                     'force_smil_url': True
                 }),
             'id': video_id,
+            '_format_sort_fields': ('res', 'proto')  # Prioritize direct http formats over HLS
         }
 
 
diff --git a/yt_dlp/extractor/scrippsnetworks.py b/yt_dlp/extractor/scrippsnetworks.py
index c3cee6e4a..adfd7e5f2 100644
--- a/yt_dlp/extractor/scrippsnetworks.py
+++ b/yt_dlp/extractor/scrippsnetworks.py
@@ -115,6 +115,7 @@ class ScrippsNetworksIE(InfoExtractor):
             'uploader': 'SCNI-SCND',
         },
         'add_ie': ['ThePlatform'],
+        'expected_warnings': ['No HLS formats found'],
     }, {
         'url': 'https://www.diynetwork.com/videos/diy-barnwood-tablet-stand-0265790',
         'only_matching': True,
diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py
index 8307b912d..99caeb5f9 100644
--- a/yt_dlp/extractor/theplatform.py
+++ b/yt_dlp/extractor/theplatform.py
@@ -19,7 +19,11 @@
     xpath_with_ns,
     mimetype2ext,
     find_xpath_attr,
+    traverse_obj,
+    update_url,
+    urlhandle_detect_ext,
 )
+from ..networking import HEADRequest
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
 _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
@@ -162,7 +166,8 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
         'params': {
             # rtmp download
             'skip_download': True,
-        }
+        },
+        'skip': '404 Not Found',
     }, {
         'url': 'https://player.theplatform.com/p/D6x-PC/pulse_preview/embed/select/media/yMBg9E8KFxZD',
         'info_dict': {
@@ -171,7 +176,8 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
             'description': 'md5:644ad9188d655b742f942bf2e06b002d',
             'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
             'uploader': 'EGSM',
-        }
+        },
+        'skip': '404 Not Found',
     }, {
         'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
         'only_matching': True,
@@ -189,6 +195,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
             'upload_date': '20150701',
             'uploader': 'NBCU-NEWS',
         },
+        'skip': '404 Not Found',
     }, {
         # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
         # geo-restricted (US), HLS encrypted with AES-128
@@ -295,6 +302,17 @@ def _real_extract(self, url):
 
         formats, subtitles = self._extract_theplatform_smil(smil_url, video_id)
 
+        # With some sites, manifest URL must be forced to extract HLS formats
+        if not traverse_obj(formats, lambda _, v: v['format_id'].startswith('hls')):
+            m3u8_url = update_url(url, query='mbr=true&manifest=m3u', fragment=None)
+            urlh = self._request_webpage(
+                HEADRequest(m3u8_url), video_id, 'Checking for HLS formats', 'No HLS formats found', fatal=False)
+            if urlh and urlhandle_detect_ext(urlh) == 'm3u8':
+                m3u8_fmts, m3u8_subs = self._extract_m3u8_formats_and_subtitles(
+                    m3u8_url, video_id, m3u8_id='hls', fatal=False)
+                formats.extend(m3u8_fmts)
+                self._merge_subtitles(m3u8_subs, target=subtitles)
+
         ret = self._extract_theplatform_metadata(path, video_id)
         combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles)
         ret.update({

From 876b70c8edf4c0147f180bd981fbc4d625cbfb9c Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Mon, 14 Aug 2023 19:29:04 +0100
Subject: [PATCH 503/871] [ie/tbsjp] Add episode, program, playlist extractors
 (#7765)

Authored by: garret1317
---
 yt_dlp/extractor/_extractors.py |   5 ++
 yt_dlp/extractor/tbsjp.py       | 152 ++++++++++++++++++++++++++++++++
 2 files changed, 157 insertions(+)
 create mode 100644 yt_dlp/extractor/tbsjp.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 63bb55ea7..d4d3b6074 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1901,6 +1901,11 @@
 from .tagesschau import TagesschauIE
 from .tass import TassIE
 from .tbs import TBSIE
+from .tbsjp import (
+    TBSJPEpisodeIE,
+    TBSJPProgramIE,
+    TBSJPPlaylistIE,
+)
 from .tdslifeway import TDSLifewayIE
 from .teachable import (
     TeachableIE,
diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py
new file mode 100644
index 000000000..77ddeca32
--- /dev/null
+++ b/yt_dlp/extractor/tbsjp.py
@@ -0,0 +1,152 @@
+from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    get_element_text_and_html_by_tag,
+    int_or_none,
+    str_or_none,
+    traverse_obj,
+    try_call,
+    unified_timestamp,
+    urljoin,
+)
+
+
+class TBSJPEpisodeIE(InfoExtractor):
+    _VALID_URL = r'https?://cu\.tbs\.co\.jp/episode/(?P<id>[\d_]+)'
+    _GEO_BYPASS = False
+    _TESTS = [{
+        'url': 'https://cu.tbs.co.jp/episode/23613_2044134_1000049010',
+        'skip': 'streams geo-restricted, Japan only. Also, will likely expire eventually',
+        'info_dict': {
+            'title': 'VIVANT 第三話 誤送金完結へ!絶体絶命の反撃開始',
+            'id': '23613_2044134_1000049010',
+            'ext': 'mp4',
+            'upload_date': '20230728',
+            'duration': 3517,
+            'release_timestamp': 1691118230,
+            'episode': '第三話 誤送金完結へ!絶体絶命の反撃開始',
+            'release_date': '20230804',
+            'categories': 'count:11',
+            'episode_number': 3,
+            'timestamp': 1690522538,
+            'description': 'md5:2b796341af1ef772034133174ba4a895',
+            'series': 'VIVANT',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        meta = self._search_json(r'window\.app\s*=', webpage, 'episode info', video_id, fatal=False)
+        episode = traverse_obj(meta, ('falcorCache', 'catalog', 'episode', video_id, 'value'))
+
+        tf_path = self._search_regex(
+            r'<script[^>]+src=["\'](/assets/tf\.[^"\']+\.js)["\']', webpage, 'stream API config')
+        tf_js = self._download_webpage(urljoin(url, tf_path), video_id, note='Downloading stream API config')
+        video_url = self._search_regex(r'videoPlaybackUrl:\s*[\'"]([^\'"]+)[\'"]', tf_js, 'stream API url')
+        api_key = self._search_regex(r'api_key:\s*[\'"]([^\'"]+)[\'"]', tf_js, 'stream API key')
+
+        try:
+            source_meta = self._download_json(f'{video_url}ref:{video_id}', video_id,
+                                              headers={'X-Streaks-Api-Key': api_key},
+                                              note='Downloading stream metadata')
+        except ExtractorError as e:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                self.raise_geo_restricted(countries=['JP'])
+            raise
+
+        formats, subtitles = [], {}
+        for src in traverse_obj(source_meta, ('sources', ..., 'src')):
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(src, video_id, fatal=False)
+            formats.extend(fmts)
+            self._merge_subtitles(subs, target=subtitles)
+
+        return {
+            'title': try_call(lambda: clean_html(get_element_text_and_html_by_tag('h3', webpage)[0])),
+            'id': video_id,
+            **traverse_obj(episode, {
+                'categories': ('keywords', {list}),
+                'id': ('content_id', {str}),
+                'description': ('description', 0, 'value'),
+                'timestamp': ('created_at', {unified_timestamp}),
+                'release_timestamp': ('pub_date', {unified_timestamp}),
+                'duration': ('tv_episode_info', 'duration', {int_or_none}),
+                'episode_number': ('tv_episode_info', 'episode_number', {int_or_none}),
+                'episode': ('title', lambda _, v: not v.get('is_phonetic'), 'value'),
+                'series': ('custom_data', 'program_name'),
+            }, get_all=False),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+
+class TBSJPProgramIE(InfoExtractor):
+    _VALID_URL = r'https?://cu\.tbs\.co\.jp/program/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://cu.tbs.co.jp/program/23601',
+        'playlist_mincount': 4,
+        'info_dict': {
+            'id': '23601',
+            'categories': ['エンタメ', 'ミライカプセル', '会社', '働く', 'バラエティ', '動画'],
+            'description': '幼少期の夢は大人になって、どう成長したのだろうか？\nそしてその夢は今後、どのように広がっていくのか？\nいま話題の会社で働く人の「夢の成長」を描く',
+            'series': 'ミライカプセル　-I have a dream-',
+            'title': 'ミライカプセル　-I have a dream-'
+        }
+    }]
+
+    def _real_extract(self, url):
+        programme_id = self._match_id(url)
+        webpage = self._download_webpage(url, programme_id)
+        meta = self._search_json(r'window\.app\s*=', webpage, 'programme info', programme_id)
+
+        programme = traverse_obj(meta, ('falcorCache', 'catalog', 'program', programme_id, 'false', 'value'))
+
+        return {
+            '_type': 'playlist',
+            'entries': [self.url_result(f'https://cu.tbs.co.jp/episode/{video_id}', TBSJPEpisodeIE, video_id)
+                        for video_id in traverse_obj(programme, ('custom_data', 'seriesList', 'episodeCode', ...))],
+            'id': programme_id,
+            **traverse_obj(programme, {
+                'categories': ('keywords', ...),
+                'id': ('tv_episode_info', 'show_content_id', {str_or_none}),
+                'description': ('custom_data', 'program_description'),
+                'series': ('custom_data', 'program_name'),
+                'title': ('custom_data', 'program_name'),
+            }),
+        }
+
+
+class TBSJPPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://cu\.tbs\.co\.jp/playlist/(?P<id>[\da-f]+)'
+    _TESTS = [{
+        'url': 'https://cu.tbs.co.jp/playlist/184f9970e7ba48e4915f1b252c55015e',
+        'playlist_mincount': 4,
+        'info_dict': {
+            'title': 'まもなく配信終了',
+            'id': '184f9970e7ba48e4915f1b252c55015e',
+        }
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        page = self._download_webpage(url, playlist_id)
+        meta = self._search_json(r'window\.app\s*=', page, 'playlist info', playlist_id)
+        playlist = traverse_obj(meta, ('falcorCache', 'playList', playlist_id))
+
+        def entries():
+            for entry in traverse_obj(playlist, ('catalogs', 'value', lambda _, v: v['content_id'])):
+                # TODO: it's likely possible to get all metadata from the playlist page json instead
+                content_id = entry['content_id']
+                content_type = entry.get('content_type')
+                if content_type == 'tv_show':
+                    yield self.url_result(
+                        f'https://cu.tbs.co.jp/program/{content_id}', TBSJPProgramIE, content_id)
+                elif content_type == 'tv_episode':
+                    yield self.url_result(
+                        f'https://cu.tbs.co.jp/episode/{content_id}', TBSJPEpisodeIE, content_id)
+                else:
+                    self.report_warning(f'Skipping "{content_id}" with unsupported content_type "{content_type}"')
+
+        return self.playlist_result(entries(), playlist_id, traverse_obj(playlist, ('display_name', 'value')))

From a0de8bb8601146b8f87bf7cd562eef8bfb4690be Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 20 Aug 2023 11:10:15 -0500
Subject: [PATCH 504/871] [ie/zee5] Update access token endpoint (#7914)

Closes #7911
Authored by: bashonly
---
 yt_dlp/extractor/zee5.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/zee5.py b/yt_dlp/extractor/zee5.py
index b4734cc8f..ca79cf0a7 100644
--- a/yt_dlp/extractor/zee5.py
+++ b/yt_dlp/extractor/zee5.py
@@ -133,8 +133,8 @@ def _perform_login(self, username, password):
     def _real_extract(self, url):
         video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
         access_token_request = self._download_json(
-            'https://useraction.zee5.com/token/platform_tokens.php?platform_name=web_app',
-            video_id, note='Downloading access token')
+            'https://launchapi.zee5.com/launch?platform_name=web_app',
+            video_id, note='Downloading access token')['platform_token']
         data = {
             'x-access-token': access_token_request['token']
         }
@@ -240,8 +240,8 @@ class Zee5SeriesIE(InfoExtractor):
 
     def _entries(self, show_id):
         access_token_request = self._download_json(
-            'https://useraction.zee5.com/token/platform_tokens.php?platform_name=web_app',
-            show_id, note='Downloading access token')
+            'https://launchapi.zee5.com/launch?platform_name=web_app',
+            show_id, note='Downloading access token')['platform_token']
         headers = {
             'X-Access-Token': access_token_request['token'],
             'Referer': 'https://www.zee5.com/',

From ed711897814f3ee0b1822e4205e74133467e8f1c Mon Sep 17 00:00:00 2001
From: trainman261 <trainman261@users.noreply.github.com>
Date: Sun, 20 Aug 2023 18:35:57 +0200
Subject: [PATCH 505/871] [ie/CBCPlayerPlaylist] Add extractor (#7870)

Authored by: trainman261
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/cbc.py         | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index d4d3b6074..194ad8356 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -303,6 +303,7 @@
 from .cbc import (
     CBCIE,
     CBCPlayerIE,
+    CBCPlayerPlaylistIE,
     CBCGemIE,
     CBCGemPlaylistIE,
     CBCGemLiveIE,
diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index 9413281a5..b3c5471f7 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -2,6 +2,7 @@
 import json
 import base64
 import time
+import urllib.parse
 
 from .common import InfoExtractor
 from ..compat import (
@@ -228,6 +229,38 @@ def _real_extract(self, url):
         }
 
 
+class CBCPlayerPlaylistIE(InfoExtractor):
+    IE_NAME = 'cbc.ca:player:playlist'
+    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?:player/)(?!play/)(?P<id>[^?#]+)'
+    _TESTS = [{
+        'url': 'https://www.cbc.ca/player/news/TV%20Shows/The%20National/Latest%20Broadcast',
+        'playlist_mincount': 25,
+        'info_dict': {
+            'id': 'news/tv shows/the national/latest broadcast',
+        }
+    }, {
+        'url': 'https://www.cbc.ca/player/news/Canada/North',
+        'playlist_mincount': 25,
+        'info_dict': {
+            'id': 'news/canada/north',
+        }
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = urllib.parse.unquote(self._match_id(url)).lower()
+        webpage = self._download_webpage(url, playlist_id)
+        json_content = self._search_json(
+            r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', playlist_id)
+
+        def entries():
+            for video_id in traverse_obj(json_content, (
+                'video', 'clipsByCategory', lambda k, _: k.lower() == playlist_id, 'items', ..., 'id'
+            )):
+                yield self.url_result(f'https://www.cbc.ca/player/play/{video_id}', CBCPlayerIE)
+
+        return self.playlist_result(entries(), playlist_id)
+
+
 class CBCGemIE(InfoExtractor):
     IE_NAME = 'gem.cbc.ca'
     _VALID_URL = r'https?://gem\.cbc\.ca/(?:media/)?(?P<id>[0-9a-z-]+/s[0-9]+[a-z][0-9]+)'

From 7cccab79e7d00ed965b48b8cefce1da8a0513409 Mon Sep 17 00:00:00 2001
From: Davin Kevin <davin.kevin@gmail.com>
Date: Sun, 20 Aug 2023 19:25:49 +0200
Subject: [PATCH 506/871] [ie/wat.tv] Fix extraction (#7898)

Closes #7303
Authored by: davinkevin
---
 yt_dlp/extractor/tf1.py | 19 +++++++++++++++++++
 yt_dlp/extractor/wat.py | 14 +++++++++++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/tf1.py b/yt_dlp/extractor/tf1.py
index 4cf0322b3..aba4927ae 100644
--- a/yt_dlp/extractor/tf1.py
+++ b/yt_dlp/extractor/tf1.py
@@ -27,6 +27,25 @@ class TF1IE(InfoExtractor):
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.tf1.fr/tmc/burger-quiz/videos/burger-quiz-du-19-aout-2023-s03-episode-21-85585666.html',
+        'info_dict': {
+            'id': '14010600',
+            'ext': 'mp4',
+            'title': 'Burger Quiz - S03 EP21 avec Eye Haidara, Anne Depétrini, Jonathan Zaccaï et Pio Marmaï',
+            'thumbnail': 'https://photos.tf1.fr/1280/720/burger-quiz-11-9adb79-0@1x.jpg',
+            'description': 'Manu Payet recevra Eye Haidara, Anne Depétrini, Jonathan Zaccaï et Pio Marmaï.',
+            'upload_date': '20230819',
+            'timestamp': 1692469471,
+            'season_number': 3,
+            'series': 'Burger Quiz',
+            'episode_number': 21,
+            'season': 'Season 3',
+            'tags': 'count:13',
+            'episode': 'Episode 21',
+            'duration': 2312
+        },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
         'only_matching': True,
diff --git a/yt_dlp/extractor/wat.py b/yt_dlp/extractor/wat.py
index 7c62d2866..9ea3fddd6 100644
--- a/yt_dlp/extractor/wat.py
+++ b/yt_dlp/extractor/wat.py
@@ -41,6 +41,18 @@ class WatIE(InfoExtractor):
             'expected_warnings': ["Ce contenu n'est pas disponible pour l'instant."],
             'skip': 'This content is no longer available',
         },
+        {
+            'url': 'wat:14010600',
+            'info_dict': {
+                'id': '14010600',
+                'title': 'Burger Quiz - S03 EP21 avec Eye Haidara, Anne Depétrini, Jonathan Zaccaï et Pio Marmaï',
+                'thumbnail': 'https://photos.tf1.fr/1280/720/burger-quiz-11-9adb79-0@1x.jpg',
+                'upload_date': '20230819',
+                'duration': 2312,
+                'ext': 'mp4',
+            },
+            'params': {'skip_download': 'm3u8'},
+        }
     ]
     _GEO_BYPASS = False
 
@@ -54,7 +66,7 @@ def _real_extract(self, url):
         #     'http://www.wat.tv/interface/contentv4s/' + video_id, video_id)
         video_data = self._download_json(
             'https://mediainfo.tf1.fr/mediainfocombo/' + video_id,
-            video_id, query={'context': 'MYTF1', 'pver': '4020003'})
+            video_id, query={'pver': '5010000'})
         video_info = video_data['media']
 
         error_desc = video_info.get('error_desc')

From fcd6a76adc49d5cd8783985c7ce35384b72e545f Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Fri, 25 Aug 2023 07:10:44 +0000
Subject: [PATCH 507/871] [tests] Add tests for socks proxies (#7908)

Authored by: coletdjnz
---
 test/conftest.py        |  21 ++
 test/test_networking.py |  16 --
 test/test_socks.py      | 529 +++++++++++++++++++++++++++++++++-------
 3 files changed, 464 insertions(+), 102 deletions(-)
 create mode 100644 test/conftest.py

diff --git a/test/conftest.py b/test/conftest.py
new file mode 100644
index 000000000..15549d30b
--- /dev/null
+++ b/test/conftest.py
@@ -0,0 +1,21 @@
+import functools
+import inspect
+
+import pytest
+
+from yt_dlp.networking import RequestHandler
+from yt_dlp.networking.common import _REQUEST_HANDLERS
+from yt_dlp.utils._utils import _YDLLogger as FakeLogger
+
+
+@pytest.fixture
+def handler(request):
+    RH_KEY = request.param
+    if inspect.isclass(RH_KEY) and issubclass(RH_KEY, RequestHandler):
+        handler = RH_KEY
+    elif RH_KEY in _REQUEST_HANDLERS:
+        handler = _REQUEST_HANDLERS[RH_KEY]
+    else:
+        pytest.skip(f'{RH_KEY} request handler is not available')
+
+    return functools.partial(handler, logger=FakeLogger)
diff --git a/test/test_networking.py b/test/test_networking.py
index 2622d24da..5308c8d6f 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -8,12 +8,10 @@
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-import functools
 import gzip
 import http.client
 import http.cookiejar
 import http.server
-import inspect
 import io
 import pathlib
 import random
@@ -40,7 +38,6 @@
     Response,
 )
 from yt_dlp.networking._urllib import UrllibRH
-from yt_dlp.networking.common import _REQUEST_HANDLERS
 from yt_dlp.networking.exceptions import (
     CertificateVerifyError,
     HTTPError,
@@ -307,19 +304,6 @@ def setup_class(cls):
         cls.https_server_thread.start()
 
 
-@pytest.fixture
-def handler(request):
-    RH_KEY = request.param
-    if inspect.isclass(RH_KEY) and issubclass(RH_KEY, RequestHandler):
-        handler = RH_KEY
-    elif RH_KEY in _REQUEST_HANDLERS:
-        handler = _REQUEST_HANDLERS[RH_KEY]
-    else:
-        pytest.skip(f'{RH_KEY} request handler is not available')
-
-    return functools.partial(handler, logger=FakeLogger)
-
-
 class TestHTTPRequestHandler(TestRequestHandlerBase):
     @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
     def test_verify_cert(self, handler):
diff --git a/test/test_socks.py b/test/test_socks.py
index 6651290d2..95ffce275 100644
--- a/test/test_socks.py
+++ b/test/test_socks.py
@@ -1,113 +1,470 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
+import threading
 import unittest
 
+import pytest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
+import abc
+import contextlib
+import enum
+import functools
+import http.server
+import json
 import random
-import subprocess
-import urllib.request
+import socket
+import struct
+import time
+from socketserver import (
+    BaseRequestHandler,
+    StreamRequestHandler,
+    ThreadingTCPServer,
+)
 
-from test.helper import FakeYDL, get_params, is_download_test
+from test.helper import http_server_port
+from yt_dlp.networking import Request
+from yt_dlp.networking.exceptions import ProxyError, TransportError
+from yt_dlp.socks import (
+    SOCKS4_REPLY_VERSION,
+    SOCKS4_VERSION,
+    SOCKS5_USER_AUTH_SUCCESS,
+    SOCKS5_USER_AUTH_VERSION,
+    SOCKS5_VERSION,
+    Socks5AddressType,
+    Socks5Auth,
+)
+
+SOCKS5_USER_AUTH_FAILURE = 0x1
 
 
-@is_download_test
-class TestMultipleSocks(unittest.TestCase):
-    @staticmethod
-    def _check_params(attrs):
-        params = get_params()
-        for attr in attrs:
-            if attr not in params:
-                print('Missing %s. Skipping.' % attr)
+class Socks4CD(enum.IntEnum):
+    REQUEST_GRANTED = 90
+    REQUEST_REJECTED_OR_FAILED = 91
+    REQUEST_REJECTED_CANNOT_CONNECT_TO_IDENTD = 92
+    REQUEST_REJECTED_DIFFERENT_USERID = 93
+
+
+class Socks5Reply(enum.IntEnum):
+    SUCCEEDED = 0x0
+    GENERAL_FAILURE = 0x1
+    CONNECTION_NOT_ALLOWED = 0x2
+    NETWORK_UNREACHABLE = 0x3
+    HOST_UNREACHABLE = 0x4
+    CONNECTION_REFUSED = 0x5
+    TTL_EXPIRED = 0x6
+    COMMAND_NOT_SUPPORTED = 0x7
+    ADDRESS_TYPE_NOT_SUPPORTED = 0x8
+
+
+class SocksTestRequestHandler(BaseRequestHandler):
+
+    def __init__(self, *args, socks_info=None, **kwargs):
+        self.socks_info = socks_info
+        super().__init__(*args, **kwargs)
+
+
+class SocksProxyHandler(BaseRequestHandler):
+    def __init__(self, request_handler_class, socks_server_kwargs, *args, **kwargs):
+        self.socks_kwargs = socks_server_kwargs or {}
+        self.request_handler_class = request_handler_class
+        super().__init__(*args, **kwargs)
+
+
+class Socks5ProxyHandler(StreamRequestHandler, SocksProxyHandler):
+
+    # SOCKS5 protocol https://tools.ietf.org/html/rfc1928
+    # SOCKS5 username/password authentication https://tools.ietf.org/html/rfc1929
+
+    def handle(self):
+        sleep = self.socks_kwargs.get('sleep')
+        if sleep:
+            time.sleep(sleep)
+        version, nmethods = self.connection.recv(2)
+        assert version == SOCKS5_VERSION
+        methods = list(self.connection.recv(nmethods))
+
+        auth = self.socks_kwargs.get('auth')
+
+        if auth is not None and Socks5Auth.AUTH_USER_PASS not in methods:
+            self.connection.sendall(struct.pack('!BB', SOCKS5_VERSION, Socks5Auth.AUTH_NO_ACCEPTABLE))
+            self.server.close_request(self.request)
+            return
+
+        elif Socks5Auth.AUTH_USER_PASS in methods:
+            self.connection.sendall(struct.pack("!BB", SOCKS5_VERSION, Socks5Auth.AUTH_USER_PASS))
+
+            _, user_len = struct.unpack('!BB', self.connection.recv(2))
+            username = self.connection.recv(user_len).decode()
+            pass_len = ord(self.connection.recv(1))
+            password = self.connection.recv(pass_len).decode()
+
+            if username == auth[0] and password == auth[1]:
+                self.connection.sendall(struct.pack('!BB', SOCKS5_USER_AUTH_VERSION, SOCKS5_USER_AUTH_SUCCESS))
+            else:
+                self.connection.sendall(struct.pack('!BB', SOCKS5_USER_AUTH_VERSION, SOCKS5_USER_AUTH_FAILURE))
+                self.server.close_request(self.request)
                 return
-        return params
 
-    def test_proxy_http(self):
-        params = self._check_params(['primary_proxy', 'primary_server_ip'])
-        if params is None:
-            return
-        ydl = FakeYDL({
-            'proxy': params['primary_proxy']
-        })
-        self.assertEqual(
-            ydl.urlopen('http://yt-dl.org/ip').read().decode(),
-            params['primary_server_ip'])
-
-    def test_proxy_https(self):
-        params = self._check_params(['primary_proxy', 'primary_server_ip'])
-        if params is None:
-            return
-        ydl = FakeYDL({
-            'proxy': params['primary_proxy']
-        })
-        self.assertEqual(
-            ydl.urlopen('https://yt-dl.org/ip').read().decode(),
-            params['primary_server_ip'])
-
-    def test_secondary_proxy_http(self):
-        params = self._check_params(['secondary_proxy', 'secondary_server_ip'])
-        if params is None:
-            return
-        ydl = FakeYDL()
-        req = urllib.request.Request('http://yt-dl.org/ip')
-        req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
-        self.assertEqual(
-            ydl.urlopen(req).read().decode(),
-            params['secondary_server_ip'])
-
-    def test_secondary_proxy_https(self):
-        params = self._check_params(['secondary_proxy', 'secondary_server_ip'])
-        if params is None:
-            return
-        ydl = FakeYDL()
-        req = urllib.request.Request('https://yt-dl.org/ip')
-        req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
-        self.assertEqual(
-            ydl.urlopen(req).read().decode(),
-            params['secondary_server_ip'])
-
-
-@is_download_test
-class TestSocks(unittest.TestCase):
-    _SKIP_SOCKS_TEST = True
-
-    def setUp(self):
-        if self._SKIP_SOCKS_TEST:
+        elif Socks5Auth.AUTH_NONE in methods:
+            self.connection.sendall(struct.pack('!BB', SOCKS5_VERSION, Socks5Auth.AUTH_NONE))
+        else:
+            self.connection.sendall(struct.pack('!BB', SOCKS5_VERSION, Socks5Auth.AUTH_NO_ACCEPTABLE))
+            self.server.close_request(self.request)
             return
 
-        self.port = random.randint(20000, 30000)
-        self.server_process = subprocess.Popen([
-            'srelay', '-f', '-i', '127.0.0.1:%d' % self.port],
-            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        version, command, _, address_type = struct.unpack('!BBBB', self.connection.recv(4))
+        socks_info = {
+            'version': version,
+            'auth_methods': methods,
+            'command': command,
+            'client_address': self.client_address,
+            'ipv4_address': None,
+            'domain_address': None,
+            'ipv6_address': None,
+        }
+        if address_type == Socks5AddressType.ATYP_IPV4:
+            socks_info['ipv4_address'] = socket.inet_ntoa(self.connection.recv(4))
+        elif address_type == Socks5AddressType.ATYP_DOMAINNAME:
+            socks_info['domain_address'] = self.connection.recv(ord(self.connection.recv(1))).decode()
+        elif address_type == Socks5AddressType.ATYP_IPV6:
+            socks_info['ipv6_address'] = socket.inet_ntop(socket.AF_INET6, self.connection.recv(16))
+        else:
+            self.server.close_request(self.request)
 
-    def tearDown(self):
-        if self._SKIP_SOCKS_TEST:
+        socks_info['port'] = struct.unpack('!H', self.connection.recv(2))[0]
+
+        # dummy response, the returned IP is just a placeholder
+        self.connection.sendall(struct.pack(
+            '!BBBBIH', SOCKS5_VERSION, self.socks_kwargs.get('reply', Socks5Reply.SUCCEEDED), 0x0, 0x1, 0x7f000001, 40000))
+
+        self.request_handler_class(self.request, self.client_address, self.server, socks_info=socks_info)
+
+
+class Socks4ProxyHandler(StreamRequestHandler, SocksProxyHandler):
+
+    # SOCKS4 protocol http://www.openssh.com/txt/socks4.protocol
+    # SOCKS4A protocol http://www.openssh.com/txt/socks4a.protocol
+
+    def _read_until_null(self):
+        return b''.join(iter(functools.partial(self.connection.recv, 1), b'\x00'))
+
+    def handle(self):
+        sleep = self.socks_kwargs.get('sleep')
+        if sleep:
+            time.sleep(sleep)
+        socks_info = {
+            'version': SOCKS4_VERSION,
+            'command': None,
+            'client_address': self.client_address,
+            'ipv4_address': None,
+            'port': None,
+            'domain_address': None,
+        }
+        version, command, dest_port, dest_ip = struct.unpack('!BBHI', self.connection.recv(8))
+        socks_info['port'] = dest_port
+        socks_info['command'] = command
+        if version != SOCKS4_VERSION:
+            self.server.close_request(self.request)
+            return
+        use_remote_dns = False
+        if 0x0 < dest_ip <= 0xFF:
+            use_remote_dns = True
+        else:
+            socks_info['ipv4_address'] = socket.inet_ntoa(struct.pack("!I", dest_ip))
+
+        user_id = self._read_until_null().decode()
+        if user_id != (self.socks_kwargs.get('user_id') or ''):
+            self.connection.sendall(struct.pack(
+                '!BBHI', SOCKS4_REPLY_VERSION, Socks4CD.REQUEST_REJECTED_DIFFERENT_USERID, 0x00, 0x00000000))
+            self.server.close_request(self.request)
             return
 
-        self.server_process.terminate()
-        self.server_process.communicate()
+        if use_remote_dns:
+            socks_info['domain_address'] = self._read_until_null().decode()
 
-    def _get_ip(self, protocol):
-        if self._SKIP_SOCKS_TEST:
-            return '127.0.0.1'
+        # dummy response, the returned IP is just a placeholder
+        self.connection.sendall(
+            struct.pack(
+                '!BBHI', SOCKS4_REPLY_VERSION,
+                self.socks_kwargs.get('cd_reply', Socks4CD.REQUEST_GRANTED), 40000, 0x7f000001))
 
-        ydl = FakeYDL({
-            'proxy': '%s://127.0.0.1:%d' % (protocol, self.port),
-        })
-        return ydl.urlopen('http://yt-dl.org/ip').read().decode()
+        self.request_handler_class(self.request, self.client_address, self.server, socks_info=socks_info)
 
-    def test_socks4(self):
-        self.assertTrue(isinstance(self._get_ip('socks4'), str))
 
-    def test_socks4a(self):
-        self.assertTrue(isinstance(self._get_ip('socks4a'), str))
+class IPv6ThreadingTCPServer(ThreadingTCPServer):
+    address_family = socket.AF_INET6
 
-    def test_socks5(self):
-        self.assertTrue(isinstance(self._get_ip('socks5'), str))
+
+class SocksHTTPTestRequestHandler(http.server.BaseHTTPRequestHandler, SocksTestRequestHandler):
+    def do_GET(self):
+        if self.path == '/socks_info':
+            payload = json.dumps(self.socks_info.copy())
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json; charset=utf-8')
+            self.send_header('Content-Length', str(len(payload)))
+            self.end_headers()
+            self.wfile.write(payload.encode())
+
+
+@contextlib.contextmanager
+def socks_server(socks_server_class, request_handler, bind_ip=None, **socks_server_kwargs):
+    server = server_thread = None
+    try:
+        bind_address = bind_ip or '127.0.0.1'
+        server_type = ThreadingTCPServer if '.' in bind_address else IPv6ThreadingTCPServer
+        server = server_type(
+            (bind_address, 0), functools.partial(socks_server_class, request_handler, socks_server_kwargs))
+        server_port = http_server_port(server)
+        server_thread = threading.Thread(target=server.serve_forever)
+        server_thread.daemon = True
+        server_thread.start()
+        if '.' not in bind_address:
+            yield f'[{bind_address}]:{server_port}'
+        else:
+            yield f'{bind_address}:{server_port}'
+    finally:
+        server.shutdown()
+        server.server_close()
+        server_thread.join(2.0)
+
+
+class SocksProxyTestContext(abc.ABC):
+    REQUEST_HANDLER_CLASS = None
+
+    def socks_server(self, server_class, *args, **kwargs):
+        return socks_server(server_class, self.REQUEST_HANDLER_CLASS, *args, **kwargs)
+
+    @abc.abstractmethod
+    def socks_info_request(self, handler, target_domain=None, target_port=None, **req_kwargs) -> dict:
+        """return a dict of socks_info"""
+
+
+class HTTPSocksTestProxyContext(SocksProxyTestContext):
+    REQUEST_HANDLER_CLASS = SocksHTTPTestRequestHandler
+
+    def socks_info_request(self, handler, target_domain=None, target_port=None, **req_kwargs):
+        request = Request(f'http://{target_domain or "127.0.0.1"}:{target_port or "40000"}/socks_info', **req_kwargs)
+        handler.validate(request)
+        return json.loads(handler.send(request).read().decode())
+
+
+CTX_MAP = {
+    'http': HTTPSocksTestProxyContext,
+}
+
+
+@pytest.fixture(scope='module')
+def ctx(request):
+    return CTX_MAP[request.param]()
+
+
+class TestSocks4Proxy:
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_socks4_no_auth(self, handler, ctx):
+        with handler() as rh:
+            with ctx.socks_server(Socks4ProxyHandler) as server_address:
+                response = ctx.socks_info_request(
+                    rh, proxies={'all': f'socks4://{server_address}'})
+                assert response['version'] == 4
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_socks4_auth(self, handler, ctx):
+        with handler() as rh:
+            with ctx.socks_server(Socks4ProxyHandler, user_id='user') as server_address:
+                with pytest.raises(ProxyError):
+                    ctx.socks_info_request(rh, proxies={'all': f'socks4://{server_address}'})
+                response = ctx.socks_info_request(
+                    rh, proxies={'all': f'socks4://user:@{server_address}'})
+                assert response['version'] == 4
+
+    @pytest.mark.parametrize('handler,ctx', [
+        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
+            reason='socks4a implementation currently broken when destination is not a domain name'))
+    ], indirect=True)
+    def test_socks4a_ipv4_target(self, handler, ctx):
+        with ctx.socks_server(Socks4ProxyHandler) as server_address:
+            with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh, target_domain='127.0.0.1')
+                assert response['version'] == 4
+                assert response['ipv4_address'] == '127.0.0.1'
+                assert response['domain_address'] is None
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_socks4a_domain_target(self, handler, ctx):
+        with ctx.socks_server(Socks4ProxyHandler) as server_address:
+            with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh, target_domain='localhost')
+                assert response['version'] == 4
+                assert response['ipv4_address'] is None
+                assert response['domain_address'] == 'localhost'
+
+    @pytest.mark.parametrize('handler,ctx', [
+        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
+            reason='source_address is not yet supported for socks4 proxies'))
+    ], indirect=True)
+    def test_ipv4_client_source_address(self, handler, ctx):
+        with ctx.socks_server(Socks4ProxyHandler) as server_address:
+            source_address = f'127.0.0.{random.randint(5, 255)}'
+            with handler(proxies={'all': f'socks4://{server_address}'},
+                         source_address=source_address) as rh:
+                response = ctx.socks_info_request(rh)
+                assert response['client_address'][0] == source_address
+                assert response['version'] == 4
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('reply_code', [
+        Socks4CD.REQUEST_REJECTED_OR_FAILED,
+        Socks4CD.REQUEST_REJECTED_CANNOT_CONNECT_TO_IDENTD,
+        Socks4CD.REQUEST_REJECTED_DIFFERENT_USERID,
+    ])
+    def test_socks4_errors(self, handler, ctx, reply_code):
+        with ctx.socks_server(Socks4ProxyHandler, cd_reply=reply_code) as server_address:
+            with handler(proxies={'all': f'socks4://{server_address}'}) as rh:
+                with pytest.raises(ProxyError):
+                    ctx.socks_info_request(rh)
+
+    @pytest.mark.parametrize('handler,ctx', [
+        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
+            reason='IPv6 socks4 proxies are not yet supported'))
+    ], indirect=True)
+    def test_ipv6_socks4_proxy(self, handler, ctx):
+        with ctx.socks_server(Socks4ProxyHandler, bind_ip='::1') as server_address:
+            with handler(proxies={'all': f'socks4://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh, target_domain='127.0.0.1')
+                assert response['client_address'][0] == '::1'
+                assert response['ipv4_address'] == '127.0.0.1'
+                assert response['version'] == 4
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_timeout(self, handler, ctx):
+        with ctx.socks_server(Socks4ProxyHandler, sleep=2) as server_address:
+            with handler(proxies={'all': f'socks4://{server_address}'}, timeout=1) as rh:
+                with pytest.raises(TransportError):
+                    ctx.socks_info_request(rh)
+
+
+class TestSocks5Proxy:
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_socks5_no_auth(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler) as server_address:
+            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh)
+                assert response['auth_methods'] == [0x0]
+                assert response['version'] == 5
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_socks5_user_pass(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler, auth=('test', 'testpass')) as server_address:
+            with handler() as rh:
+                with pytest.raises(ProxyError):
+                    ctx.socks_info_request(rh, proxies={'all': f'socks5://{server_address}'})
+
+                response = ctx.socks_info_request(
+                    rh, proxies={'all': f'socks5://test:testpass@{server_address}'})
+
+                assert response['auth_methods'] == [Socks5Auth.AUTH_NONE, Socks5Auth.AUTH_USER_PASS]
+                assert response['version'] == 5
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_socks5_ipv4_target(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler) as server_address:
+            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh, target_domain='127.0.0.1')
+                assert response['ipv4_address'] == '127.0.0.1'
+                assert response['version'] == 5
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_socks5_domain_target(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler) as server_address:
+            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh, target_domain='localhost')
+                assert response['ipv4_address'] == '127.0.0.1'
+                assert response['version'] == 5
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_socks5h_domain_target(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler) as server_address:
+            with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh, target_domain='localhost')
+                assert response['ipv4_address'] is None
+                assert response['domain_address'] == 'localhost'
+                assert response['version'] == 5
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_socks5h_ip_target(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler) as server_address:
+            with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh, target_domain='127.0.0.1')
+                assert response['ipv4_address'] == '127.0.0.1'
+                assert response['domain_address'] is None
+                assert response['version'] == 5
+
+    @pytest.mark.parametrize('handler,ctx', [
+        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
+            reason='IPv6 destination addresses are not yet supported'))
+    ], indirect=True)
+    def test_socks5_ipv6_destination(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler) as server_address:
+            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh, target_domain='[::1]')
+                assert response['ipv6_address'] == '::1'
+                assert response['port'] == 80
+                assert response['version'] == 5
+
+    @pytest.mark.parametrize('handler,ctx', [
+        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
+            reason='IPv6 socks5 proxies are not yet supported'))
+    ], indirect=True)
+    def test_ipv6_socks5_proxy(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler, bind_ip='::1') as server_address:
+            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+                response = ctx.socks_info_request(rh, target_domain='127.0.0.1')
+                assert response['client_address'][0] == '::1'
+                assert response['ipv4_address'] == '127.0.0.1'
+                assert response['version'] == 5
+
+    # XXX: is there any feasible way of testing IPv6 source addresses?
+    # Same would go for non-proxy source_address test...
+    @pytest.mark.parametrize('handler,ctx', [
+        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
+            reason='source_address is not yet supported for socks5 proxies'))
+    ], indirect=True)
+    def test_ipv4_client_source_address(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler) as server_address:
+            source_address = f'127.0.0.{random.randint(5, 255)}'
+            with handler(proxies={'all': f'socks5://{server_address}'}, source_address=source_address) as rh:
+                response = ctx.socks_info_request(rh)
+                assert response['client_address'][0] == source_address
+                assert response['version'] == 5
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('reply_code', [
+        Socks5Reply.GENERAL_FAILURE,
+        Socks5Reply.CONNECTION_NOT_ALLOWED,
+        Socks5Reply.NETWORK_UNREACHABLE,
+        Socks5Reply.HOST_UNREACHABLE,
+        Socks5Reply.CONNECTION_REFUSED,
+        Socks5Reply.TTL_EXPIRED,
+        Socks5Reply.COMMAND_NOT_SUPPORTED,
+        Socks5Reply.ADDRESS_TYPE_NOT_SUPPORTED,
+    ])
+    def test_socks5_errors(self, handler, ctx, reply_code):
+        with ctx.socks_server(Socks5ProxyHandler, reply=reply_code) as server_address:
+            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+                with pytest.raises(ProxyError):
+                    ctx.socks_info_request(rh)
+
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    def test_timeout(self, handler, ctx):
+        with ctx.socks_server(Socks5ProxyHandler, sleep=2) as server_address:
+            with handler(proxies={'all': f'socks5://{server_address}'}, timeout=1) as rh:
+                with pytest.raises(TransportError):
+                    ctx.socks_info_request(rh)
 
 
 if __name__ == '__main__':

From 1be0a96a4d14f629097509fcc89d15f69a8243c7 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 26 Aug 2023 22:29:56 +0200
Subject: [PATCH 508/871] [docs] Update collaborators

Authored by: Grub4K
---
 CONTRIBUTORS     |  2 +-
 Collaborators.md | 10 ----------
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 6ccd08931..6b9b9f470 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -2,7 +2,6 @@ pukkandan (owner)
 shirt-dev (collaborator)
 coletdjnz/colethedj (collaborator)
 Ashish0804 (collaborator)
-nao20010128nao/Lesmiscore (collaborator)
 bashonly (collaborator)
 Grub4K (collaborator)
 h-h-h-h
@@ -467,3 +466,4 @@ nnoboa
 rdamas
 RfadnjdExt
 urectanc
+nao20010128nao/Lesmiscore
diff --git a/Collaborators.md b/Collaborators.md
index a0976dd8c..70ab616f1 100644
--- a/Collaborators.md
+++ b/Collaborators.md
@@ -44,16 +44,6 @@ ## [Ashish0804](https://github.com/Ashish0804) <sub><sup>[Inactive]</sup></sub>
 * Improved/fixed support for HiDive, HotStar, Hungama, LBRY, LinkedInLearning, Mxplayer, SonyLiv, TV2, Vimeo, VLive etc
 
 
-## [Lesmiscore](https://github.com/Lesmiscore)
-
-**Bitcoin**: bc1qfd02r007cutfdjwjmyy9w23rjvtls6ncve7r3s  
-**Monacoin**: mona1q3tf7dzvshrhfe3md379xtvt2n22duhglv5dskr
-
-* Download live from start to end for YouTube
-* Added support for new websites AbemaTV, mildom, PixivSketch, skeb, radiko, voicy, mirrativ, openrec, whowatch, damtomo, 17.live, mixch etc
-* Improved/fixed support for fc2, YahooJapanNews, tver, iwara etc
-
-
 ## [bashonly](https://github.com/bashonly)
 
 * `--update-to`, automated release, nightly builds

From 59e92b1f1833440bb2190f847eb735cf0f90bc85 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 27 Aug 2023 00:13:30 +0200
Subject: [PATCH 509/871] [rh/urllib] Simplify gzip decoding (#7611)

Authored by: Grub4K
---
 yt_dlp/networking/_urllib.py | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index 0c4794954..5a804d99b 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import functools
-import gzip
 import http.client
 import io
 import socket
@@ -155,20 +154,9 @@ def brotli(data):
 
     @staticmethod
     def gz(data):
-        gz = gzip.GzipFile(fileobj=io.BytesIO(data), mode='rb')
-        try:
-            return gz.read()
-        except OSError as original_oserror:
-            # There may be junk add the end of the file
-            # See http://stackoverflow.com/q/4928560/35070 for details
-            for i in range(1, 1024):
-                try:
-                    gz = gzip.GzipFile(fileobj=io.BytesIO(data[:-i]), mode='rb')
-                    return gz.read()
-                except OSError:
-                    continue
-            else:
-                raise original_oserror
+        # There may be junk added the end of the file
+        # We ignore it by only ever decoding a single gzip payload
+        return zlib.decompress(data, wbits=zlib.MAX_WBITS | 16)
 
     def http_request(self, req):
         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not

From d7aee8e310b2c4f21d50aac0b420e1b3abde21a4 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Fri, 25 Aug 2023 08:44:05 -0500
Subject: [PATCH 510/871] [ie/Mzaalo] Improve `_VALID_URL`

Authored by: bashonly
---
 yt_dlp/extractor/mzaalo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/mzaalo.py b/yt_dlp/extractor/mzaalo.py
index c6f420cea..1996368cc 100644
--- a/yt_dlp/extractor/mzaalo.py
+++ b/yt_dlp/extractor/mzaalo.py
@@ -8,7 +8,7 @@
 
 
 class MzaaloIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mzaalo\.com/play/(?P<type>movie|original|clip)/(?P<id>[a-fA-F0-9-]+)/[\w-]+'
+    _VALID_URL = r'(?i)https?://(?:www\.)?mzaalo\.com/(?:play|watch)/(?P<type>movie|original|clip)/(?P<id>[a-f0-9-]+)/[\w-]+'
     _TESTS = [{
         # Movies
         'url': 'https://www.mzaalo.com/play/movie/c0958d9f-f90e-4503-a755-44358758921d/Jamun',
@@ -55,6 +55,9 @@ class MzaaloIE(InfoExtractor):
             'language': 'hin',
         },
         'params': {'skip_download': 'm3u8'}
+    }, {
+        'url': 'https://mzaalo.com/watch/MOVIE/389c892d-0b65-4019-bf73-d4edcb1c014f/Chalo-Dilli',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 56b3dc03354b75be995759d8441d2754c0442b9a Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 27 Aug 2023 18:33:25 -0500
Subject: [PATCH 511/871] [ie/StagePlus] Fix m3u8 extraction (#7929)

Closes #7928
Authored by: bashonly
---
 yt_dlp/extractor/stageplus.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/stageplus.py b/yt_dlp/extractor/stageplus.py
index adb4ebbc2..4bed4d646 100644
--- a/yt_dlp/extractor/stageplus.py
+++ b/yt_dlp/extractor/stageplus.py
@@ -484,18 +484,15 @@ def _real_extract(self, url):
             'url': 'url',
         })) or None
 
-        m3u8_headers = {'jwt': self._TOKEN}
-
         entries = []
         for idx, video in enumerate(traverse_obj(data, (
                 'performanceWorks', lambda _, v: v['id'] and url_or_none(v['stream']['url']))), 1):
             formats, subtitles = self._extract_m3u8_formats_and_subtitles(
-                video['stream']['url'], video['id'], 'mp4', m3u8_id='hls', headers=m3u8_headers)
+                video['stream']['url'], video['id'], 'mp4', m3u8_id='hls', query={'token': self._TOKEN})
             entries.append({
                 'id': video['id'],
                 'formats': formats,
                 'subtitles': subtitles,
-                'http_headers': m3u8_headers,
                 'album': metadata.get('title'),
                 'album_artist': metadata.get('artist'),
                 'track_number': idx,

From c2d8ee0000302aba63476b7d5bd8793e57b6c8c6 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Mon, 28 Aug 2023 23:09:14 +0200
Subject: [PATCH 512/871] [ie/weverse] Support extraction without auth (#7924)

Authored by: seproDev
---
 yt_dlp/extractor/weverse.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/weverse.py b/yt_dlp/extractor/weverse.py
index 9a08b8e43..bbf62856a 100644
--- a/yt_dlp/extractor/weverse.py
+++ b/yt_dlp/extractor/weverse.py
@@ -70,10 +70,8 @@ def _real_initialize(self):
             return
 
         token = try_call(lambda: self._get_cookies('https://weverse.io/')['we2_access_token'].value)
-        if not token:
-            self.raise_login_required()
-
-        WeverseBaseIE._API_HEADERS['Authorization'] = f'Bearer {token}'
+        if token:
+            WeverseBaseIE._API_HEADERS['Authorization'] = f'Bearer {token}'
 
     def _call_api(self, ep, video_id, data=None, note='Downloading API JSON'):
         # Ref: https://ssl.pstatic.net/static/wevweb/2_3_2_11101725/public/static/js/2488.a09b41ff.chunk.js
@@ -101,11 +99,14 @@ def _call_api(self, ep, video_id, data=None, note='Downloading API JSON'):
                 self.raise_login_required(
                     'Session token has expired. Log in again or refresh cookies in browser')
             elif isinstance(e.cause, HTTPError) and e.cause.status == 403:
-                raise ExtractorError('Your account does not have access to this content', expected=True)
+                if 'Authorization' in self._API_HEADERS:
+                    raise ExtractorError('Your account does not have access to this content', expected=True)
+                self.raise_login_required()
             raise
 
     def _call_post_api(self, video_id):
-        return self._call_api(f'/post/v1.0/post-{video_id}?fieldSet=postV1', video_id)
+        path = '' if 'Authorization' in self._API_HEADERS else '/preview'
+        return self._call_api(f'/post/v1.0/post-{video_id}{path}?fieldSet=postV1', video_id)
 
     def _get_community_id(self, channel):
         return str(self._call_api(

From b9f2bc2dbed2323734a0d18e65e1e2e23dc833d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Touz=C3=A9?=
 <60022007+nathantouze@users.noreply.github.com>
Date: Mon, 28 Aug 2023 23:33:48 +0200
Subject: [PATCH 513/871] [ie/Dropbox] Fix extractor (#7926)

Closes #7005, Closes #7696
Authored by: nathantouze, bashonly, denhotte
---
 yt_dlp/extractor/dropbox.py | 42 ++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/dropbox.py b/yt_dlp/extractor/dropbox.py
index 214b309bf..ec86d7ad2 100644
--- a/yt_dlp/extractor/dropbox.py
+++ b/yt_dlp/extractor/dropbox.py
@@ -1,3 +1,4 @@
+import base64
 import os.path
 import re
 
@@ -5,14 +6,13 @@
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
-    traverse_obj,
-    try_get,
+    update_url_query,
     url_basename,
 )
 
 
 class DropboxIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dropbox[.]com/sh?/(?P<id>[a-zA-Z0-9]{15})/.*'
+    _VALID_URL = r'https?://(?:www\.)?dropbox\.com/(?:(?:e/)?scl/fi|sh?)/(?P<id>\w+)'
     _TESTS = [
         {
             'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0',
@@ -22,7 +22,16 @@ class DropboxIE(InfoExtractor):
                 'title': 'youtube-dl test video \'ä"BaW_jenozKc'
             }
         }, {
-            'url': 'https://www.dropbox.com/sh/662glsejgzoj9sr/AAByil3FGH9KFNZ13e08eSa1a/Pregame%20Ceremony%20Program%20PA%2020140518.m4v',
+            'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh',
+            'only_matching': True,
+        }, {
+            'url': 'https://www.dropbox.com/sh/2mgpiuq7kv8nqdf/AABy-fW4dkydT4GmWi2mdOUDa?dl=0&preview=Drone+Shot.mp4',
+            'only_matching': True,
+        }, {
+            'url': 'https://www.dropbox.com/scl/fi/r2kd2skcy5ylbbta5y1pz/DJI_0003.MP4?dl=0&rlkey=wcdgqangn7t3lnmmv6li9mu9h',
+            'only_matching': True,
+        }, {
+            'url': 'https://www.dropbox.com/e/scl/fi/r2kd2skcy5ylbbta5y1pz/DJI_0003.MP4?dl=0&rlkey=wcdgqangn7t3lnmmv6li9mu9h',
             'only_matching': True,
         },
     ]
@@ -53,16 +62,25 @@ def _real_extract(self, url):
             else:
                 raise ExtractorError('Password protected video, use --video-password <password>', expected=True)
 
-        info_json = self._search_json(r'InitReact\.mountComponent\(.*?,', webpage, 'mountComponent', video_id,
-                                      contains_pattern=r'{.+?"preview".+?}', end_pattern=r'\)')['props']
-        transcode_url = traverse_obj(info_json, ((None, 'preview'), 'file', 'preview', 'content', 'transcode_url'), get_all=False)
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(transcode_url, video_id)
+        formats, subtitles, has_anonymous_download = [], {}, False
+        for encoded in reversed(re.findall(r'registerStreamedPrefetch\s*\(\s*"[\w/+=]+"\s*,\s*"([\w/+=]+)"', webpage)):
+            decoded = base64.b64decode(encoded).decode('utf-8', 'ignore')
+            transcode_url = self._search_regex(
+                r'\n\x03(https://[^\x12\x03\n]+\.m3u8)', decoded, 'transcode url', default=None)
+            if not transcode_url:
+                continue
+            formats, subtitles = self._extract_m3u8_formats_and_subtitles(transcode_url, video_id)
+            has_anonymous_download = self._search_regex(r'(anonymous:\tanonymous)', decoded, 'anonymous', default=False)
+            break
 
         # downloads enabled we can get the original file
-        if 'anonymous' in (try_get(info_json, lambda x: x['sharePermission']['canDownloadRoles']) or []):
-            video_url = re.sub(r'[?&]dl=0', '', url)
-            video_url += ('?' if '?' not in video_url else '&') + 'dl=1'
-            formats.append({'url': video_url, 'format_id': 'original', 'format_note': 'Original', 'quality': 1})
+        if has_anonymous_download:
+            formats.append({
+                'url': update_url_query(url, {'dl': '1'}),
+                'format_id': 'original',
+                'format_note': 'Original',
+                'quality': 1
+            })
 
         return {
             'id': video_id,

From 665876034c8d3c031443f6b4958bed02ccdf4164 Mon Sep 17 00:00:00 2001
From: Stavros Ntentos <133706+stdedos@users.noreply.github.com>
Date: Tue, 29 Aug 2023 03:05:49 +0300
Subject: [PATCH 514/871] [ie/antenna] Support antenna.gr (#7584)

Authored by: stdedos
---
 yt_dlp/extractor/_extractors.py               |  4 +-
 .../extractor/{ant1newsgr.py => antenna.py}   | 53 ++++++++++++-------
 2 files changed, 36 insertions(+), 21 deletions(-)
 rename yt_dlp/extractor/{ant1newsgr.py => antenna.py} (72%)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 194ad8356..f11554bdd 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1699,8 +1699,8 @@
     MegaTVComIE,
     MegaTVComEmbedIE,
 )
-from .ant1newsgr import (
-    Ant1NewsGrWatchIE,
+from .antenna import (
+    AntennaGrWatchIE,
     Ant1NewsGrArticleIE,
     Ant1NewsGrEmbedIE,
 )
diff --git a/yt_dlp/extractor/ant1newsgr.py b/yt_dlp/extractor/antenna.py
similarity index 72%
rename from yt_dlp/extractor/ant1newsgr.py
rename to yt_dlp/extractor/antenna.py
index 217e3acc4..c78717aa9 100644
--- a/yt_dlp/extractor/ant1newsgr.py
+++ b/yt_dlp/extractor/antenna.py
@@ -5,22 +5,26 @@
 from ..utils import (
     ExtractorError,
     determine_ext,
+    make_archive_id,
     scale_thumbnails_to_max_format_width,
 )
 
 
-class Ant1NewsGrBaseIE(InfoExtractor):
+class AntennaBaseIE(InfoExtractor):
     def _download_and_extract_api_data(self, video_id, netloc, cid=None):
-        url = f'{self.http_scheme()}//{netloc}{self._API_PATH}'
-        info = self._download_json(url, video_id, query={'cid': cid or video_id})
-        try:
-            source = info['url']
-        except KeyError:
-            raise ExtractorError('no source found for %s' % video_id)
-        formats, subs = (self._extract_m3u8_formats_and_subtitles(source, video_id, 'mp4')
-                         if determine_ext(source) == 'm3u8' else ([{'url': source}], {}))
+        info = self._download_json(f'{self.http_scheme()}//{netloc}{self._API_PATH}',
+                                   video_id, query={'cid': cid or video_id})
+        if not info.get('url'):
+            raise ExtractorError(f'No source found for {video_id}')
+
+        ext = determine_ext(info['url'])
+        if ext == 'm3u8':
+            formats, subs = self._extract_m3u8_formats_and_subtitles(info['url'], video_id, 'mp4')
+        else:
+            formats, subs = [{'url': info['url'], 'format_id': ext}], {}
+
         thumbnails = scale_thumbnails_to_max_format_width(
-            formats, [{'url': info['thumb']}], r'(?<=/imgHandler/)\d+')
+            formats, [{'url': info['thumb']}], r'(?<=/imgHandler/)\d+') if info.get('thumb') else []
         return {
             'id': video_id,
             'title': info.get('title'),
@@ -30,21 +34,31 @@ def _download_and_extract_api_data(self, video_id, netloc, cid=None):
         }
 
 
-class Ant1NewsGrWatchIE(Ant1NewsGrBaseIE):
-    IE_NAME = 'ant1newsgr:watch'
-    IE_DESC = 'ant1news.gr videos'
-    _VALID_URL = r'https?://(?P<netloc>(?:www\.)?ant1news\.gr)/watch/(?P<id>\d+)/'
+class AntennaGrWatchIE(AntennaBaseIE):
+    IE_NAME = 'antenna:watch'
+    IE_DESC = 'antenna.gr and ant1news.gr videos'
+    _VALID_URL = r'https?://(?P<netloc>(?:www\.)?(?:antenna|ant1news)\.gr)/watch/(?P<id>\d+)/'
     _API_PATH = '/templates/data/player'
 
     _TESTS = [{
         'url': 'https://www.ant1news.gr/watch/1506168/ant1-news-09112021-stis-18-45',
-        'md5': '95925e6b32106754235f2417e0d2dfab',
+        'md5': 'c472d9dd7cd233c63aff2ea42201cda6',
         'info_dict': {
             'id': '1506168',
             'ext': 'mp4',
             'title': 'md5:0ad00fa66ecf8aa233d26ab0dba7514a',
             'description': 'md5:18665af715a6dcfeac1d6153a44f16b0',
-            'thumbnail': 'https://ant1media.azureedge.net/imgHandler/640/26d46bf6-8158-4f02-b197-7096c714b2de.jpg',
+            'thumbnail': r're:https://ant1media\.azureedge\.net/imgHandler/\d+/26d46bf6-8158-4f02-b197-7096c714b2de\.jpg',
+        },
+    }, {
+        'url': 'https://www.antenna.gr/watch/1643812/oi-prodotes-epeisodio-01',
+        'md5': '8f6f7dd3b1dba4d835ba990e25f31243',
+        'info_dict': {
+            'id': '1643812',
+            'ext': 'mp4',
+            'format_id': 'mp4',
+            'title': 'ΟΙ ΠΡΟΔΟΤΕΣ – ΕΠΕΙΣΟΔΙΟ 01',
+            'thumbnail': r're:https://ant1media\.azureedge\.net/imgHandler/\d+/b3d63096-e72d-43c4-87a0-00d4363d242f\.jpg',
         },
     }]
 
@@ -52,11 +66,12 @@ def _real_extract(self, url):
         video_id, netloc = self._match_valid_url(url).group('id', 'netloc')
         webpage = self._download_webpage(url, video_id)
         info = self._download_and_extract_api_data(video_id, netloc)
-        info['description'] = self._og_search_description(webpage)
+        info['description'] = self._og_search_description(webpage, default=None)
+        info['_old_archive_ids'] = [make_archive_id('Ant1NewsGrWatch', video_id)],
         return info
 
 
-class Ant1NewsGrArticleIE(Ant1NewsGrBaseIE):
+class Ant1NewsGrArticleIE(AntennaBaseIE):
     IE_NAME = 'ant1newsgr:article'
     IE_DESC = 'ant1news.gr articles'
     _VALID_URL = r'https?://(?:www\.)?ant1news\.gr/[^/]+/article/(?P<id>\d+)/'
@@ -96,7 +111,7 @@ def _real_extract(self, url):
             video_kwargs={'url_transparent': True, 'timestamp': info.get('timestamp')})
 
 
-class Ant1NewsGrEmbedIE(Ant1NewsGrBaseIE):
+class Ant1NewsGrEmbedIE(AntennaBaseIE):
     IE_NAME = 'ant1newsgr:embed'
     IE_DESC = 'ant1news.gr embedded videos'
     _BASE_PLAYER_URL_RE = r'(?:https?:)?//(?:[a-zA-Z0-9\-]+\.)?(?:antenna|ant1news)\.gr/templates/pages/player'

From 4b3a6ef1b3e235ba9a45142830b6edb357c71696 Mon Sep 17 00:00:00 2001
From: Omar Atef <85079143+Yalab7@users.noreply.github.com>
Date: Tue, 29 Aug 2023 03:49:29 +0300
Subject: [PATCH 515/871] [ie/hungama] Overhaul extractors (#7757)

Closes #7754
Authored by: Yalab7, bashonly
---
 yt_dlp/extractor/hungama.py | 109 +++++++++++++++++++++++++++---------
 1 file changed, 82 insertions(+), 27 deletions(-)

diff --git a/yt_dlp/extractor/hungama.py b/yt_dlp/extractor/hungama.py
index 2e9939601..cdec36838 100644
--- a/yt_dlp/extractor/hungama.py
+++ b/yt_dlp/extractor/hungama.py
@@ -1,19 +1,32 @@
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    remove_end,
+    traverse_obj,
     try_get,
+    unified_timestamp,
+    url_or_none,
     urlencode_postdata,
 )
 
 
-class HungamaIE(InfoExtractor):
+class HungamaBaseIE(InfoExtractor):
+    def _call_api(self, path, content_id, fatal=False):
+        return traverse_obj(self._download_json(
+            f'https://cpage.api.hungama.com/v2/page/content/{content_id}/{path}/detail',
+            content_id, fatal=fatal, query={
+                'device': 'web',
+                'platform': 'a',
+                'storeId': '1',
+            }), ('data', {dict})) or {}
+
+
+class HungamaIE(HungamaBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:www\.)?hungama\.com/
+                        (?:www\.|un\.)?hungama\.com/
                         (?:
-                            (?:video|movie)/[^/]+/|
+                            (?:video|movie|short-film)/[^/]+/|
                             tv-show/(?:[^/]+/){2}\d+/episode/[^/]+/
                         )
                         (?P<id>\d+)
@@ -25,13 +38,28 @@ class HungamaIE(InfoExtractor):
             'id': '39349649',
             'ext': 'mp4',
             'title': 'Krishna Chants',
-            'description': 'Watch Krishna Chants video now. You can also watch other latest videos only at Hungama',
+            'description': ' ',
             'upload_date': '20180829',
             'duration': 264,
             'timestamp': 1535500800,
             'view_count': int,
-            'thumbnail': 'https://images.hungama.com/c/1/0dc/2ca/39349649/39349649_700x394.jpg',
-        }
+            'thumbnail': 'https://images1.hungama.com/tr:n-a_169_m/c/1/0dc/2ca/39349649/39349649_350x197.jpg?v=8',
+            'tags': 'count:6',
+        },
+    }, {
+        'url': 'https://un.hungama.com/short-film/adira/102524179/',
+        'md5': '2278463f5dc9db9054d0c02602d44666',
+        'info_dict': {
+            'id': '102524179',
+            'ext': 'mp4',
+            'title': 'Adira',
+            'description': 'md5:df20cd4d41eabb33634f06de1025a4b4',
+            'upload_date': '20230417',
+            'timestamp': 1681689600,
+            'view_count': int,
+            'thumbnail': 'https://images1.hungama.com/tr:n-a_23_m/c/1/197/ac9/102524179/102524179_350x525.jpg?v=1',
+            'tags': 'count:7',
+        },
     }, {
         'url': 'https://www.hungama.com/movie/kahaani-2/44129919/',
         'only_matching': True,
@@ -51,14 +79,19 @@ def _real_extract(self, url):
                 'c': 'common',
                 'm': 'get_video_mdn_url',
             })
-
         formats = self._extract_m3u8_formats(video_json['stream_url'], video_id, ext='mp4', m3u8_id='hls')
-
-        json_ld = self._search_json_ld(
-            self._download_webpage(url, video_id, fatal=False) or '', video_id, fatal=False)
+        metadata = self._call_api('movie', video_id)
 
         return {
-            **json_ld,
+            **traverse_obj(metadata, ('head', 'data', {
+                'title': ('title', {str}),
+                'description': ('misc', 'description', {str}),
+                'duration': ('duration', {int}),  # duration in JSON is incorrect if string
+                'timestamp': ('releasedate', {unified_timestamp}),
+                'view_count': ('misc', 'playcount', {int_or_none}),
+                'thumbnail': ('image', {url_or_none}),
+                'tags': ('misc', 'keywords', ..., {str}),
+            })),
             'id': video_id,
             'formats': formats,
             'subtitles': {
@@ -71,10 +104,10 @@ def _real_extract(self, url):
 
 
 class HungamaSongIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hungama\.com/song/[^/]+/(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.|un\.)?hungama\.com/song/[^/]+/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'https://www.hungama.com/song/kitni-haseen-zindagi/2931166/',
-        'md5': 'd4a6a05a394ad0453a9bea3ca00e6024',
+        'md5': '964f46828e8b250aa35e5fdcfdcac367',
         'info_dict': {
             'id': '2931166',
             'ext': 'mp3',
@@ -83,8 +116,22 @@ class HungamaSongIE(InfoExtractor):
             'artist': 'Lucky Ali',
             'album': None,
             'release_year': 2000,
-        }
-    }
+            'thumbnail': 'https://stat2.hungama.ind.in/assets/images/default_images/da-200x200.png',
+        },
+    }, {
+        'url': 'https://un.hungama.com/song/tum-kya-mile-from-rocky-aur-rani-kii-prem-kahaani/103553672',
+        'md5': '964f46828e8b250aa35e5fdcfdcac367',
+        'info_dict': {
+            'id': '103553672',
+            'ext': 'mp3',
+            'title': 'md5:5ebeb1e10771b634ce5f700ce68ae5f4',
+            'track': 'Tum Kya Mile (From "Rocky Aur Rani Kii Prem Kahaani")',
+            'artist': 'Pritam Chakraborty, Arijit Singh, Shreya Ghoshal, Amitabh Bhattacharya',
+            'album': 'Tum Kya Mile (From "Rocky Aur Rani Kii Prem Kahaani")',
+            'release_year': 2023,
+            'thumbnail': 'https://images.hungama.com/c/1/7c2/c7b/103553671/103553671_200x200.jpg',
+        },
+    }]
 
     def _real_extract(self, url):
         audio_id = self._match_id(url)
@@ -122,8 +169,8 @@ def _real_extract(self, url):
         }
 
 
-class HungamaAlbumPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hungama\.com/(?:playlists|album)/[^/]+/(?P<id>\d+)'
+class HungamaAlbumPlaylistIE(HungamaBaseIE):
+    _VALID_URL = r'https?://(?:www\.|un\.)?hungama\.com/(?P<path>playlists|album)/[^/]+/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.hungama.com/album/bhuj-the-pride-of-india/69481490/',
         'playlist_mincount': 7,
@@ -132,16 +179,24 @@ class HungamaAlbumPlaylistIE(InfoExtractor):
         },
     }, {
         'url': 'https://www.hungama.com/playlists/hindi-jan-to-june-2021/123063/',
-        'playlist_mincount': 50,
+        'playlist_mincount': 33,
         'info_dict': {
             'id': '123063',
         },
+    }, {
+        'url': 'https://un.hungama.com/album/what-jhumka-%3F-from-rocky-aur-rani-kii-prem-kahaani/103891805/',
+        'playlist_mincount': 1,
+        'info_dict': {
+            'id': '103891805',
+        },
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        ptrn = r'<meta[^>]+?property=[\"\']?music:song:url[\"\']?[^>]+?content=[\"\']?([^\"\']+)'
-        items = re.findall(ptrn, webpage)
-        entries = [self.url_result(item, ie=HungamaSongIE.ie_key()) for item in items]
-        return self.playlist_result(entries, video_id)
+        playlist_id, path = self._match_valid_url(url).group('id', 'path')
+        data = self._call_api(remove_end(path, 's'), playlist_id, fatal=True)
+
+        def entries():
+            for song_url in traverse_obj(data, ('body', 'rows', ..., 'data', 'misc', 'share', {url_or_none})):
+                yield self.url_result(song_url, HungamaSongIE)
+
+        return self.playlist_result(entries(), playlist_id)

From 099fb1b35cf835303306549f5113d1802d79c9c7 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Tue, 29 Aug 2023 08:06:02 -0500
Subject: [PATCH 516/871] Bugfix for b9f2bc2dbed2323734a0d18e65e1e2e23dc833d8

Authored by: bashonly
---
 yt_dlp/extractor/dropbox.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/dropbox.py b/yt_dlp/extractor/dropbox.py
index ec86d7ad2..bc2efce12 100644
--- a/yt_dlp/extractor/dropbox.py
+++ b/yt_dlp/extractor/dropbox.py
@@ -66,10 +66,10 @@ def _real_extract(self, url):
         for encoded in reversed(re.findall(r'registerStreamedPrefetch\s*\(\s*"[\w/+=]+"\s*,\s*"([\w/+=]+)"', webpage)):
             decoded = base64.b64decode(encoded).decode('utf-8', 'ignore')
             transcode_url = self._search_regex(
-                r'\n\x03(https://[^\x12\x03\n]+\.m3u8)', decoded, 'transcode url', default=None)
+                r'\n.(https://[^\x03\x08\x12\n]+\.m3u8)', decoded, 'transcode url', default=None)
             if not transcode_url:
                 continue
-            formats, subtitles = self._extract_m3u8_formats_and_subtitles(transcode_url, video_id)
+            formats, subtitles = self._extract_m3u8_formats_and_subtitles(transcode_url, video_id, 'mp4')
             has_anonymous_download = self._search_regex(r'(anonymous:\tanonymous)', decoded, 'anonymous', default=False)
             break
 

From bae4834245a708fff97219849ec880c319c88bc6 Mon Sep 17 00:00:00 2001
From: RedDeffender <74822209+RedDeffender@users.noreply.github.com>
Date: Thu, 31 Aug 2023 01:26:45 +0200
Subject: [PATCH 517/871] [ie/NoodleMagazine] Fix extraction (#7830)

Closes #7917
Authored by: RedDeffender
---
 yt_dlp/extractor/noodlemagazine.py | 31 ++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/yt_dlp/extractor/noodlemagazine.py b/yt_dlp/extractor/noodlemagazine.py
index e6208956f..1cea0dbda 100644
--- a/yt_dlp/extractor/noodlemagazine.py
+++ b/yt_dlp/extractor/noodlemagazine.py
@@ -1,9 +1,14 @@
 from .common import InfoExtractor
 from ..utils import (
-    parse_duration,
+    extract_attributes,
+    get_element_html_by_id,
+    int_or_none,
     parse_count,
-    unified_strdate
+    parse_duration,
+    unified_strdate,
+    urljoin,
 )
+from ..utils.traversal import traverse_obj
 
 
 class NoodleMagazineIE(InfoExtractor):
@@ -37,15 +42,21 @@ def _real_extract(self, url):
         like_count = parse_count(self._html_search_meta('ya:ovs:likes', webpage, default=None))
         upload_date = unified_strdate(self._html_search_meta('ya:ovs:upload_date', webpage, default=''))
 
-        key = self._html_search_regex(rf'/{video_id}\?(?:.*&)?m=([^&"\'\s,]+)', webpage, 'key')
-        playlist_info = self._download_json(f'https://adult.noodlemagazine.com/playlist/{video_id}?m={key}', video_id)
-        thumbnail = self._og_search_property('image', webpage, default=None) or playlist_info.get('image')
+        player_path = extract_attributes(get_element_html_by_id('iplayer', webpage) or '')['src']
+        player_iframe = self._download_webpage(
+            urljoin('https://adult.noodlemagazine.com', player_path), video_id, 'Downloading iframe page')
+        playlist_url = self._search_regex(
+            r'window\.playlistUrl\s*=\s*["\']([^"\']+)["\']', player_iframe, 'playlist url')
+        playlist_info = self._download_json(
+            urljoin('https://adult.noodlemagazine.com', playlist_url), video_id, headers={'Referer': url})
 
-        formats = [{
-            'url': source.get('file'),
-            'quality': source.get('label'),
-            'ext': source.get('type'),
-        } for source in playlist_info.get('sources')]
+        thumbnail = self._og_search_property('image', webpage, default=None) or playlist_info.get('image')
+        formats = traverse_obj(playlist_info, ('sources', lambda _, v: v['file'], {
+            'url': 'file',
+            'format_id': 'label',
+            'height': ('label', {int_or_none}),
+            'ext': 'type',
+        }))
 
         return {
             'id': video_id,

From 630a55df8de7747e79aa680959d785dfff2c4b76 Mon Sep 17 00:00:00 2001
From: Grabien <60237587+Grabien@users.noreply.github.com>
Date: Thu, 31 Aug 2023 02:49:42 +0300
Subject: [PATCH 518/871] [ie/Mediaite] Fix extraction (#7923)

Authored by: Grabien
---
 yt_dlp/extractor/mediaite.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/mediaite.py b/yt_dlp/extractor/mediaite.py
index 0f9079b11..ab253920b 100644
--- a/yt_dlp/extractor/mediaite.py
+++ b/yt_dlp/extractor/mediaite.py
@@ -81,10 +81,24 @@ class MediaiteIE(InfoExtractor):
             'upload_date': '20210930',
         },
         'params': {'skip_download': True}
+    }, {
+        'url': 'https://www.mediaite.com/politics/i-cant-read-it-fast-enough-while-defending-trump-larry-kudlow-overwhelmed-by-volume-of-ex-presidents-legal-troubles/',
+        'info_dict': {
+            'id': 'E6EhDX5z',
+            'ext': 'mp4',
+            'title': 'Fox Business Network - 4:00 PM - 5:00 PM - 1:39:42 pm - 1:42:20 pm',
+            'description': '',
+            'thumbnail': 'https://cdn.jwplayer.com/v2/media/E6EhDX5z/poster.jpg?width=720',
+            'duration': 157,
+            'timestamp': 1691015535,
+            'upload_date': '20230802',
+        },
+        'params': {'skip_download': True}
     }]
 
     def _real_extract(self, url):
         webpage = self._download_webpage(url, None)
-        id = self._search_regex(r'data-video-id\s?=\s?\"([^\"]+)\"', webpage, 'id')
-        data_json = self._download_json(f'https://cdn.jwplayer.com/v2/media/{id}', id)
+        video_id = self._search_regex(
+            [r'"https://cdn\.jwplayer\.com/players/(\w+)', r'data-video-id\s*=\s*\"([^\"]+)\"'], webpage, 'id')
+        data_json = self._download_json(f'https://cdn.jwplayer.com/v2/media/{video_id}', video_id)
         return self._parse_jwplayer_data(data_json)

From 30ea88591b728cca0896018dbf67c2298070c669 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Thu, 31 Aug 2023 15:45:11 -0500
Subject: [PATCH 519/871] [ie/hotstar] Make metadata extraction non-fatal

Authored by: bashonly
---
 yt_dlp/extractor/hotstar.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/hotstar.py b/yt_dlp/extractor/hotstar.py
index cdd937941..6cadfb5b7 100644
--- a/yt_dlp/extractor/hotstar.py
+++ b/yt_dlp/extractor/hotstar.py
@@ -200,8 +200,10 @@ def _real_extract(self, url):
         video_type = self._TYPE.get(video_type, video_type)
         cookies = self._get_cookies(url)  # Cookies before any request
 
-        video_data = self._call_api_v1(f'{video_type}/detail', video_id,
-                                       query={'tas': 10000, 'contentId': video_id})['body']['results']['item']
+        video_data = traverse_obj(
+            self._call_api_v1(
+                f'{video_type}/detail', video_id, fatal=False, query={'tas': 10000, 'contentId': video_id}),
+            ('body', 'results', 'item', {dict})) or {}
         if not self.get_param('allow_unplayable_formats') and video_data.get('drmProtected'):
             self.report_drm(video_id)
 

From 7237c8dca0590aa7438ade93f927df88c9381ec7 Mon Sep 17 00:00:00 2001
From: Rajeshwaran <54212165+Rajeshwaran2001@users.noreply.github.com>
Date: Fri, 1 Sep 2023 02:18:52 +0530
Subject: [PATCH 520/871] [ie/hotstar] Extract `release_year` (#7869)

Authored by: Rajeshwaran2001
---
 yt_dlp/extractor/hotstar.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/yt_dlp/extractor/hotstar.py b/yt_dlp/extractor/hotstar.py
index 6cadfb5b7..541792b90 100644
--- a/yt_dlp/extractor/hotstar.py
+++ b/yt_dlp/extractor/hotstar.py
@@ -142,6 +142,26 @@ class HotStarIE(HotStarBaseIE):
             'duration': 1272,
             'channel_id': 3,
         },
+        'skip': 'HTTP Error 504: Gateway Time-out',  # XXX: Investigate 504 errors on some episodes
+    }, {
+        'url': 'https://www.hotstar.com/in/shows/kana-kaanum-kaalangal/1260097087/back-to-school/1260097320',
+        'info_dict': {
+            'id': '1260097320',
+            'ext': 'mp4',
+            'title': 'Back To School',
+            'season': 'Chapter 1',
+            'description': 'md5:b0d6a4c8a650681491e7405496fc7e13',
+            'timestamp': 1650564000,
+            'channel': 'Hotstar Specials',
+            'series': 'Kana Kaanum Kaalangal',
+            'season_number': 1,
+            'season_id': 9441,
+            'upload_date': '20220421',
+            'episode': 'Back To School',
+            'episode_number': 1,
+            'duration': 1810,
+            'channel_id': 54,
+        },
     }, {
         'url': 'https://www.hotstar.com/in/clips/e3-sairat-kahani-pyaar-ki/1000262286',
         'info_dict': {
@@ -154,6 +174,19 @@ class HotStarIE(HotStarBaseIE):
             'timestamp': 1622943900,
             'duration': 5395,
         },
+    }, {
+        'url': 'https://www.hotstar.com/in/movies/premam/1000091195',
+        'info_dict': {
+            'id': '1000091195',
+            'ext': 'mp4',
+            'title': 'Premam',
+            'release_year': 2015,
+            'description': 'md5:d833c654e4187b5e34757eafb5b72d7f',
+            'timestamp': 1462149000,
+            'upload_date': '20160502',
+            'episode': 'Premam',
+            'duration': 8994,
+        },
     }, {
         'url': 'https://www.hotstar.com/movies/radha-gopalam/1000057157',
         'only_matching': True,
@@ -288,6 +321,7 @@ def _real_extract(self, url):
             'description': video_data.get('description'),
             'duration': int_or_none(video_data.get('duration')),
             'timestamp': int_or_none(traverse_obj(video_data, 'broadcastDate', 'startDate')),
+            'release_year': int_or_none(video_data.get('year')),
             'formats': formats,
             'subtitles': subs,
             'channel': video_data.get('channelName'),

From 77bff23ee97565bab2e0d75b893a21bf7983219a Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 2 Sep 2023 15:18:04 +0200
Subject: [PATCH 521/871] Bugfix for 59e92b1f1833440bb2190f847eb735cf0f90bc85

Closes #8012

Authored by: Grub4K
---
 yt_dlp/networking/_urllib.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index 5a804d99b..b3e705b84 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -156,6 +156,8 @@ def brotli(data):
     def gz(data):
         # There may be junk added the end of the file
         # We ignore it by only ever decoding a single gzip payload
+        if not data:
+            return data
         return zlib.decompress(data, wbits=zlib.MAX_WBITS | 16)
 
     def http_request(self, req):

From 2301b5c1b77a65abbb46b72f91e1e4666fd5d985 Mon Sep 17 00:00:00 2001
From: Mattias Wadman <mattias.wadman@gmail.com>
Date: Sat, 2 Sep 2023 16:40:11 +0200
Subject: [PATCH 522/871] [ie/SVTPlay] Fix extraction (#7789)

Closes #5595
Authored by: wader, dirkf
---
 yt_dlp/extractor/svt.py | 61 +++++++++++++++++++++++++++++++++--------
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/extractor/svt.py b/yt_dlp/extractor/svt.py
index 31bf7f97e..18da87534 100644
--- a/yt_dlp/extractor/svt.py
+++ b/yt_dlp/extractor/svt.py
@@ -1,3 +1,4 @@
+import json
 import re
 
 from .common import InfoExtractor
@@ -6,10 +7,11 @@
     determine_ext,
     dict_get,
     int_or_none,
-    unified_timestamp,
     str_or_none,
     strip_or_none,
+    traverse_obj,
     try_get,
+    unified_timestamp,
 )
 
 
@@ -163,10 +165,46 @@ class SVTPlayIE(SVTPlayBaseIE):
             },
         },
         'params': {
-            # skip for now due to download test asserts that segment is > 10000 bytes and svt uses
-            # init segments that are smaller
-            # AssertionError: Expected test_SVTPlay_jNwpV9P.mp4 to be at least 9.77KiB, but it's only 864.00B
-            'skip_download': True,
+            'skip_download': 'm3u8',
+        },
+        'skip': 'Episode is no longer available',
+    }, {
+        'url': 'https://www.svtplay.se/video/emBxBQj',
+        'md5': '2382036fd6f8c994856c323fe51c426e',
+        'info_dict': {
+            'id': 'eyBd9aj',
+            'ext': 'mp4',
+            'title': '1. Farlig kryssning',
+            'timestamp': 1491019200,
+            'upload_date': '20170401',
+            'duration': 2566,
+            'thumbnail': r're:^https?://(?:.*[\.-]jpg|www.svtstatic.se/image/.*)$',
+            'age_limit': 0,
+            'episode': '1. Farlig kryssning',
+            'series': 'Rederiet',
+            'subtitles': {
+                'sv': 'count:3'
+            },
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }, {
+        'url': 'https://www.svtplay.se/video/jz2rYz7/anders-hansen-moter/james-fallon?info=visa',
+        'info_dict': {
+            'id': 'jvXAGVb',
+            'ext': 'mp4',
+            'title': 'James Fallon',
+            'timestamp': 1673917200,
+            'upload_date': '20230117',
+            'duration': 1081,
+            'thumbnail': r're:^https?://(?:.*[\.-]jpg|www.svtstatic.se/image/.*)$',
+            'age_limit': 0,
+            'episode': 'James Fallon',
+            'series': 'Anders Hansen möter...',
+        },
+        'params': {
+            'skip_download': 'dash',
         },
     }, {
         'url': 'https://www.svtplay.se/video/30479064/husdrommar/husdrommar-sasong-8-designdrommar-i-stenungsund?modalId=8zVbDPA',
@@ -247,15 +285,16 @@ def _real_extract(self, url):
                 data, lambda x: x['statistics']['dataLake']['content']['id'],
                 compat_str)
 
+        if not svt_id:
+            nextjs_data = self._search_nextjs_data(webpage, video_id, fatal=False)
+            svt_id = traverse_obj(nextjs_data, (
+                'props', 'urqlState', ..., 'data', {json.loads}, 'detailsPageByPath',
+                'video', 'svtId', {str}), get_all=False)
+
         if not svt_id:
             svt_id = self._search_regex(
                 (r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
-                 r'<[^>]+\bdata-rt=["\']top-area-play-button["\'][^>]+\bhref=["\'][^"\']*video/%s/[^"\']*\b(?:modalId|id)=([\da-zA-Z-]+)' % re.escape(video_id),
-                 r'["\']videoSvtId["\']\s*:\s*["\']([\da-zA-Z-]+)',
-                 r'["\']videoSvtId\\?["\']\s*:\s*\\?["\']([\da-zA-Z-]+)',
-                 r'"content"\s*:\s*{.*?"id"\s*:\s*"([\da-zA-Z-]+)"',
-                 r'["\']svtId["\']\s*:\s*["\']([\da-zA-Z-]+)',
-                 r'["\']svtId\\?["\']\s*:\s*\\?["\']([\da-zA-Z-]+)'),
+                 r'<[^>]+\bdata-rt=["\']top-area-play-button["\'][^>]+\bhref=["\'][^"\']*video/[\w-]+/[^"\']*\b(?:modalId|id)=([\w-]+)'),
                 webpage, 'video id')
 
         info_dict = self._extract_by_video_id(svt_id, webpage)

From 69dbfe01c47cd078682a87f179f5846e2679e927 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Mon, 4 Sep 2023 11:18:59 -0500
Subject: [PATCH 523/871] Bugfix for bae4834245a708fff97219849ec880c319c88bc6

Authored by: bashonly
---
 yt_dlp/extractor/noodlemagazine.py | 35 ++++++++++++++++++------------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/yt_dlp/extractor/noodlemagazine.py b/yt_dlp/extractor/noodlemagazine.py
index 1cea0dbda..1c1a763dc 100644
--- a/yt_dlp/extractor/noodlemagazine.py
+++ b/yt_dlp/extractor/noodlemagazine.py
@@ -1,7 +1,5 @@
 from .common import InfoExtractor
 from ..utils import (
-    extract_attributes,
-    get_element_html_by_id,
     int_or_none,
     parse_count,
     parse_duration,
@@ -42,27 +40,36 @@ def _real_extract(self, url):
         like_count = parse_count(self._html_search_meta('ya:ovs:likes', webpage, default=None))
         upload_date = unified_strdate(self._html_search_meta('ya:ovs:upload_date', webpage, default=''))
 
-        player_path = extract_attributes(get_element_html_by_id('iplayer', webpage) or '')['src']
+        def build_url(url_or_path):
+            return urljoin('https://adult.noodlemagazine.com', url_or_path)
+
+        headers = {'Referer': url}
+        player_path = self._html_search_regex(
+            r'<iframe[^>]+\bid="iplayer"[^>]+\bsrc="([^"]+)"', webpage, 'player path')
         player_iframe = self._download_webpage(
-            urljoin('https://adult.noodlemagazine.com', player_path), video_id, 'Downloading iframe page')
+            build_url(player_path), video_id, 'Downloading iframe page', headers=headers)
         playlist_url = self._search_regex(
             r'window\.playlistUrl\s*=\s*["\']([^"\']+)["\']', player_iframe, 'playlist url')
-        playlist_info = self._download_json(
-            urljoin('https://adult.noodlemagazine.com', playlist_url), video_id, headers={'Referer': url})
+        playlist_info = self._download_json(build_url(playlist_url), video_id, headers=headers)
 
-        thumbnail = self._og_search_property('image', webpage, default=None) or playlist_info.get('image')
-        formats = traverse_obj(playlist_info, ('sources', lambda _, v: v['file'], {
-            'url': 'file',
-            'format_id': 'label',
-            'height': ('label', {int_or_none}),
-            'ext': 'type',
-        }))
+        formats = []
+        for source in traverse_obj(playlist_info, ('sources', lambda _, v: v['file'])):
+            if source.get('type') == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    build_url(source['file']), video_id, 'mp4', fatal=False, m3u8_id='hls'))
+            else:
+                formats.append(traverse_obj(source, {
+                    'url': ('file', {build_url}),
+                    'format_id': 'label',
+                    'height': ('label', {int_or_none}),
+                    'ext': 'type',
+                }))
 
         return {
             'id': video_id,
             'formats': formats,
             'title': title,
-            'thumbnail': thumbnail,
+            'thumbnail': self._og_search_property('image', webpage, default=None) or playlist_info.get('image'),
             'duration': duration,
             'description': description,
             'tags': tags,

From c6ef553792ed48462f9fd0e78143bef6b1a71c2e Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Tue, 5 Sep 2023 01:54:14 -0500
Subject: [PATCH 524/871] [ie/twitter:spaces] Pass referer header to downloader

Closes #8029
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 34b8625c3..f86216f8f 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1618,6 +1618,7 @@ def _real_extract(self, url):
         is_live = live_status == 'is_live'
 
         formats = []
+        headers = {'Referer': 'https://twitter.com/'}
         if live_status == 'is_upcoming':
             self.raise_no_formats('Twitter Space not started yet', expected=True)
         elif not is_live and not metadata.get('is_space_available_for_replay'):
@@ -1628,7 +1629,7 @@ def _real_extract(self, url):
                 ('source', ('noRedirectPlaybackUrl', 'location'), {url_or_none}), get_all=False)
             formats = self._extract_m3u8_formats(  # XXX: Some Spaces need ffmpeg as downloader
                 source, metadata['media_key'], 'm4a', entry_protocol='m3u8', live=is_live,
-                headers={'Referer': 'https://twitter.com/'}, fatal=False) if source else []
+                headers=headers, fatal=False) if source else []
             for fmt in formats:
                 fmt.update({'vcodec': 'none', 'acodec': 'aac'})
                 if not is_live:
@@ -1653,6 +1654,7 @@ def _real_extract(self, url):
                 lambda: int_or_none(metadata['scheduled_start'], scale=1000)),
             'timestamp': int_or_none(metadata.get('created_at'), scale=1000),
             'formats': formats,
+            'http_headers': headers,
         }
 
 

From 99c99c7185f5d8e9b3699a6fc7f86ec663d7b97e Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Tue, 5 Sep 2023 14:58:02 -0500
Subject: [PATCH 525/871] [ie/gofile] Update token

Closes #7235
Authored by: bashonly
---
 yt_dlp/extractor/gofile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/gofile.py b/yt_dlp/extractor/gofile.py
index ddbce2ee8..898390583 100644
--- a/yt_dlp/extractor/gofile.py
+++ b/yt_dlp/extractor/gofile.py
@@ -66,7 +66,7 @@ def _entries(self, file_id):
         query_params = {
             'contentId': file_id,
             'token': self._TOKEN,
-            'websiteToken': 12345,
+            'websiteToken': '7fd94ds12fds4',  # From https://gofile.io/dist/js/alljs.js
         }
         password = self.get_param('videopassword')
         if password:

From d3d81cc98f554d0adb87d24bfd6fabaaa803944d Mon Sep 17 00:00:00 2001
From: ringus1 <ringus1@users.noreply.github.com>
Date: Tue, 5 Sep 2023 22:35:23 +0200
Subject: [PATCH 526/871] [ie/facebook] Fix webpage extraction (#7890)

Closes #7901
Authored by: ringus1
---
 yt_dlp/extractor/facebook.py | 41 +++++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 4fd17b574..c30a6b06a 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -74,6 +74,22 @@ class FacebookIE(InfoExtractor):
     _VIDEO_PAGE_TAHOE_TEMPLATE = 'https://www.facebook.com/video/tahoe/async/%s/?chain=true&isvideo=true&payloadtype=primary'
 
     _TESTS = [{
+        'url': 'https://www.facebook.com/radiokicksfm/videos/3676516585958356/',
+        'info_dict': {
+            'id': '3676516585958356',
+            'ext': 'mp4',
+            'title': 'dr Adam Przygoda',
+            'description': 'md5:34675bda53336b1d16400265c2bb9b3b',
+            'uploader': 'RADIO KICKS FM',
+            'upload_date': '20230818',
+            'timestamp': 1692346159,
+            'thumbnail': r're:^https?://.*',
+            'uploader_id': '100063551323670',
+            'duration': 3132.184,
+            'view_count': int,
+            'concurrent_view_count': 0,
+        },
+    }, {
         'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
         'md5': '6a40d33c0eccbb1af76cf0485a052659',
         'info_dict': {
@@ -97,7 +113,7 @@ class FacebookIE(InfoExtractor):
             'upload_date': '20140506',
             'timestamp': 1399398998,
             'thumbnail': r're:^https?://.*',
-            'uploader_id': 'pfbid04scW44U4P9iTyLZAGy8y8W3pR3i2VugvHCimiRudUAVbN3MPp9eXBaYFcgVworZwl',
+            'uploader_id': 'pfbid028wxorhX2ErLFJ578N6P3crHD3PHmXTCqCvfBpsnbSLmbokwSY75p5hWBjHGkG4zxl',
             'duration': 131.03,
             'concurrent_view_count': int,
         },
@@ -179,7 +195,7 @@ class FacebookIE(InfoExtractor):
             'timestamp': 1486648217,
             'upload_date': '20170209',
             'uploader': 'Yaroslav Korpan',
-            'uploader_id': 'pfbid029y8j22EwH3ikeqgH3SEP9G3CAi9kmWKgXJJG9s5geV7mo3J2bvURqHCdgucRgAyhl',
+            'uploader_id': 'pfbid06AScABAWcW91qpiuGrLt99Ef9tvwHoXP6t8KeFYEqkSfreMtfa9nTveh8b2ZEVSWl',
             'concurrent_view_count': int,
             'thumbnail': r're:^https?://.*',
             'view_count': int,
@@ -274,7 +290,7 @@ class FacebookIE(InfoExtractor):
             'title': 'Josef',
             'thumbnail': r're:^https?://.*',
             'concurrent_view_count': int,
-            'uploader_id': 'pfbid02gXHbDwxumkaKJQaTGUf3znYfYzTuidGEWawiramNx4YamSj2afwYSRkpcjtHtMRJl',
+            'uploader_id': 'pfbid0cibUN6tV7DYgdbJdsUFN46wc4jKpVSPAvJQhFofGqBGmVn3V3JtAs2tfUwziw2hUl',
             'timestamp': 1549275572,
             'duration': 3.413,
             'uploader': 'Josef Novak',
@@ -401,9 +417,9 @@ def _extract_from_url(self, url, video_id):
 
         def extract_metadata(webpage):
             post_data = [self._parse_json(j, video_id, fatal=False) for j in re.findall(
-                r'handleWithCustomApplyEach\(\s*ScheduledApplyEach\s*,\s*(\{.+?\})\s*\);', webpage)]
+                r'data-sjs>({.*?ScheduledServerJS.*?})</script>', webpage)]
             post = traverse_obj(post_data, (
-                ..., 'require', ..., ..., ..., '__bbox', 'result', 'data'), expected_type=dict) or []
+                ..., 'require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., '__bbox', 'result', 'data'), expected_type=dict) or []
             media = traverse_obj(post, (..., 'attachments', ..., lambda k, v: (
                 k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')), expected_type=dict)
             title = get_first(media, ('title', 'text'))
@@ -493,14 +509,14 @@ def process_formats(info):
 
         def extract_relay_data(_filter):
             return self._parse_json(self._search_regex(
-                r'handleWithCustomApplyEach\([^,]+,\s*({.*?%s.*?})\);' % _filter,
+                r'data-sjs>({.*?%s.*?})</script>' % _filter,
                 webpage, 'replay data', default='{}'), video_id, fatal=False) or {}
 
         def extract_relay_prefetched_data(_filter):
-            replay_data = extract_relay_data(_filter)
-            for require in (replay_data.get('require') or []):
-                if require[0] == 'RelayPrefetchedStreamCache':
-                    return try_get(require, lambda x: x[3][1]['__bbox']['result']['data'], dict) or {}
+            return traverse_obj(extract_relay_data(_filter), (
+                'require', (None, (..., ..., ..., '__bbox', 'require')),
+                lambda _, v: 'RelayPrefetchedStreamCache' in v, ..., ...,
+                '__bbox', 'result', 'data', {dict}), get_all=False) or {}
 
         if not video_data:
             server_js_data = self._parse_json(self._search_regex([
@@ -511,7 +527,7 @@ def extract_relay_prefetched_data(_filter):
 
         if not video_data:
             data = extract_relay_prefetched_data(
-                r'"(?:dash_manifest|playable_url(?:_quality_hd)?)"\s*:\s*"[^"]+"')
+                r'"(?:dash_manifest|playable_url(?:_quality_hd)?)')
             if data:
                 entries = []
 
@@ -526,7 +542,8 @@ def parse_graphql_video(video):
                     formats = []
                     q = qualities(['sd', 'hd'])
                     for key, format_id in (('playable_url', 'sd'), ('playable_url_quality_hd', 'hd'),
-                                           ('playable_url_dash', '')):
+                                           ('playable_url_dash', ''), ('browser_native_hd_url', 'hd'),
+                                           ('browser_native_sd_url', 'sd')):
                         playable_url = video.get(key)
                         if not playable_url:
                             continue

From fe371dcf0ba5ce8d42480eade54eeeac99ab3cb0 Mon Sep 17 00:00:00 2001
From: ifan-t <jacifan2000@gmail.com>
Date: Fri, 8 Sep 2023 13:25:43 +0100
Subject: [PATCH 527/871] [ie/S4C] Add series support and extract subs/thumbs
 (#7776)

Authored by: ifan-t
---
 yt_dlp/extractor/_extractors.py |  5 ++-
 yt_dlp/extractor/s4c.py         | 57 +++++++++++++++++++++++++++++----
 2 files changed, 54 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index f11554bdd..b788737a2 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1710,7 +1710,10 @@
     RuvIE,
     RuvSpilaIE
 )
-from .s4c import S4CIE
+from .s4c import (
+    S4CIE,
+    S4CSeriesIE
+)
 from .safari import (
     SafariIE,
     SafariApiIE,
diff --git a/yt_dlp/extractor/s4c.py b/yt_dlp/extractor/s4c.py
index 38a905896..990ea2b44 100644
--- a/yt_dlp/extractor/s4c.py
+++ b/yt_dlp/extractor/s4c.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..utils import traverse_obj
+from ..utils import traverse_obj, url_or_none
 
 
 class S4CIE(InfoExtractor):
@@ -11,7 +11,8 @@ class S4CIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Y Swn',
             'description': 'md5:f7681a30e4955b250b3224aa9fe70cf0',
-            'duration': 5340
+            'duration': 5340,
+            'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Y_Swn_2023S4C_099_ii.jpg'
         },
     }, {
         'url': 'https://www.s4c.cymru/clic/programme/856636948',
@@ -21,6 +22,7 @@ class S4CIE(InfoExtractor):
             'title': 'Am Dro',
             'duration': 2880,
             'description': 'md5:100d8686fc9a632a0cb2db52a3433ffe',
+            'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Am_Dro_2022-23S4C_P6_4005.jpg'
         },
     }]
 
@@ -30,7 +32,7 @@ def _real_extract(self, url):
             f'https://www.s4c.cymru/df/full_prog_details?lang=e&programme_id={video_id}',
             video_id, fatal=False)
 
-        filename = self._download_json(
+        player_config = self._download_json(
             'https://player-api.s4c-cdn.co.uk/player-configuration/prod', video_id, query={
                 'programme_id': video_id,
                 'signed': '0',
@@ -38,7 +40,13 @@ def _real_extract(self, url):
                 'mode': 'od',
                 'appId': 'clic',
                 'streamName': '',
-            }, note='Downloading player config JSON')['filename']
+            }, note='Downloading player config JSON')
+        subtitles = {}
+        for sub in traverse_obj(player_config, ('subtitles', lambda _, v: url_or_none(v['0']))):
+            subtitles.setdefault(sub.get('3', 'en'), []).append({
+                'url': sub['0'],
+                'name': sub.get('1'),
+            })
         m3u8_url = self._download_json(
             'https://player-api.s4c-cdn.co.uk/streaming-urls/prod', video_id, query={
                 'mode': 'od',
@@ -46,17 +54,52 @@ def _real_extract(self, url):
                 'region': 'WW',
                 'extra': 'false',
                 'thirdParty': 'false',
-                'filename': filename,
+                'filename': player_config['filename'],
             }, note='Downloading streaming urls JSON')['hls']
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
 
         return {
             'id': video_id,
-            'formats': formats,
+            'formats': self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls'),
             'subtitles': subtitles,
+            'thumbnail': url_or_none(player_config.get('poster')),
             **traverse_obj(details, ('full_prog_details', 0, {
                 'title': (('programme_title', 'series_title'), {str}),
                 'description': ('full_billing', {str.strip}),
                 'duration': ('duration', {lambda x: int(x) * 60}),
             }), get_all=False),
         }
+
+
+class S4CSeriesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?s4c\.cymru/clic/series/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.s4c.cymru/clic/series/864982911',
+        'playlist_mincount': 6,
+        'info_dict': {
+            'id': '864982911',
+            'title': 'Iaith ar Daith',
+            'description': 'md5:e878ebf660dce89bd2ef521d7ce06397'
+        },
+    }, {
+        'url': 'https://www.s4c.cymru/clic/series/866852587',
+        'playlist_mincount': 8,
+        'info_dict': {
+            'id': '866852587',
+            'title': 'FFIT Cymru',
+            'description': 'md5:abcb3c129cb68dbb6cd304fd33b07e96'
+        },
+    }]
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        series_details = self._download_json(
+            'https://www.s4c.cymru/df/series_details', series_id, query={
+                'lang': 'e',
+                'series_id': series_id,
+                'show_prog_in_series': 'Y'
+            }, note='Downloading series details JSON')
+
+        return self.playlist_result(
+            [self.url_result(f'https://www.s4c.cymru/clic/programme/{episode_id}', S4CIE, episode_id)
+             for episode_id in traverse_obj(series_details, ('other_progs_in_series', ..., 'id'))],
+            series_id, traverse_obj(series_details, ('full_prog_details', 0, 'series_title', {str})))

From 5d0395498d7065aa5e55bac85fa9354b4b0d48eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Szaby=20Gr=C3=BCnwald?= <szaby.gruenwald@web.de>
Date: Fri, 8 Sep 2023 14:54:41 +0200
Subject: [PATCH 528/871] [ie/wdr] Fix extraction (#7979)

Closes #7461
Authored by: szabyg
---
 yt_dlp/extractor/wdr.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/wdr.py b/yt_dlp/extractor/wdr.py
index de5dc2666..6767f2654 100644
--- a/yt_dlp/extractor/wdr.py
+++ b/yt_dlp/extractor/wdr.py
@@ -173,6 +173,7 @@ class WDRPageIE(WDRIE):  # XXX: Do not subclass from concrete IE
             'skip': 'HTTP Error 404: Not Found',
         },
         {
+            # FIXME: Asset JSON is directly embedded in webpage
             'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
             'info_dict': {
                 'id': 'mdb-2296252',
@@ -221,6 +222,8 @@ class WDRPageIE(WDRIE):  # XXX: Do not subclass from concrete IE
                 'id': 'mdb-869971',
                 'ext': 'mp4',
                 'title': r're:^COSMO Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'alt_title': 'COSMO Livestream',
+                'live_status': 'is_live',
                 'upload_date': '20160101',
             },
             'params': {
@@ -248,6 +251,16 @@ class WDRPageIE(WDRIE):  # XXX: Do not subclass from concrete IE
             'url': 'https://kinder.wdr.de/tv/die-sendung-mit-dem-elefanten/av/video-folge---astronaut-100.html',
             'only_matching': True,
         },
+        {
+            'url': 'https://www1.wdr.de/mediathek/video/sendungen/rockpalast/video-baroness---freak-valley-festival--100.html',
+            'info_dict': {
+                'id': 'mdb-2741028',
+                'ext': 'mp4',
+                'title': 'Baroness - Freak Valley Festival 2022',
+                'alt_title': 'Rockpalast',
+                'upload_date': '20220725',
+            },
+        }
     ]
 
     def _real_extract(self, url):
@@ -259,7 +272,7 @@ def _real_extract(self, url):
 
         # Article with several videos
 
-        # for wdr.de the data-extension is in a tag with the class "mediaLink"
+        # for wdr.de the data-extension-ard is in a tag with the class "mediaLink"
         # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
         # for wdrmaus, in a tag with the class "videoButton" (previously a link
         # to the page in a multiline "videoLink"-tag)
@@ -268,7 +281,7 @@ def _real_extract(self, url):
                     (?:
                         (["\'])(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b.*?\1[^>]+|
                         (["\'])videoLink\b.*?\2[\s]*>\n[^\n]*
-                    )data-extension=(["\'])(?P<data>(?:(?!\3).)+)\3
+                    )data-extension(?:-ard)?=(["\'])(?P<data>(?:(?!\3).)+)\3
                     ''', webpage):
             media_link_obj = self._parse_json(
                 mobj.group('data'), display_id, transform_source=js_to_json,
@@ -295,7 +308,7 @@ def _real_extract(self, url):
                     compat_urlparse.urljoin(url, mobj.group('href')),
                     ie=WDRPageIE.ie_key())
                 for mobj in re.finditer(
-                    r'<a[^>]+\bhref=(["\'])(?P<href>(?:(?!\1).)+)\1[^>]+\bdata-extension=',
+                    r'<a[^>]+\bhref=(["\'])(?P<href>(?:(?!\1).)+)\1[^>]+\bdata-extension(?:-ard)?=',
                     webpage) if re.match(self._PAGE_REGEX, mobj.group('href'))
             ]
 

From a006ce2b27357c15792eb5c18f06765e640b801c Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 9 Sep 2023 10:14:49 -0500
Subject: [PATCH 529/871] [ie/twitter] Fix retweet extraction and syndication
 API (#8016)

Authored by: bashonly
---
 README.md                   |   2 +-
 yt_dlp/extractor/twitter.py | 181 ++++++++++++++++++++++++++----------
 2 files changed, 132 insertions(+), 51 deletions(-)

diff --git a/README.md b/README.md
index b82d92a6e..c7b73f4fd 100644
--- a/README.md
+++ b/README.md
@@ -1854,7 +1854,7 @@ #### rokfinchannel
 * `tab`: Which tab to download - one of `new`, `top`, `videos`, `podcasts`, `streams`, `stacks`
 
 #### twitter
-* `legacy_api`: Force usage of the legacy Twitter API instead of the GraphQL API for tweet extraction. Has no effect if login cookies are passed
+* `api`: Select one of `graphql` (default), `legacy` or `syndication` as the API for tweet extraction. Has no effect if logged in
 
 #### stacommu, wrestleuniverse
 * `device_id`: UUID value assigned by the website and used to enforce device limits for paid livestream content. Can be found in browser local storage
diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index f86216f8f..4065acbaa 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1,9 +1,10 @@
-import functools
 import json
+import random
 import re
 
 from .common import InfoExtractor
 from .periscope import PeriscopeBaseIE, PeriscopeIE
+from ..compat import functools  # isort: split
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse_unquote,
@@ -147,10 +148,14 @@ def _search_dimensions_in_video_url(a_format, video_url):
     def is_logged_in(self):
         return bool(self._get_cookies(self._API_BASE).get('auth_token'))
 
+    @functools.cached_property
+    def _selected_api(self):
+        return self._configuration_arg('api', ['graphql'], ie_key='Twitter')[0]
+
     def _fetch_guest_token(self, display_id):
         guest_token = traverse_obj(self._download_json(
             f'{self._API_BASE}guest/activate.json', display_id, 'Downloading guest token', data=b'',
-            headers=self._set_base_headers(legacy=display_id and self._configuration_arg('legacy_api'))),
+            headers=self._set_base_headers(legacy=display_id and self._selected_api == 'legacy')),
             ('guest_token', {str}))
         if not guest_token:
             raise ExtractorError('Could not retrieve guest token')
@@ -295,7 +300,7 @@ def input_dict(subtask_id, text):
         self.report_login()
 
     def _call_api(self, path, video_id, query={}, graphql=False):
-        headers = self._set_base_headers(legacy=not graphql and self._configuration_arg('legacy_api'))
+        headers = self._set_base_headers(legacy=not graphql and self._selected_api == 'legacy')
         headers.update({
             'x-twitter-auth-type': 'OAuth2Session',
             'x-twitter-client-language': 'en',
@@ -707,6 +712,7 @@ class TwitterIE(TwitterBaseIE):
             'tags': [],
             'age_limit': 0,
         },
+        'skip': 'This Tweet is unavailable',
     }, {
         # not available in Periscope
         'url': 'https://twitter.com/ViviEducation/status/1136534865145286656',
@@ -721,6 +727,7 @@ class TwitterIE(TwitterBaseIE):
             'view_count': int,
         },
         'add_ie': ['TwitterBroadcast'],
+        'skip': 'Broadcast no longer exists',
     }, {
         # unified card
         'url': 'https://twitter.com/BrooklynNets/status/1349794411333394432?s=20',
@@ -773,9 +780,9 @@ class TwitterIE(TwitterBaseIE):
         'url': 'https://twitter.com/UltimaShadowX/status/1577719286659006464',
         'info_dict': {
             'id': '1577719286659006464',
-            'title': 'Ultima📛 | #вʟм - Test',
+            'title': 'Ultima📛| New Era - Test',
             'description': 'Test https://t.co/Y3KEZD7Dad',
-            'uploader': 'Ultima📛 | #вʟм',
+            'uploader': 'Ultima📛| New Era',
             'uploader_id': 'UltimaShadowX',
             'uploader_url': 'https://twitter.com/UltimaShadowX',
             'upload_date': '20221005',
@@ -811,7 +818,7 @@ class TwitterIE(TwitterBaseIE):
             'age_limit': 0,
         },
     }, {
-        # Adult content, fails if not logged in (GraphQL)
+        # Adult content, fails if not logged in
         'url': 'https://twitter.com/Rizdraws/status/1575199173472927762',
         'info_dict': {
             'id': '1575199163847000068',
@@ -831,9 +838,10 @@ class TwitterIE(TwitterBaseIE):
             'age_limit': 18,
             'tags': []
         },
+        'params': {'skip_download': 'The media could not be played'},
         'skip': 'Requires authentication',
     }, {
-        # Playlist result only with auth
+        # Playlist result only with graphql API
         'url': 'https://twitter.com/Srirachachau/status/1395079556562706435',
         'playlist_mincount': 2,
         'info_dict': {
@@ -898,7 +906,7 @@ class TwitterIE(TwitterBaseIE):
             'uploader_id': 'MoniqueCamarra',
             'live_status': 'was_live',
             'release_timestamp': 1658417414,
-            'description': 'md5:4dc8e972f1d8b3c6580376fabb02a3ad',
+            'description': 'md5:acce559345fd49f129c20dbcda3f1201',
             'timestamp': 1658407771,
             'release_date': '20220721',
             'upload_date': '20220721',
@@ -1007,10 +1015,10 @@ class TwitterIE(TwitterBaseIE):
             'view_count': int,
             'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1600009362759733248/pu/img/XVhFQivj75H_YxxV.jpg?name=orig',
             'age_limit': 0,
-            'uploader': 'Mün The Friend Of YWAP',
+            'uploader': 'Mün',
             'repost_count': int,
             'upload_date': '20221206',
-            'title': 'Mün The Friend Of YWAP - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
+            'title': 'Mün - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
             'comment_count': int,
             'like_count': int,
             'tags': [],
@@ -1019,7 +1027,7 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1670306984.0,
         },
     }, {
-        # url to retweet id w/ legacy api
+        # retweeted_status (private)
         'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
         'info_dict': {
             'id': '1623274794488659969',
@@ -1039,32 +1047,84 @@ class TwitterIE(TwitterBaseIE):
             'like_count': int,
             'repost_count': int,
         },
-        'params': {'extractor_args': {'twitter': {'legacy_api': ['']}}},
         'skip': 'Protected tweet',
     }, {
-        # orig tweet w/ graphql
-        'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
+        # retweeted_status
+        'url': 'https://twitter.com/playstrumpcard/status/1695424220702888009',
         'info_dict': {
-            'id': '1623274794488659969',
-            'display_id': '1623739803874349067',
+            'id': '1694928337846538240',
             'ext': 'mp4',
-            'title': '@selfisekai@hackerspace.pl 🐀 - RT @Johnnybull3ts: Me after going viral to over 30million people:    Whoopsie-daisy',
-            'description': 'md5:9258bdbb54793bdc124fe1cd47e96c6a',
-            'uploader': '@selfisekai@hackerspace.pl 🐀',
-            'uploader_id': 'liberdalau',
-            'uploader_url': 'https://twitter.com/liberdalau',
+            'display_id': '1695424220702888009',
+            'title': 'md5:e8daa9527bc2b947121395494f786d9d',
+            'description': 'md5:004f2d37fd58737724ec75bc7e679938',
+            'uploader': 'Benny Johnson',
+            'uploader_id': 'bennyjohnson',
+            'uploader_url': 'https://twitter.com/bennyjohnson',
             'age_limit': 0,
             'tags': [],
-            'duration': 8.033,
-            'timestamp': 1675964711.0,
-            'upload_date': '20230209',
-            'thumbnail': r're:https://pbs\.twimg\.com/ext_tw_video_thumb/.+',
+            'duration': 45.001,
+            'timestamp': 1692962814.0,
+            'upload_date': '20230825',
+            'thumbnail': r're:https://pbs\.twimg\.com/amplify_video_thumb/.+',
             'like_count': int,
-            'view_count': int,
             'repost_count': int,
+            'view_count': int,
             'comment_count': int,
         },
-        'skip': 'Protected tweet',
+    }, {
+        # retweeted_status w/ legacy API
+        'url': 'https://twitter.com/playstrumpcard/status/1695424220702888009',
+        'info_dict': {
+            'id': '1694928337846538240',
+            'ext': 'mp4',
+            'display_id': '1695424220702888009',
+            'title': 'md5:e8daa9527bc2b947121395494f786d9d',
+            'description': 'md5:004f2d37fd58737724ec75bc7e679938',
+            'uploader': 'Benny Johnson',
+            'uploader_id': 'bennyjohnson',
+            'uploader_url': 'https://twitter.com/bennyjohnson',
+            'age_limit': 0,
+            'tags': [],
+            'duration': 45.001,
+            'timestamp': 1692962814.0,
+            'upload_date': '20230825',
+            'thumbnail': r're:https://pbs\.twimg\.com/amplify_video_thumb/.+',
+            'like_count': int,
+            'repost_count': int,
+        },
+        'params': {'extractor_args': {'twitter': {'api': ['legacy']}}},
+    }, {
+        # Broadcast embedded in tweet
+        'url': 'https://twitter.com/JessicaDobsonWX/status/1693057346933600402',
+        'info_dict': {
+            'id': '1yNGaNLjEblJj',
+            'ext': 'mp4',
+            'title': 'Jessica Dobson - WAVE Weather Now - Saturday 8/19/23 Update',
+            'uploader': 'Jessica Dobson',
+            'uploader_id': '1DZEoDwDovRQa',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'view_count': int,
+        },
+        'add_ie': ['TwitterBroadcast'],
+    }, {
+        # Animated gif and quote tweet video, with syndication API
+        'url': 'https://twitter.com/BAKKOOONN/status/1696256659889565950',
+        'playlist_mincount': 2,
+        'info_dict': {
+            'id': '1696256659889565950',
+            'title': 'BAKOON - https://t.co/zom968d0a0',
+            'description': 'https://t.co/zom968d0a0',
+            'tags': [],
+            'uploader': 'BAKOON',
+            'uploader_id': 'BAKKOOONN',
+            'uploader_url': 'https://twitter.com/BAKKOOONN',
+            'age_limit': 18,
+            'timestamp': 1693254077.0,
+            'upload_date': '20230828',
+            'like_count': int,
+        },
+        'params': {'extractor_args': {'twitter': {'api': ['syndication']}}},
+        'expected_warnings': ['Not all metadata'],
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -1103,6 +1163,14 @@ class TwitterIE(TwitterBaseIE):
         'only_matching': True,
     }]
 
+    _MEDIA_ID_RE = re.compile(r'_video/(\d+)/')
+
+    @property
+    def _GRAPHQL_ENDPOINT(self):
+        if self.is_logged_in:
+            return 'zZXycP0V6H7m-2r0mOnFcA/TweetDetail'
+        return '2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId'
+
     def _graphql_to_legacy(self, data, twid):
         result = traverse_obj(data, (
             'threaded_conversation_with_injections_v2', 'instructions', 0, 'entries',
@@ -1130,9 +1198,14 @@ def _graphql_to_legacy(self, data, twid):
             'user': ('core', 'user_results', 'result', 'legacy'),
             'card': ('card', 'legacy'),
             'quoted_status': ('quoted_status_result', 'result', 'legacy'),
+            'retweeted_status': ('legacy', 'retweeted_status_result', 'result', 'legacy'),
         }, expected_type=dict, default={}))
 
-        # extra transformation is needed since result does not match legacy format
+        # extra transformations needed since result does not match legacy format
+        if status.get('retweeted_status'):
+            status['retweeted_status']['user'] = traverse_obj(status, (
+                'retweeted_status_result', 'result', 'core', 'user_results', 'result', 'legacy', {dict})) or {}
+
         binding_values = {
             binding_value.get('key'): binding_value.get('value')
             for binding_value in traverse_obj(status, ('card', 'binding_values', ..., {dict}))
@@ -1208,33 +1281,42 @@ def _build_graphql_query(self, media_id):
         }
 
     def _extract_status(self, twid):
-        if self.is_logged_in:
-            return self._graphql_to_legacy(
-                self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid), twid)
+        if self.is_logged_in or self._selected_api == 'graphql':
+            status = self._graphql_to_legacy(self._call_graphql_api(self._GRAPHQL_ENDPOINT, twid), twid)
 
-        try:
-            if not self._configuration_arg('legacy_api'):
-                return self._graphql_to_legacy(
-                    self._call_graphql_api('2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId', twid), twid)
-            return traverse_obj(self._call_api(f'statuses/show/{twid}.json', twid, {
+        elif self._selected_api == 'legacy':
+            status = self._call_api(f'statuses/show/{twid}.json', twid, {
                 'cards_platform': 'Web-12',
                 'include_cards': 1,
                 'include_reply_count': 1,
                 'include_user_entities': 0,
                 'tweet_mode': 'extended',
-            }), 'retweeted_status', None)
+            })
 
-        except ExtractorError as e:
-            if e.expected:
-                raise
+        elif self._selected_api == 'syndication':
             self.report_warning(
-                f'{e.orig_msg}. Falling back to syndication endpoint; some metadata may be missing', twid)
+                'Not all metadata or media is available via syndication endpoint', twid, only_once=True)
+            status = self._download_json(
+                'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
+                headers={'User-Agent': 'Googlebot'}, query={
+                    'id': twid,
+                    # TODO: token = ((Number(twid) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '')
+                    'token': ''.join(random.choices('123456789abcdefghijklmnopqrstuvwxyz', k=10)),
+                })
+            if not status:
+                raise ExtractorError('Syndication endpoint returned empty JSON response')
+            # Transform the result so its structure matches that of legacy/graphql
+            media = []
+            for detail in traverse_obj(status, ((None, 'quoted_tweet'), 'mediaDetails', ..., {dict})):
+                detail['id_str'] = traverse_obj(detail, (
+                    'video_info', 'variants', ..., 'url', {self._MEDIA_ID_RE.search}, 1), get_all=False) or twid
+                media.append(detail)
+            status['extended_entities'] = {'media': media}
 
-        status = self._download_json(
-            'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
-            headers={'User-Agent': 'Googlebot'}, query={'id': twid})
-        status['extended_entities'] = {'media': status.get('mediaDetails')}
-        return status
+        else:
+            raise ExtractorError(f'"{self._selected_api}" is not a valid API selection', expected=True)
+
+        return traverse_obj(status, 'retweeted_status', None, expected_type=dict) or {}
 
     def _real_extract(self, url):
         twid, selected_index = self._match_valid_url(url).group('id', 'index')
@@ -1266,10 +1348,7 @@ def _real_extract(self, url):
         }
 
         def extract_from_video_info(media):
-            media_id = traverse_obj(media, 'id_str', 'id', (
-                'video_info', 'variants', ..., 'url',
-                {functools.partial(re.search, r'_video/(\d+)/')}, 1
-            ), get_all=False, expected_type=str_or_none) or twid
+            media_id = traverse_obj(media, 'id_str', 'id', expected_type=str_or_none)
             self.write_debug(f'Extracting from video info: {media_id}')
 
             formats = []
@@ -1503,6 +1582,8 @@ def _real_extract(self, url):
         broadcast = self._call_api(
             'broadcasts/show.json', broadcast_id,
             {'ids': broadcast_id})['broadcasts'][broadcast_id]
+        if not broadcast:
+            raise ExtractorError('Broadcast no longer exists', expected=True)
         info = self._parse_broadcast_data(broadcast, broadcast_id)
         media_key = broadcast['media_key']
         source = self._call_api(

From 66cc64ff6696f9921ff112a278542f8d999ffea4 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Mon, 11 Sep 2023 09:51:39 -0500
Subject: [PATCH 530/871] [ie/zoom] Extract duration

Closes #8080
Authored by: bashonly
---
 yt_dlp/extractor/zoom.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/yt_dlp/extractor/zoom.py b/yt_dlp/extractor/zoom.py
index 3d7ccca76..1e41d0434 100644
--- a/yt_dlp/extractor/zoom.py
+++ b/yt_dlp/extractor/zoom.py
@@ -127,6 +127,7 @@ def _real_extract(self, url):
         return {
             'id': video_id,
             'title': str_or_none(traverse_obj(data, ('meet', 'topic'))),
+            'duration': int_or_none(data.get('duration')),
             'subtitles': subtitles,
             'formats': formats,
             'http_headers': {

From 7b71643cc986de9a3768dac4ac9b64f4d05e7f5e Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Fri, 15 Sep 2023 18:18:51 +0100
Subject: [PATCH 531/871] [ie/mixcloud] Update API URL (#8114)

Closes #8104
Authored by: garret1317
---
 yt_dlp/extractor/mixcloud.py | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/mixcloud.py b/yt_dlp/extractor/mixcloud.py
index fb5a08ca2..8a95d1a5d 100644
--- a/yt_dlp/extractor/mixcloud.py
+++ b/yt_dlp/extractor/mixcloud.py
@@ -20,7 +20,7 @@ class MixcloudBaseIE(InfoExtractor):
     def _call_api(self, object_type, object_fields, display_id, username, slug=None):
         lookup_key = object_type + 'Lookup'
         return self._download_json(
-            'https://www.mixcloud.com/graphql', display_id, query={
+            'https://app.mixcloud.com/graphql', display_id, query={
                 'query': '''{
   %s(lookup: {username: "%s"%s}) {
     %s
@@ -46,7 +46,15 @@ class MixcloudIE(MixcloudBaseIE):
             'view_count': int,
             'timestamp': 1321359578,
             'upload_date': '20111115',
+            'uploader_url': 'https://www.mixcloud.com/dholbach/',
+            'artist': 'Submorphics & Chino , Telekinesis, Porter Robinson, Enei, Breakage ft Jess Mills',
+            'duration': 3723,
+            'tags': [],
+            'comment_count': int,
+            'repost_count': int,
+            'like_count': int,
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
         'info_dict': {
@@ -60,7 +68,14 @@ class MixcloudIE(MixcloudBaseIE):
             'view_count': int,
             'timestamp': 1422987057,
             'upload_date': '20150203',
+            'uploader_url': 'https://www.mixcloud.com/gillespeterson/',
+            'duration': 2992,
+            'tags': [],
+            'comment_count': int,
+            'repost_count': int,
+            'like_count': int,
         },
+        'params': {'skip_download': '404 playback error on site'},
     }, {
         'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
         'only_matching': True,
@@ -259,9 +274,9 @@ def _real_extract(self, url):
                 cloudcast_url = cloudcast.get('url')
                 if not cloudcast_url:
                     continue
-                slug = try_get(cloudcast, lambda x: x['slug'], compat_str)
+                item_slug = try_get(cloudcast, lambda x: x['slug'], compat_str)
                 owner_username = try_get(cloudcast, lambda x: x['owner']['username'], compat_str)
-                video_id = '%s_%s' % (owner_username, slug) if slug and owner_username else None
+                video_id = f'{owner_username}_{item_slug}' if item_slug and owner_username else None
                 entries.append(self.url_result(
                     cloudcast_url, MixcloudIE.ie_key(), video_id))
 
@@ -284,7 +299,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_uploads',
             'title': 'Daniel Holbach (uploads)',
-            'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
+            'description': 'md5:a3f468a60ac8c3e1f8616380fc469b2b',
         },
         'playlist_mincount': 36,
     }, {
@@ -292,7 +307,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_uploads',
             'title': 'Daniel Holbach (uploads)',
-            'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
+            'description': 'md5:a3f468a60ac8c3e1f8616380fc469b2b',
         },
         'playlist_mincount': 36,
     }, {
@@ -300,7 +315,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_favorites',
             'title': 'Daniel Holbach (favorites)',
-            'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
+            'description': 'md5:a3f468a60ac8c3e1f8616380fc469b2b',
         },
         # 'params': {
         #     'playlist_items': '1-100',
@@ -323,9 +338,9 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'FirstEar_stream',
             'title': 'First Ear (stream)',
-            'description': 'Curators of good music\r\n\r\nfirstearmusic.com',
+            'description': 'we maraud for ears',
         },
-        'playlist_mincount': 271,
+        'playlist_mincount': 269,
     }]
 
     _TITLE_KEY = 'displayName'

From 497bbbbd7328cb705f70eced94dbd90993819a46 Mon Sep 17 00:00:00 2001
From: SevenLives <410355694@qq.com>
Date: Sat, 16 Sep 2023 17:37:04 +0800
Subject: [PATCH 532/871] [ie/abematv] Fix proxy handling (#8046)

Fixes https://github.com/yt-dlp/yt-dlp/issues/8036

Authored by: SevenLives
---
 yt_dlp/extractor/abematv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index 163b83c6d..2a093580c 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -12,7 +12,7 @@
 import urllib.request
 import urllib.response
 import uuid
-
+from ..utils.networking import clean_proxies
 from .common import InfoExtractor
 from ..aes import aes_ecb_decrypt
 from ..utils import (
@@ -35,7 +35,10 @@ def add_opener(ydl, handler):  # FIXME: Create proper API in .networking
     rh = ydl._request_director.handlers['Urllib']
     if 'abematv-license' in rh._SUPPORTED_URL_SCHEMES:
         return
-    opener = rh._get_instance(cookiejar=ydl.cookiejar, proxies=ydl.proxies)
+    headers = ydl.params['http_headers'].copy()
+    proxies = ydl.proxies.copy()
+    clean_proxies(proxies, headers)
+    opener = rh._get_instance(cookiejar=ydl.cookiejar, proxies=proxies)
     assert isinstance(opener, urllib.request.OpenerDirector)
     opener.add_handler(handler)
     rh._SUPPORTED_URL_SCHEMES = (*rh._SUPPORTED_URL_SCHEMES, 'abematv-license')

From 578a82e497502b951036ce9da6fe0dac6937ac27 Mon Sep 17 00:00:00 2001
From: Kshitiz Gupta <Kshitiz305@live.com>
Date: Sat, 16 Sep 2023 15:13:05 +0530
Subject: [PATCH 533/871] [ie/banbye] Support video ids containing a hyphen
 (#8059)

Fixes https://github.com/yt-dlp/yt-dlp/issues/7895

Authored by: kshitiz305
---
 yt_dlp/extractor/banbye.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/banbye.py b/yt_dlp/extractor/banbye.py
index c87342565..e0fc93b97 100644
--- a/yt_dlp/extractor/banbye.py
+++ b/yt_dlp/extractor/banbye.py
@@ -31,7 +31,7 @@ def _extract_playlist(self, playlist_id):
 
 
 class BanByeIE(BanByeBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?banbye.com/(?:en/)?watch/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?banbye.com/(?:en/)?watch/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://banbye.com/watch/v_ytfmvkVYLE8T',
         'md5': '2f4ea15c5ca259a73d909b2cfd558eb5',
@@ -59,7 +59,27 @@ class BanByeIE(BanByeBaseIE):
             'title': 'Krzysztof Karoń',
             'id': 'p_Ld82N6gBw_OJ',
         },
-        'playlist_count': 9,
+        'playlist_mincount': 9,
+    }, {
+        'url': 'https://banbye.com/watch/v_kb6_o1Kyq-CD',
+        'info_dict': {
+            'id': 'v_kb6_o1Kyq-CD',
+            'ext': 'mp4',
+            'title': 'Co tak naprawdę dzieje się we Francji?! Czy Warszawa a potem cała Polska będzie drugim Paryżem?!🤔🇵🇱',
+            'description': 'md5:82be4c0e13eae8ea1ca8b9f2e07226a8',
+            'uploader': 'Marcin Rola - MOIM ZDANIEM!🇵🇱',
+            'channel_id': 'ch_QgWnHvDG2fo5',
+            'channel_url': 'https://banbye.com/channel/ch_QgWnHvDG2fo5',
+            'duration': 597,
+            'timestamp': 1688642656,
+            'upload_date': '20230706',
+            'thumbnail': 'https://cdn.banbye.com/video/v_kb6_o1Kyq-CD/96.webp',
+            'tags': ['Paryż', 'Francja', 'Polska', 'Imigranci', 'Morawiecki', 'Tusk'],
+            'like_count': int,
+            'dislike_count': int,
+            'view_count': int,
+            'comment_count': int,
+        },
     }]
 
     def _real_extract(self, url):

From aee6b9b88c0bcccf27fd23b7e00fc0b7b168928f Mon Sep 17 00:00:00 2001
From: barsnick <barsnick@users.noreply.github.com>
Date: Sat, 16 Sep 2023 12:04:08 +0200
Subject: [PATCH 534/871] [ie/Axs] Add extractor (#8094)

Authored by: barsnick
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/axs.py         | 87 +++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)
 create mode 100644 yt_dlp/extractor/axs.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b788737a2..b836fe8a3 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -165,6 +165,7 @@
     AWAANLiveIE,
     AWAANSeasonIE,
 )
+from .axs import AxsIE
 from .azmedien import AZMedienIE
 from .baidu import BaiduVideoIE
 from .banbye import (
diff --git a/yt_dlp/extractor/axs.py b/yt_dlp/extractor/axs.py
new file mode 100644
index 000000000..4b263725f
--- /dev/null
+++ b/yt_dlp/extractor/axs.py
@@ -0,0 +1,87 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    js_to_json,
+    parse_iso8601,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class AxsIE(InfoExtractor):
+    IE_NAME = 'axs.tv'
+    _VALID_URL = r'https?://(?:www\.)?axs\.tv/(?:channel/(?:[^/?#]+/)+)?video/(?P<id>[^/?#]+)'
+
+    _TESTS = [{
+        'url': 'https://www.axs.tv/video/5f4dc776b70e4f1c194f22ef/',
+        'md5': '8d97736ae8e50c64df528e5e676778cf',
+        'info_dict': {
+            'id': '5f4dc776b70e4f1c194f22ef',
+            'title': 'Small Town',
+            'ext': 'mp4',
+            'description': 'md5:e314d28bfaa227a4d7ec965fae19997f',
+            'upload_date': '20230602',
+            'timestamp': 1685729564,
+            'duration': 1284.216,
+            'series': 'Rock & Roll Road Trip with Sammy Hagar',
+            'season': 2,
+            'episode': '3',
+            'thumbnail': 'https://images.dotstudiopro.com/5f4e9d330a0c3b295a7e8394',
+        },
+    }, {
+        'url': 'https://www.axs.tv/channel/rock-star-interview/video/daryl-hall',
+        'md5': '300ae795cd8f9984652c0949734ffbdc',
+        'info_dict': {
+            'id': '5f488148b70e4f392572977c',
+            'display_id': 'daryl-hall',
+            'title': 'Daryl Hall',
+            'ext': 'mp4',
+            'description': 'md5:e54ecaa0f4b5683fc9259e9e4b196628',
+            'upload_date': '20230214',
+            'timestamp': 1676403615,
+            'duration': 2570.668,
+            'series': 'The Big Interview with Dan Rather',
+            'season': 3,
+            'episode': '5',
+            'thumbnail': 'https://images.dotstudiopro.com/5f4d1901f340b50d937cec32',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        webpage_json_data = self._search_json(
+            r'mountObj\s*=', webpage, 'video ID data', display_id,
+            transform_source=js_to_json)
+        video_id = webpage_json_data['video_id']
+        company_id = webpage_json_data['company_id']
+
+        meta = self._download_json(
+            f'https://api.myspotlight.tv/dotplayer/video/{company_id}/{video_id}',
+            video_id, query={'device_type': 'desktop_web'})['video']
+
+        formats = self._extract_m3u8_formats(
+            meta['video_m3u8'], video_id, 'mp4', m3u8_id='hls')
+
+        subtitles = {}
+        for cc in traverse_obj(meta, ('closeCaption', lambda _, v: url_or_none(v['srtPath']))):
+            subtitles.setdefault(cc.get('srtShortLang') or 'en', []).append(
+                {'ext': cc.get('srtExt'), 'url': cc['srtPath']})
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'formats': formats,
+            **traverse_obj(meta, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'series': ('seriestitle', {str}),
+                'season': ('season', {int}),
+                'episode': ('episode', {str}),
+                'duration': ('duration', {float_or_none}),
+                'timestamp': ('updated_at', {parse_iso8601}),
+                'thumbnail': ('thumb', {url_or_none}),
+            }),
+            'subtitles': subtitles,
+        }

From 6e07e4bc7e59f5bdb60e93c011e57b18b009f2b5 Mon Sep 17 00:00:00 2001
From: zhallgato <zhallgato@gmail.com>
Date: Sat, 16 Sep 2023 12:12:18 +0200
Subject: [PATCH 535/871] [ie/mediaklikk] Fix extractor (#8086)

Fixes https://github.com/yt-dlp/yt-dlp/issues/8053

Authored by: bashonly, zhallgato
---
 yt_dlp/extractor/mediaklikk.py | 72 ++++++++++++++++++++++++++++++----
 1 file changed, 64 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/mediaklikk.py b/yt_dlp/extractor/mediaklikk.py
index 46365081b..fcc4827b5 100644
--- a/yt_dlp/extractor/mediaklikk.py
+++ b/yt_dlp/extractor/mediaklikk.py
@@ -1,5 +1,8 @@
 from ..utils import (
-    unified_strdate
+    ExtractorError,
+    traverse_obj,
+    unified_strdate,
+    url_or_none,
 )
 from .common import InfoExtractor
 from ..compat import (
@@ -15,7 +18,7 @@ class MediaKlikkIE(InfoExtractor):
                         (?P<id>[^/#?_]+)'''
 
     _TESTS = [{
-        # mediaklikk. date in html.
+        # (old) mediaklikk. date in html.
         'url': 'https://mediaklikk.hu/video/hazajaro-delnyugat-bacska-a-duna-menten-palankatol-doroszloig/',
         'info_dict': {
             'id': '4754129',
@@ -23,9 +26,21 @@ class MediaKlikkIE(InfoExtractor):
             'ext': 'mp4',
             'upload_date': '20210901',
             'thumbnail': 'http://mediaklikk.hu/wp-content/uploads/sites/4/2014/02/hazajarouj_JO.jpg'
+        },
+        'skip': 'Webpage redirects to 404 page',
+    }, {
+        # mediaklikk. date in html.
+        'url': 'https://mediaklikk.hu/video/hazajaro-fabova-hegyseg-kishont-koronaja/',
+        'info_dict': {
+            'id': '6696133',
+            'title': 'Hazajáró, Fabova-hegység - Kishont koronája',
+            'display_id': 'hazajaro-fabova-hegyseg-kishont-koronaja',
+            'ext': 'mp4',
+            'upload_date': '20230903',
+            'thumbnail': 'https://mediaklikk.hu/wp-content/uploads/sites/4/2014/02/hazajarouj_JO.jpg'
         }
     }, {
-        # m4sport
+        # (old) m4sport
         'url': 'https://m4sport.hu/video/2021/08/30/gyemant-liga-parizs/',
         'info_dict': {
             'id': '4754999',
@@ -33,6 +48,18 @@ class MediaKlikkIE(InfoExtractor):
             'ext': 'mp4',
             'upload_date': '20210830',
             'thumbnail': 'http://m4sport.hu/wp-content/uploads/sites/4/2021/08/vlcsnap-2021-08-30-18h21m20s10-1024x576.jpg'
+        },
+        'skip': 'Webpage redirects to 404 page',
+    }, {
+        # m4sport
+        'url': 'https://m4sport.hu/sportkozvetitesek/video/2023/09/08/atletika-gyemant-liga-brusszel/',
+        'info_dict': {
+            'id': '6711136',
+            'title': 'Atlétika – Gyémánt Liga, Brüsszel',
+            'display_id': 'atletika-gyemant-liga-brusszel',
+            'ext': 'mp4',
+            'upload_date': '20230908',
+            'thumbnail': 'https://m4sport.hu/wp-content/uploads/sites/4/2023/09/vlcsnap-2023-09-08-22h43m18s691.jpg'
         }
     }, {
         # m4sport with *video/ url and no date
@@ -40,20 +67,33 @@ class MediaKlikkIE(InfoExtractor):
         'info_dict': {
             'id': '4492099',
             'title': 'Real Madrid - Chelsea 1-1',
+            'display_id': 'real-madrid-chelsea-1-1',
             'ext': 'mp4',
-            'thumbnail': 'http://m4sport.hu/wp-content/uploads/sites/4/2021/04/Sequence-01.Still001-1024x576.png'
+            'thumbnail': 'https://m4sport.hu/wp-content/uploads/sites/4/2021/04/Sequence-01.Still001-1024x576.png'
         }
     }, {
-        # hirado
+        # (old) hirado
         'url': 'https://hirado.hu/videok/felteteleket-szabott-a-fovaros/',
         'info_dict': {
             'id': '4760120',
             'title': 'Feltételeket szabott a főváros',
             'ext': 'mp4',
             'thumbnail': 'http://hirado.hu/wp-content/uploads/sites/4/2021/09/vlcsnap-2021-09-01-20h20m37s165.jpg'
+        },
+        'skip': 'Webpage redirects to video list page',
+    }, {
+        # hirado
+        'url': 'https://hirado.hu/belfold/video/2023/09/11/marad-az-eves-elszamolas-a-napelemekre-beruhazo-csaladoknal',
+        'info_dict': {
+            'id': '6716068',
+            'title': 'Marad az éves elszámolás a napelemekre beruházó családoknál',
+            'display_id': 'marad-az-eves-elszamolas-a-napelemekre-beruhazo-csaladoknal',
+            'ext': 'mp4',
+            'upload_date': '20230911',
+            'thumbnail': 'https://hirado.hu/wp-content/uploads/sites/4/2023/09/vlcsnap-2023-09-11-09h16m09s882.jpg'
         }
     }, {
-        # petofilive
+        # (old) petofilive
         'url': 'https://petofilive.hu/video/2021/06/07/tha-shudras-az-akusztikban/',
         'info_dict': {
             'id': '4571948',
@@ -61,6 +101,18 @@ class MediaKlikkIE(InfoExtractor):
             'ext': 'mp4',
             'upload_date': '20210607',
             'thumbnail': 'http://petofilive.hu/wp-content/uploads/sites/4/2021/06/vlcsnap-2021-06-07-22h14m23s915-1024x576.jpg'
+        },
+        'skip': 'Webpage redirects to empty page',
+    }, {
+        # petofilive
+        'url': 'https://petofilive.hu/video/2023/09/09/futball-fesztival-a-margitszigeten/',
+        'info_dict': {
+            'id': '6713233',
+            'title': 'Futball Fesztivál a Margitszigeten',
+            'display_id': 'futball-fesztival-a-margitszigeten',
+            'ext': 'mp4',
+            'upload_date': '20230909',
+            'thumbnail': 'https://petofilive.hu/wp-content/uploads/sites/4/2023/09/Clipboard11-2.jpg'
         }
     }]
 
@@ -84,8 +136,12 @@ def _real_extract(self, url):
 
         player_data['video'] = player_data.pop('token')
         player_page = self._download_webpage('https://player.mediaklikk.hu/playernew/player.php', video_id, query=player_data)
-        playlist_url = self._proto_relative_url(compat_urllib_parse_unquote(
-            self._html_search_regex(r'\"file\":\s*\"(\\?/\\?/.*playlist\.m3u8)\"', player_page, 'playlist_url')).replace('\\/', '/'))
+        player_json = self._search_json(
+            r'\bpl\.setup\s*\(', player_page, 'player json', video_id, end_pattern=r'\);')
+        playlist_url = traverse_obj(
+            player_json, ('playlist', lambda _, v: v['type'] == 'hls', 'file', {url_or_none}), get_all=False)
+        if not playlist_url:
+            raise ExtractorError('Unable to extract playlist url')
 
         formats = self._extract_wowza_formats(
             playlist_url, video_id, skip_protocols=['f4m', 'smil', 'dash'])

From 98eac0e6ba0e510ae7dfdfd249d42ee71fb272b1 Mon Sep 17 00:00:00 2001
From: hatsomatt <143712404+hatsomatt@users.noreply.github.com>
Date: Sat, 16 Sep 2023 16:02:37 +0200
Subject: [PATCH 536/871] [ie/videa] Fix extraction (#8003)

Closes #7427
Authored by: hatsomatt, aky-01

Co-authored-by: aky-01 <65510015+aky-01@users.noreply.github.com>
---
 yt_dlp/extractor/videa.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/videa.py b/yt_dlp/extractor/videa.py
index 59ae933b0..634d2edea 100644
--- a/yt_dlp/extractor/videa.py
+++ b/yt_dlp/extractor/videa.py
@@ -38,6 +38,7 @@ class VideaIE(InfoExtractor):
             'title': 'Az őrült kígyász 285 kígyót enged szabadon',
             'thumbnail': r're:^https?://.*',
             'duration': 21,
+            'age_limit': 0,
         },
     }, {
         'url': 'http://videa.hu/videok/origo/jarmuvek/supercars-elozes-jAHDWfWSJH5XuFhH',
@@ -48,6 +49,7 @@ class VideaIE(InfoExtractor):
             'title': 'Supercars előzés',
             'thumbnail': r're:^https?://.*',
             'duration': 64,
+            'age_limit': 0,
         },
     }, {
         'url': 'http://videa.hu/player?v=8YfIAjxwWGwT8HVQ',
@@ -58,6 +60,7 @@ class VideaIE(InfoExtractor):
             'title': 'Az őrült kígyász 285 kígyót enged szabadon',
             'thumbnail': r're:^https?://.*',
             'duration': 21,
+            'age_limit': 0,
         },
     }, {
         'url': 'http://videa.hu/player/v/8YfIAjxwWGwT8HVQ?autoplay=1',
@@ -124,7 +127,7 @@ def _real_extract(self, url):
         query['_t'] = result[:16]
 
         b64_info, handle = self._download_webpage_handle(
-            'http://videa.hu/videaplayer_get_xml.php', video_id, query=query)
+            'http://videa.hu/player/xml', video_id, query=query)
         if b64_info.startswith('<?xml'):
             info = self._parse_xml(b64_info, video_id)
         else:

From 7d3d658f4c558ee7d72b1c01b46f2126948681cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Duval?= <jerome.duval@gmail.com>
Date: Sat, 16 Sep 2023 16:24:11 +0200
Subject: [PATCH 537/871] [ie/TV5MondePlus] Fix extractor (#7952)

Closes #4978
Authored by: korli, dirkf
---
 yt_dlp/extractor/tv5mondeplus.py | 98 ++++++++++++++++++++++++++------
 1 file changed, 80 insertions(+), 18 deletions(-)

diff --git a/yt_dlp/extractor/tv5mondeplus.py b/yt_dlp/extractor/tv5mondeplus.py
index bd0be784d..4da1b26d1 100644
--- a/yt_dlp/extractor/tv5mondeplus.py
+++ b/yt_dlp/extractor/tv5mondeplus.py
@@ -1,10 +1,14 @@
+import urllib.parse
+
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     extract_attributes,
     int_or_none,
     parse_duration,
+    traverse_obj,
     try_get,
+    url_or_none,
 )
 
 
@@ -12,6 +16,36 @@ class TV5MondePlusIE(InfoExtractor):
     IE_DESC = 'TV5MONDE+'
     _VALID_URL = r'https?://(?:www\.)?(?:tv5mondeplus|revoir\.tv5monde)\.com/toutes-les-videos/[^/]+/(?P<id>[^/?#]+)'
     _TESTS = [{
+        # movie
+        'url': 'https://revoir.tv5monde.com/toutes-les-videos/cinema/les-novices',
+        'md5': 'c86f60bf8b75436455b1b205f9745955',
+        'info_dict': {
+            'id': 'ZX0ipMyFQq_6D4BA7b',
+            'display_id': 'les-novices',
+            'ext': 'mp4',
+            'title': 'Les novices',
+            'description': 'md5:2e7c33ba3ad48dabfcc2a956b88bde2b',
+            'upload_date': '20230821',
+            'thumbnail': 'https://revoir.tv5monde.com/uploads/media/video_thumbnail/0738/60/01e952b7ccf36b7c6007ec9131588954ab651de9.jpeg',
+            'duration': 5177,
+            'episode': 'Les novices',
+        },
+    }, {
+        # series episode
+        'url': 'https://revoir.tv5monde.com/toutes-les-videos/series-fictions/opj-les-dents-de-la-terre-2',
+        'info_dict': {
+            'id': 'wJ0eeEPozr_6D4BA7b',
+            'display_id': 'opj-les-dents-de-la-terre-2',
+            'ext': 'mp4',
+            'title': "OPJ - Les dents de la Terre (2)",
+            'description': 'md5:288f87fd68d993f814e66e60e5302d9d',
+            'upload_date': '20230823',
+            'series': 'OPJ',
+            'episode': 'Les dents de la Terre (2)',
+            'duration': 2877,
+            'thumbnail': 'https://dl-revoir.tv5monde.com/images/1a/5753448.jpg'
+        },
+    }, {
         # movie
         'url': 'https://revoir.tv5monde.com/toutes-les-videos/cinema/ceux-qui-travaillent',
         'md5': '32fa0cde16a4480d1251502a66856d5f',
@@ -23,6 +57,7 @@ class TV5MondePlusIE(InfoExtractor):
             'description': 'md5:570e8bb688036ace873b2d50d24c026d',
             'upload_date': '20210819',
         },
+        'skip': 'no longer available',
     }, {
         # series episode
         'url': 'https://revoir.tv5monde.com/toutes-les-videos/series-fictions/vestiaires-caro-actrice',
@@ -39,6 +74,7 @@ class TV5MondePlusIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'no longer available',
     }, {
         'url': 'https://revoir.tv5monde.com/toutes-les-videos/series-fictions/neuf-jours-en-hiver-neuf-jours-en-hiver',
         'only_matching': True,
@@ -63,20 +99,45 @@ def _real_extract(self, url):
         video_files = self._parse_json(
             vpl_data['data-broadcast'], display_id)
         formats = []
-        for video_file in video_files:
-            v_url = video_file.get('url')
-            if not v_url:
-                continue
-            video_format = video_file.get('format') or determine_ext(v_url)
-            if video_format == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    v_url, display_id, 'mp4', 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-            else:
-                formats.append({
-                    'url': v_url,
-                    'format_id': video_format,
-                })
+        video_id = None
+
+        def process_video_files(v):
+            nonlocal video_id
+            for video_file in v:
+                v_url = video_file.get('url')
+                if not v_url:
+                    continue
+                if video_file.get('type') == 'application/deferred':
+                    d_param = urllib.parse.quote(v_url)
+                    token = video_file.get('token')
+                    if not token:
+                        continue
+                    deferred_json = self._download_json(
+                        f'https://api.tv5monde.com/player/asset/{d_param}/resolve?condenseKS=true', display_id,
+                        note='Downloading deferred info', headers={'Authorization': f'Bearer {token}'}, fatal=False)
+                    v_url = traverse_obj(deferred_json, (0, 'url', {url_or_none}))
+                    if not v_url:
+                        continue
+                    # data-guid from the webpage isn't stable, use the material id from the json urls
+                    video_id = self._search_regex(
+                        r'materials/([\da-zA-Z]{10}_[\da-fA-F]{7})/', v_url, 'video id', default=None)
+                    process_video_files(deferred_json)
+
+                video_format = video_file.get('format') or determine_ext(v_url)
+                if video_format == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        v_url, display_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
+                elif video_format == 'mpd':
+                    formats.extend(self._extract_mpd_formats(
+                        v_url, display_id, fatal=False))
+                else:
+                    formats.append({
+                        'url': v_url,
+                        'format_id': video_format,
+                    })
+
+        process_video_files(video_files)
 
         metadata = self._parse_json(
             vpl_data['data-metadata'], display_id)
@@ -100,10 +161,11 @@ def _real_extract(self, url):
         if upload_date:
             upload_date = upload_date.replace('_', '')
 
-        video_id = self._search_regex(
-            (r'data-guid=["\']([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
-             r'id_contenu["\']\s:\s*(\d+)'), webpage, 'video id',
-            default=display_id)
+        if not video_id:
+            video_id = self._search_regex(
+                (r'data-guid=["\']([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+                 r'id_contenu["\']\s:\s*(\d+)'), webpage, 'video id',
+                default=display_id)
 
         return {
             'id': video_id,

From f659e6439444ac64305b5c80688cd82f59d2279c Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 16 Sep 2023 17:50:06 +0200
Subject: [PATCH 538/871] [ie/bpb] Overhaul extractor (#8119)

Authored by: Grub4K
---
 yt_dlp/extractor/bpb.py | 174 +++++++++++++++++++++++++++++++++-------
 yt_dlp/utils/_utils.py  |   1 +
 2 files changed, 145 insertions(+), 30 deletions(-)

diff --git a/yt_dlp/extractor/bpb.py b/yt_dlp/extractor/bpb.py
index f28e581b8..7fe089944 100644
--- a/yt_dlp/extractor/bpb.py
+++ b/yt_dlp/extractor/bpb.py
@@ -1,56 +1,170 @@
+import functools
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
+    extract_attributes,
+    get_element_text_and_html_by_tag,
+    get_elements_by_class,
+    join_nonempty,
     js_to_json,
-    determine_ext,
+    mimetype2ext,
+    unified_strdate,
+    url_or_none,
+    urljoin,
+    variadic,
 )
+from ..utils.traversal import traverse_obj
+
+
+def html_get_element(tag=None, cls=None):
+    assert tag or cls, 'One of tag or class is required'
+
+    if cls:
+        func = functools.partial(get_elements_by_class, cls, tag=tag)
+    else:
+        func = functools.partial(get_element_text_and_html_by_tag, tag)
+
+    def html_get_element_wrapper(html):
+        return variadic(func(html))[0]
+
+    return html_get_element_wrapper
 
 
 class BpbIE(InfoExtractor):
     IE_DESC = 'Bundeszentrale für politische Bildung'
-    _VALID_URL = r'https?://(?:www\.)?bpb\.de/mediathek/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.|m\.)?bpb\.de/(?:[^/?#]+/)*(?P<id>\d+)(?:[/?#]|$)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
-        'md5': 'c4f84c8a8044ca9ff68bb8441d300b3f',
         'info_dict': {
             'id': '297',
             'ext': 'mp4',
+            'creator': 'Kooperative Berlin',
+            'description': 'md5:f4f75885ba009d3e2b156247a8941ce6',
+            'release_date': '20160115',
+            'series': 'Interview auf dem Geschichtsforum 1989 | 2009',
+            'tags': ['Friedliche Revolution', 'Erinnerungskultur', 'Vergangenheitspolitik', 'DDR 1949 - 1990', 'Freiheitsrecht', 'BStU', 'Deutschland'],
+            'thumbnail': 'https://www.bpb.de/cache/images/7/297_teaser_16x9_1240.jpg?8839D',
             'title': 'Joachim Gauck zu 1989 und die Erinnerung an die DDR',
-            'description': 'Joachim Gauck, erster Beauftragter für die Stasi-Unterlagen, spricht auf dem Geschichtsforum über die friedliche Revolution 1989 und eine "gewisse Traurigkeit" im Umgang mit der DDR-Vergangenheit.'
+            'uploader': 'Bundeszentrale für politische Bildung',
+        },
+    }, {
+        'url': 'https://www.bpb.de/mediathek/video/522184/krieg-flucht-und-falschmeldungen-wirstattdesinformation-2/',
+        'info_dict': {
+            'id': '522184',
+            'ext': 'mp4',
+            'creator': 'Institute for Strategic Dialogue Germany gGmbH (ISD)',
+            'description': 'md5:f83c795ff8f825a69456a9e51fc15903',
+            'release_date': '20230621',
+            'tags': ['Desinformation', 'Ukraine', 'Russland', 'Geflüchtete'],
+            'thumbnail': 'https://www.bpb.de/cache/images/4/522184_teaser_16x9_1240.png?EABFB',
+            'title': 'md5:9b01ccdbf58dbf9e5c9f6e771a803b1c',
+            'uploader': 'Bundeszentrale für politische Bildung',
+        },
+    }, {
+        'url': 'https://www.bpb.de/lernen/bewegtbild-und-politische-bildung/webvideo/518789/krieg-flucht-und-falschmeldungen-wirstattdesinformation-1/',
+        'info_dict': {
+            'id': '518789',
+            'ext': 'mp4',
+            'creator': 'Institute for Strategic Dialogue Germany gGmbH (ISD)',
+            'description': 'md5:85228aed433e84ff0ff9bc582abd4ea8',
+            'release_date': '20230302',
+            'tags': ['Desinformation', 'Ukraine', 'Russland', 'Geflüchtete'],
+            'thumbnail': 'https://www.bpb.de/cache/images/9/518789_teaser_16x9_1240.jpeg?56D0D',
+            'title': 'md5:3e956f264bb501f6383f10495a401da4',
+            'uploader': 'Bundeszentrale für politische Bildung',
+        },
+    }, {
+        'url': 'https://www.bpb.de/mediathek/podcasts/apuz-podcast/539727/apuz-20-china/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.bpb.de/mediathek/audio/315813/folge-1-eine-einfuehrung/',
+        'info_dict': {
+            'id': '315813',
+            'ext': 'mp3',
+            'creator': 'Axel Schröder',
+            'description': 'md5:eda9d1af34e5912efef5baf54fba4427',
+            'release_date': '20200921',
+            'series': 'Auf Endlagersuche. Der deutsche Weg zu einem sicheren Atommülllager',
+            'tags': ['Atomenergie', 'Endlager', 'hoch-radioaktiver Abfall', 'Endlagersuche', 'Atommüll', 'Atomendlager', 'Gorleben', 'Deutschland'],
+            'thumbnail': 'https://www.bpb.de/cache/images/3/315813_teaser_16x9_1240.png?92A94',
+            'title': 'Folge 1: Eine Einführung',
+            'uploader': 'Bundeszentrale für politische Bildung',
+        },
+    }, {
+        'url': 'https://www.bpb.de/517806/die-weltanschauung-der-neuen-rechten/',
+        'info_dict': {
+            'id': '517806',
+            'ext': 'mp3',
+            'creator': 'Bundeszentrale für politische Bildung',
+            'description': 'md5:594689600e919912aade0b2871cc3fed',
+            'release_date': '20230127',
+            'series': 'Vorträge des Fachtags "Modernisierer. Grenzgänger. Anstifter. Sechs Jahrzehnte \'Neue Rechte\'"',
+            'tags': ['Rechtsextremismus', 'Konservatismus', 'Konservativismus', 'neue Rechte', 'Rechtspopulismus', 'Schnellroda', 'Deutschland'],
+            'thumbnail': 'https://www.bpb.de/cache/images/6/517806_teaser_16x9_1240.png?7A7A0',
+            'title': 'Die Weltanschauung der "Neuen Rechten"',
+            'uploader': 'Bundeszentrale für politische Bildung',
+        },
+    }, {
+        'url': 'https://www.bpb.de/mediathek/reihen/zahlen-und-fakten-soziale-situation-filme/520153/zahlen-und-fakten-die-soziale-situation-in-deutschland-migration/',
+        'only_matching': True,
+    }]
+
+    _TITLE_RE = re.compile('(?P<title>[^<]*)<[^>]+>(?P<series>[^<]*)')
+
+    def _parse_vue_attributes(self, name, string, video_id):
+        attributes = extract_attributes(self._search_regex(rf'(<{name}(?:"[^"]*?"|[^>])*>)', string, name))
+
+        for key, value in attributes.items():
+            if key.startswith(':'):
+                attributes[key] = self._parse_json(value, video_id, transform_source=js_to_json, fatal=False)
+
+        return attributes
+
+    @staticmethod
+    def _process_source(source):
+        url = url_or_none(source['src'])
+        if not url:
+            return None
+
+        source_type = source.get('type', '')
+        extension = mimetype2ext(source_type)
+        is_video = source_type.startswith('video')
+        note = url.rpartition('.')[0].rpartition('_')[2] if is_video else None
+
+        return {
+            'url': url,
+            'ext': extension,
+            'vcodec': None if is_video else 'none',
+            'quality': 10 if note == 'high' else 0,
+            'format_note': note,
+            'format_id': join_nonempty(extension, note),
         }
-    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'<h2 class="white">(.*?)</h2>', webpage, 'title')
-        video_info_dicts = re.findall(
-            r"({\s*src\s*:\s*'https?://film\.bpb\.de/[^}]+})", webpage)
-
-        formats = []
-        for video_info in video_info_dicts:
-            video_info = self._parse_json(
-                video_info, video_id, transform_source=js_to_json, fatal=False)
-            if not video_info:
-                continue
-            video_url = video_info.get('src')
-            if not video_url:
-                continue
-            quality = 'high' if '_high' in video_url else 'low'
-            formats.append({
-                'url': video_url,
-                'quality': 10 if quality == 'high' else 0,
-                'format_note': quality,
-                'format_id': '%s-%s' % (quality, determine_ext(video_url)),
-            })
+        title_result = traverse_obj(webpage, ({html_get_element(cls='opening-header__title')}, {self._TITLE_RE.match}))
+        json_lds = list(self._yield_json_ld(webpage, video_id, fatal=False))
 
         return {
             'id': video_id,
-            'formats': formats,
-            'title': title,
-            'description': self._og_search_description(webpage),
+            'title': traverse_obj(title_result, ('title', {str.strip})) or None,
+            # This metadata could be interpreted otherwise, but it fits "series" the most
+            'series': traverse_obj(title_result, ('series', {str.strip})) or None,
+            'description': join_nonempty(*traverse_obj(webpage, [(
+                {html_get_element(cls='opening-intro')},
+                [{html_get_element(tag='bpb-accordion-item')}, {html_get_element(cls='text-content')}],
+            ), {clean_html}]), delim='\n\n') or None,
+            'creator': self._html_search_meta('author', webpage),
+            'uploader': self._html_search_meta('publisher', webpage),
+            'release_date': unified_strdate(self._html_search_meta('date', webpage)),
+            'tags': traverse_obj(json_lds, (..., 'keywords', {lambda x: x.split(',')}, ...)),
+            **traverse_obj(self._parse_vue_attributes('bpb-player', webpage, video_id), {
+                'formats': (':sources', ..., {self._process_source}),
+                'thumbnail': ('poster', {lambda x: urljoin(url, x)}),
+            }),
         }
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index f5552ce80..180bec245 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -2847,6 +2847,7 @@ def mimetype2ext(mt, default=NO_DEFAULT):
         'quicktime': 'mov',
         'webm': 'webm',
         'vp9': 'vp9',
+        'video/ogg': 'ogv',
         'x-flv': 'flv',
         'x-m4v': 'm4v',
         'x-matroska': 'mkv',

From 069cbece9dba6384f1cc5fcfc7ce562a31af42fc Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 16 Sep 2023 13:28:14 -0500
Subject: [PATCH 539/871] [ie/tiktok] Fix webpage extraction

Closes #8089
Authored by: bashonly
---
 yt_dlp/extractor/tiktok.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index f14c4f9d6..f26972cff 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -15,7 +15,6 @@
     UserNotLive,
     determine_ext,
     format_field,
-    get_element_by_id,
     get_first,
     int_or_none,
     join_nonempty,
@@ -50,8 +49,9 @@ def _create_url(user_id, video_id):
         return f'https://www.tiktok.com/@{user_id or "_"}/video/{video_id}'
 
     def _get_sigi_state(self, webpage, display_id):
-        return self._parse_json(get_element_by_id(
-            'SIGI_STATE|sigi-persisted-data', webpage, escape_value=False), display_id)
+        return self._search_json(
+            r'<script[^>]+\bid="(?:SIGI_STATE|sigi-persisted-data)"[^>]*>', webpage,
+            'sigi state', display_id, end_pattern=r'</script>')
 
     def _call_api_impl(self, ep, query, manifest_app_version, video_id, fatal=True,
                        note='Downloading API JSON', errnote='Unable to download API page'):

From cebbd33b1c678149fc8f0e254db6fc0da317ea80 Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Sat, 16 Sep 2023 16:43:12 -0400
Subject: [PATCH 540/871] [ie/twitcasting] Improve `_VALID_URL` (#8120)

Closes #7597
Authored by: c-basalt
---
 yt_dlp/extractor/twitcasting.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py
index dff353a4f..3890d5d8f 100644
--- a/yt_dlp/extractor/twitcasting.py
+++ b/yt_dlp/extractor/twitcasting.py
@@ -22,7 +22,7 @@
 
 
 class TwitCastingIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+\.)?twitcasting\.tv/(?P<uploader_id>[^/]+)/(?:movie|twplayer)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:[^/?#]+\.)?twitcasting\.tv/(?P<uploader_id>[^/?#]+)/(?:movie|twplayer)/(?P<id>\d+)'
     _M3U8_HEADERS = {
         'Origin': 'https://twitcasting.tv',
         'Referer': 'https://twitcasting.tv/',
@@ -231,7 +231,7 @@ def find_dmu(x):
 
 
 class TwitCastingLiveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+\.)?twitcasting\.tv/(?P<id>[^/]+)/?(?:[#?]|$)'
+    _VALID_URL = r'https?://(?:[^/?#]+\.)?twitcasting\.tv/(?P<id>[^/?#]+)/?(?:[#?]|$)'
     _TESTS = [{
         'url': 'https://twitcasting.tv/ivetesangalo',
         'only_matching': True,
@@ -265,8 +265,15 @@ def _real_extract(self, url):
 
 
 class TwitCastingUserIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+\.)?twitcasting\.tv/(?P<id>[^/]+)/show/?(?:[#?]|$)'
+    _VALID_URL = r'https?://(?:[^/?#]+\.)?twitcasting\.tv/(?P<id>[^/?#]+)/(:?show|archive)/?(?:[#?]|$)'
     _TESTS = [{
+        'url': 'https://twitcasting.tv/natsuiromatsuri/archive/',
+        'info_dict': {
+            'id': 'natsuiromatsuri',
+            'title': 'natsuiromatsuri - Live History',
+        },
+        'playlist_mincount': 235,
+    }, {
         'url': 'https://twitcasting.tv/noriyukicas/show',
         'only_matching': True,
     }]

From 9bf14be775289bd88cc1f5c89fd761ae51879484 Mon Sep 17 00:00:00 2001
From: makeworld <25111343+makew0rld@users.noreply.github.com>
Date: Sat, 16 Sep 2023 16:49:43 -0400
Subject: [PATCH 541/871] [ie/cbc] Ignore any 426 from API (#7689)

Closes #7477
Authored by: makew0rld
---
 yt_dlp/extractor/cbc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index b3c5471f7..2920b9027 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -339,12 +339,12 @@ def _new_claims_token(self, email, password):
         data = json.dumps({'jwt': sig}).encode()
         headers = {'content-type': 'application/json', 'ott-device-type': 'web'}
         resp = self._download_json('https://services.radio-canada.ca/ott/cbc-api/v2/token',
-                                   None, data=data, headers=headers)
+                                   None, data=data, headers=headers, expected_status=426)
         cbc_access_token = resp['accessToken']
 
         headers = {'content-type': 'application/json', 'ott-device-type': 'web', 'ott-access-token': cbc_access_token}
         resp = self._download_json('https://services.radio-canada.ca/ott/cbc-api/v2/profile',
-                                   None, headers=headers)
+                                   None, headers=headers, expected_status=426)
         return resp['claimsToken']
 
     def _get_claims_token_expiry(self):

From 5336bf57a7061e0955a37f0542fc8ebf50d55b17 Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Sat, 16 Sep 2023 16:53:57 -0400
Subject: [PATCH 542/871] [ie/bilibili] Extract `format_id` (#7555)

Authored by: c-basalt
---
 yt_dlp/extractor/bilibili.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index cb7ab2a17..290340078 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -3,6 +3,7 @@
 import hashlib
 import itertools
 import math
+import re
 import time
 import urllib.parse
 
@@ -38,6 +39,8 @@
 
 
 class BilibiliBaseIE(InfoExtractor):
+    _FORMAT_ID_RE = re.compile(r'-(\d+)\.m4s\?')
+
     def extract_formats(self, play_info):
         format_names = {
             r['quality']: traverse_obj(r, 'new_description', 'display_desc')
@@ -54,7 +57,8 @@ def extract_formats(self, play_info):
             'acodec': audio.get('codecs'),
             'vcodec': 'none',
             'tbr': float_or_none(audio.get('bandwidth'), scale=1000),
-            'filesize': int_or_none(audio.get('size'))
+            'filesize': int_or_none(audio.get('size')),
+            'format_id': str_or_none(audio.get('id')),
         } for audio in audios]
 
         formats.extend({
@@ -68,6 +72,9 @@ def extract_formats(self, play_info):
             'tbr': float_or_none(video.get('bandwidth'), scale=1000),
             'filesize': int_or_none(video.get('size')),
             'quality': int_or_none(video.get('id')),
+            'format_id': traverse_obj(
+                video, (('baseUrl', 'base_url'), {self._FORMAT_ID_RE.search}, 1),
+                ('id', {str_or_none}), get_all=False),
             'format': format_names.get(video.get('id')),
         } for video in traverse_obj(play_info, ('dash', 'video', ...)))
 

From 9d376c4daeaf1279a011582f3f0e6ae42af520dd Mon Sep 17 00:00:00 2001
From: Aniruddh Joshi <aniruddh@ebincoweb.com>
Date: Sun, 17 Sep 2023 02:28:21 +0530
Subject: [PATCH 543/871] [ie/AmazonMiniTV] Fix extractor (#8103)

Closes #7817
Authored by: Aniruddh-J
---
 yt_dlp/extractor/amazonminitv.py | 63 +++++---------------------------
 1 file changed, 9 insertions(+), 54 deletions(-)

diff --git a/yt_dlp/extractor/amazonminitv.py b/yt_dlp/extractor/amazonminitv.py
index b57d985d1..ad23b16bd 100644
--- a/yt_dlp/extractor/amazonminitv.py
+++ b/yt_dlp/extractor/amazonminitv.py
@@ -37,7 +37,7 @@ def _call_api(self, asin, data=None, note=None):
         return resp['data'][data['operationName']]
 
 
-class AmazonMiniTVIE(AmazonMiniTVBaseIE):
+class AmazonMiniTVIE(InfoExtractor):
     _VALID_URL = r'(?:https?://(?:www\.)?amazon\.in/minitv/tp/|amazonminitv:(?:amzn1\.dv\.gti\.)?)(?P<id>[a-f0-9-]+)'
     _TESTS = [{
         'url': 'https://www.amazon.in/minitv/tp/75fe3a75-b8fe-4499-8100-5c9424344840?referrer=https%3A%2F%2Fwww.amazon.in%2Fminitv',
@@ -86,56 +86,14 @@ class AmazonMiniTVIE(AmazonMiniTVBaseIE):
         'only_matching': True,
     }]
 
-    _GRAPHQL_QUERY_CONTENT = '''
-query content($sessionIdToken: String!, $deviceLocale: String, $contentId: ID!, $contentType: ContentType!, $clientId: String) {
-  content(
-    applicationContextInput: {deviceLocale: $deviceLocale, sessionIdToken: $sessionIdToken, clientId: $clientId}
-    contentId: $contentId
-    contentType: $contentType
-  ) {
-    contentId
-    name
-    ... on Episode {
-      contentId
-      vodType
-      name
-      images
-      description {
-        synopsis
-        contentLengthInSeconds
-      }
-      publicReleaseDateUTC
-      audioTracks
-      seasonId
-      seriesId
-      seriesName
-      seasonNumber
-      episodeNumber
-      timecode {
-        endCreditsTime
-      }
-    }
-    ... on MovieContent {
-      contentId
-      vodType
-      name
-      description {
-        synopsis
-        contentLengthInSeconds
-      }
-      images
-      publicReleaseDateUTC
-      audioTracks
-    }
-  }
-}'''
-
     def _real_extract(self, url):
-        asin = f'amzn1.dv.gti.{self._match_id(url)}'
-        prs = self._call_api(asin, note='Downloading playback info')
+        video_uuid = self._match_id(url)
+        asin = f'amzn1.dv.gti.{video_uuid}'
+        webpage = self._download_webpage(f'https://www.amazon.in/minitv/tp/{video_uuid}', asin)
+        data = self._search_nextjs_data(webpage, asin)['props']['pageProps']['ssrProps']
 
         formats, subtitles = [], {}
-        for type_, asset in prs['playbackAssets'].items():
+        for type_, asset in traverse_obj(data, ('playbackData', 'playbackAssets', {dict.items}, ...)):
             if not traverse_obj(asset, 'manifestUrl'):
                 continue
             if type_ == 'hls':
@@ -152,12 +110,7 @@ def _real_extract(self, url):
             else:
                 self.report_warning(f'Unknown asset type: {type_}')
 
-        title_info = self._call_api(
-            asin, note='Downloading title info', data={
-                'operationName': 'content',
-                'variables': {'contentId': asin},
-                'query': self._GRAPHQL_QUERY_CONTENT,
-            })
+        title_info = traverse_obj(data, ('contentData', {dict})) or {}
         credits_time = try_get(title_info, lambda x: x['timecode']['endCreditsTime'] / 1000)
         is_episode = title_info.get('vodType') == 'EPISODE'
 
@@ -192,6 +145,7 @@ class AmazonMiniTVSeasonIE(AmazonMiniTVBaseIE):
     IE_NAME = 'amazonminitv:season'
     _VALID_URL = r'amazonminitv:season:(?:amzn1\.dv\.gti\.)?(?P<id>[a-f0-9-]+)'
     IE_DESC = 'Amazon MiniTV Season, "minitv:season:" prefix'
+    _WORKING = False
     _TESTS = [{
         'url': 'amazonminitv:season:amzn1.dv.gti.0aa996eb-6a1b-4886-a342-387fbd2f1db0',
         'playlist_mincount': 6,
@@ -251,6 +205,7 @@ class AmazonMiniTVSeriesIE(AmazonMiniTVBaseIE):
     IE_NAME = 'amazonminitv:series'
     _VALID_URL = r'amazonminitv:series:(?:amzn1\.dv\.gti\.)?(?P<id>[a-f0-9-]+)'
     IE_DESC = 'Amazon MiniTV Series, "minitv:series:" prefix'
+    _WORKING = False
     _TESTS = [{
         'url': 'amazonminitv:series:amzn1.dv.gti.56521d46-b040-4fd5-872e-3e70476a04b0',
         'playlist_mincount': 3,

From a83da3717d30697102e76f63a6f29d77f9373c2a Mon Sep 17 00:00:00 2001
From: ApoorvShah111 <79164543+ApoorvShah111@users.noreply.github.com>
Date: Sun, 17 Sep 2023 02:31:26 +0530
Subject: [PATCH 544/871] [ie/nitter] Fix title extraction fallback (#8102)

Closes #7575
Authored by: ApoorvShah111
---
 yt_dlp/extractor/nitter.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/nitter.py b/yt_dlp/extractor/nitter.py
index 5d1ca1f5d..35d1311dc 100644
--- a/yt_dlp/extractor/nitter.py
+++ b/yt_dlp/extractor/nitter.py
@@ -265,6 +265,26 @@ class NitterIE(InfoExtractor):
                 'repost_count': int,
                 'comment_count': int,
             }
+        }, {  # no OpenGraph title
+            'url': f'https://{current_instance}/LocalBateman/status/1678455464038735895#m',
+            'info_dict': {
+                'id': '1678455464038735895',
+                'ext': 'mp4',
+                'title': 'Your Typical Local Man - Local man, what did Romanians ever do to you?',
+                'description': 'Local man, what did Romanians ever do to you?',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'uploader': 'Your Typical Local Man',
+                'uploader_id': 'LocalBateman',
+                'uploader_url': f'https://{current_instance}/LocalBateman',
+                'upload_date': '20230710',
+                'timestamp': 1689009900,
+                'view_count': int,
+                'like_count': int,
+                'repost_count': int,
+                'comment_count': int,
+            },
+            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+            'params': {'skip_download': 'm3u8'},
         }
     ]
 
@@ -292,7 +312,7 @@ def _real_extract(self, url):
                 'ext': ext
             }]
 
-        title = description = self._og_search_description(full_webpage) or self._html_search_regex(
+        title = description = self._og_search_description(full_webpage, default=None) or self._html_search_regex(
             r'<div class="tweet-content[^>]+>([^<]+)</div>', webpage, 'title', fatal=False)
 
         uploader_id = self._html_search_regex(

From ecef42c3adbcb6a84405139047923c4967316f28 Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Sun, 17 Sep 2023 05:04:10 +0800
Subject: [PATCH 545/871] [ie/zaiko] Improve thumbnail extraction (#8054)

Authored by: pzhlkj6612
---
 yt_dlp/extractor/zaiko.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/zaiko.py b/yt_dlp/extractor/zaiko.py
index 0ccacbb6a..2b6221da2 100644
--- a/yt_dlp/extractor/zaiko.py
+++ b/yt_dlp/extractor/zaiko.py
@@ -9,6 +9,7 @@
     traverse_obj,
     try_call,
     unescapeHTML,
+    url_basename,
     url_or_none,
 )
 
@@ -45,12 +46,14 @@ class ZaikoIE(ZaikoBaseIE):
             'uploader_id': '454',
             'uploader': 'ZAIKO ZERO',
             'release_timestamp': 1583809200,
-            'thumbnail': r're:https://[a-z0-9]+.cloudfront.net/[a-z0-9_]+/[a-z0-9_]+',
+            'thumbnail': r're:^https://[\w.-]+/\w+/\w+',
+            'thumbnails': 'maxcount:2',
             'release_date': '20200310',
             'categories': ['Tech House'],
             'live_status': 'was_live',
         },
         'params': {'skip_download': 'm3u8'},
+        'skip': 'Your account does not have tickets to this event',
     }]
 
     def _real_extract(self, url):
@@ -83,6 +86,12 @@ def _real_extract(self, url):
         if not formats:
             self.raise_no_formats(msg, expected=expected)
 
+        thumbnail_urls = [
+            traverse_obj(player_meta, ('initial_event_info', 'poster_url')),
+            self._og_search_thumbnail(self._download_webpage(
+                f'https://zaiko.io/event/{video_id}', video_id, 'Downloading event page', fatal=False) or ''),
+        ]
+
         return {
             'id': video_id,
             'formats': formats,
@@ -96,8 +105,8 @@ def _real_extract(self, url):
             }),
             **traverse_obj(player_meta, ('initial_event_info', {
                 'alt_title': ('title', {str}),
-                'thumbnail': ('poster_url', {url_or_none}),
             })),
+            'thumbnails': [{'url': url, 'id': url_basename(url)} for url in thumbnail_urls if url_or_none(url)]
         }
 
 

From 0ce1f48bf1cb78d40d734ce73ee1c90eccf92274 Mon Sep 17 00:00:00 2001
From: 04-pasha-04 <89145825+04-pasha-04@users.noreply.github.com>
Date: Sat, 16 Sep 2023 23:06:00 +0200
Subject: [PATCH 546/871] [ie/funker530] Fix extraction (#8040)

Authored by: 04-pasha-04
---
 yt_dlp/extractor/funker530.py | 1 +
 yt_dlp/extractor/rumble.py    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/funker530.py b/yt_dlp/extractor/funker530.py
index ba5ab7d4e..62fd7f6dd 100644
--- a/yt_dlp/extractor/funker530.py
+++ b/yt_dlp/extractor/funker530.py
@@ -60,6 +60,7 @@ class Funker530IE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
+        info = {}
         rumble_url = list(RumbleEmbedIE._extract_embed_urls(url, webpage))
         if rumble_url:
             info = {'url': rumble_url[0], 'ie_key': RumbleEmbedIE.ie_key()}
diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py
index f8bf4a182..96c192581 100644
--- a/yt_dlp/extractor/rumble.py
+++ b/yt_dlp/extractor/rumble.py
@@ -144,7 +144,7 @@ def _extract_embed_urls(cls, url, webpage):
         if embeds:
             return embeds
         return [f'https://rumble.com/embed/{mobj.group("id")}' for mobj in re.finditer(
-            r'<script>[^<]*\bRumble\(\s*"play"\s*,\s*{\s*[\'"]?video[\'"]?\s*:\s*[\'"](?P<id>[0-9a-z]+)[\'"]', webpage)]
+            r'<script>[^<]*\bRumble\(\s*"play"\s*,\s*{[^}]*[\'"]?video[\'"]?\s*:\s*[\'"](?P<id>[0-9a-z]+)[\'"]', webpage)]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 23d829a3420450bcfb0788e6fb2cf4f6acdbe596 Mon Sep 17 00:00:00 2001
From: Tristan Lee <lee.tristan.evans@gmail.com>
Date: Sat, 16 Sep 2023 16:08:15 -0500
Subject: [PATCH 547/871] [ie/Rumble] Fix embed extraction (#8035)

Authored by: trislee
---
 yt_dlp/extractor/rumble.py | 59 ++++++++++++++++++++++----------------
 1 file changed, 34 insertions(+), 25 deletions(-)

diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py
index 96c192581..85567d9a2 100644
--- a/yt_dlp/extractor/rumble.py
+++ b/yt_dlp/extractor/rumble.py
@@ -33,7 +33,7 @@ class RumbleEmbedIE(InfoExtractor):
             'upload_date': '20191020',
             'channel_url': 'https://rumble.com/c/WMAR',
             'channel': 'WMAR',
-            'thumbnail': 'https://sp.rmbl.ws/s8/1/5/M/z/1/5Mz1a.OvCc-small-WMAR-2-News-Latest-Headline.jpg',
+            'thumbnail': 'https://sp.rmbl.ws/s8/1/5/M/z/1/5Mz1a.qR4e-small-WMAR-2-News-Latest-Headline.jpg',
             'duration': 234,
             'uploader': 'WMAR',
             'live_status': 'not_live',
@@ -84,7 +84,7 @@ class RumbleEmbedIE(InfoExtractor):
         'info_dict': {
             'id': 'v1essrt',
             'ext': 'mp4',
-            'title': 'startswith:lofi hip hop radio - beats to relax/study',
+            'title': 'startswith:lofi hip hop radio 📚 - beats to relax/study to',
             'timestamp': 1661519399,
             'upload_date': '20220826',
             'channel_url': 'https://rumble.com/c/LofiGirl',
@@ -99,7 +99,7 @@ class RumbleEmbedIE(InfoExtractor):
         'url': 'https://rumble.com/embed/v1amumr',
         'info_dict': {
             'id': 'v1amumr',
-            'ext': 'webm',
+            'ext': 'mp4',
             'fps': 60,
             'title': 'Turning Point USA 2022 Student Action Summit DAY 1  - Rumble Exclusive Live',
             'timestamp': 1658518457,
@@ -129,7 +129,7 @@ class RumbleEmbedIE(InfoExtractor):
                 'duration': 92,
                 'title': '911 Audio From The Man Who Wanted To Kill Supreme Court Justice Kavanaugh',
                 'channel_url': 'https://rumble.com/c/RichSementa',
-                'thumbnail': 'https://sp.rmbl.ws/s8/1/P/j/f/A/PjfAe.OvCc-small-911-Audio-From-The-Man-Who-.jpg',
+                'thumbnail': 'https://sp.rmbl.ws/s8/1/P/j/f/A/PjfAe.qR4e-small-911-Audio-From-The-Man-Who-.jpg',
                 'timestamp': 1654892716,
                 'uploader': 'Mr Producer Media',
                 'upload_date': '20220610',
@@ -236,7 +236,9 @@ def _real_extract(self, url):
 
 class RumbleIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rumble\.com/(?P<id>v(?!ideos)[\w.-]+)[^/]*$'
-    _EMBED_REGEX = [r'<a class=video-item--a href=(?P<url>/v[\w.-]+\.html)>']
+    _EMBED_REGEX = [
+        r'<a class=video-item--a href=(?P<url>/v[\w.-]+\.html)>',
+        r'<a[^>]+class="videostream__link link"[^>]+href=(?P<url>/v[\w.-]+\.html)[^>]*>']
     _TESTS = [{
         'add_ie': ['RumbleEmbed'],
         'url': 'https://rumble.com/vdmum1-moose-the-dog-helps-girls-dig-a-snow-fort.html',
@@ -254,6 +256,7 @@ class RumbleIE(InfoExtractor):
             'thumbnail': r're:https://.+\.jpg',
             'duration': 103,
             'like_count': int,
+            'dislike_count': int,
             'view_count': int,
             'live_status': 'not_live',
         }
@@ -278,6 +281,9 @@ class RumbleIE(InfoExtractor):
             'channel_url': 'https://rumble.com/c/Redacted',
             'live_status': 'not_live',
             'thumbnail': 'https://sp.rmbl.ws/s8/1/d/x/2/O/dx2Oi.qR4e-small-The-U.S.-CANNOT-hide-this-i.jpg',
+            'like_count': int,
+            'dislike_count': int,
+            'view_count': int,
         },
     }, {
         'url': 'https://rumble.com/v2e7fju-the-covid-twitter-files-drop-protecting-fauci-while-censoring-the-truth-wma.html',
@@ -296,12 +302,15 @@ class RumbleIE(InfoExtractor):
             'channel_url': 'https://rumble.com/c/KimIversen',
             'channel': 'Kim Iversen',
             'thumbnail': 'https://sp.rmbl.ws/s8/1/6/b/w/O/6bwOi.qR4e-small-The-Covid-Twitter-Files-Dro.jpg',
+            'like_count': int,
+            'dislike_count': int,
+            'view_count': int,
         },
     }]
 
     _WEBPAGE_TESTS = [{
         'url': 'https://rumble.com/videos?page=2',
-        'playlist_count': 25,
+        'playlist_mincount': 24,
         'info_dict': {
             'id': 'videos?page=2',
             'title': 'All videos',
@@ -309,17 +318,16 @@ class RumbleIE(InfoExtractor):
             'age_limit': 0,
         },
     }, {
-        'url': 'https://rumble.com/live-videos',
-        'playlist_mincount': 19,
+        'url': 'https://rumble.com/browse/live',
+        'playlist_mincount': 25,
         'info_dict': {
-            'id': 'live-videos',
-            'title': 'Live Videos',
-            'description': 'Live videos on Rumble.com',
+            'id': 'live',
+            'title': 'Browse',
             'age_limit': 0,
         },
     }, {
         'url': 'https://rumble.com/search/video?q=rumble&sort=views',
-        'playlist_count': 24,
+        'playlist_mincount': 24,
         'info_dict': {
             'id': 'video?q=rumble&sort=views',
             'title': 'Search results for: rumble',
@@ -334,19 +342,20 @@ def _real_extract(self, url):
         if not url_info:
             raise UnsupportedError(url)
 
-        release_ts_str = self._search_regex(
-            r'(?:Livestream begins|Streamed on):\s+<time datetime="([^"]+)',
-            webpage, 'release date', fatal=False, default=None)
-        view_count_str = self._search_regex(r'<span class="media-heading-info">([\d,]+) Views',
-                                            webpage, 'view count', fatal=False, default=None)
-
-        return self.url_result(
-            url_info['url'], ie_key=url_info['ie_key'], url_transparent=True,
-            view_count=parse_count(view_count_str),
-            release_timestamp=parse_iso8601(release_ts_str),
-            like_count=parse_count(get_element_by_class('rumbles-count', webpage)),
-            description=clean_html(get_element_by_class('media-description', webpage)),
-        )
+        return {
+            '_type': 'url_transparent',
+            'ie_key': url_info['ie_key'],
+            'url': url_info['url'],
+            'release_timestamp': parse_iso8601(self._search_regex(
+                r'(?:Livestream begins|Streamed on):\s+<time datetime="([^"]+)', webpage, 'release date', default=None)),
+            'view_count': int_or_none(self._search_regex(
+                r'"userInteractionCount"\s*:\s*(\d+)', webpage, 'view count', default=None)),
+            'like_count': parse_count(self._search_regex(
+                r'<span data-js="rumbles_up_votes">\s*([\d,.KM]+)', webpage, 'like count', default=None)),
+            'dislike_count': parse_count(self._search_regex(
+                r'<span data-js="rumbles_down_votes">\s*([\d,.KM]+)', webpage, 'dislike count', default=None)),
+            'description': clean_html(get_element_by_class('media-description', webpage))
+        }
 
 
 class RumbleChannelIE(InfoExtractor):

From b4c1c408c63724339eb12b16c91b253a7ee62cfa Mon Sep 17 00:00:00 2001
From: barsnick <barsnick@users.noreply.github.com>
Date: Sat, 16 Sep 2023 23:11:05 +0200
Subject: [PATCH 548/871] [ie/Bild.de] Extract HLS formats (#8032)

Closes #7951
Authored by: barsnick
---
 yt_dlp/extractor/bild.py | 32 +++++++++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/bild.py b/yt_dlp/extractor/bild.py
index f3dea33c4..eb289329d 100644
--- a/yt_dlp/extractor/bild.py
+++ b/yt_dlp/extractor/bild.py
@@ -1,6 +1,7 @@
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    traverse_obj,
     unescapeHTML,
 )
 
@@ -8,7 +9,8 @@
 class BildIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bild\.de/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+)(?:,auto=true)?\.bild\.html'
     IE_DESC = 'Bild.de'
-    _TEST = {
+    _TESTS = [{
+        'note': 'static MP4 only',
         'url': 'http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html',
         'md5': 'dd495cbd99f2413502a1713a1156ac8a',
         'info_dict': {
@@ -19,7 +21,19 @@ class BildIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 196,
         }
-    }
+    }, {
+        'note': 'static MP4 and HLS',
+        'url': 'https://www.bild.de/video/clip/news-ausland/deftiger-abgang-vom-10m-turm-bademeister-sorgt-fuer-skandal-85158620.bild.html',
+        'md5': 'fb0ed4f09c495d4ba7ce2eee0bb90de1',
+        'info_dict': {
+            'id': '85158620',
+            'ext': 'mp4',
+            'title': 'Der Sprungturm-Skandal',
+            'description': 'md5:709b543c24dc31bbbffee73bccda34ad',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 69,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -27,11 +41,23 @@ def _real_extract(self, url):
         video_data = self._download_json(
             url.split('.bild.html')[0] + ',view=json.bild.html', video_id)
 
+        formats = []
+        for src in traverse_obj(video_data, ('clipList', 0, 'srces', lambda _, v: v['src'])):
+            src_type = src.get('type')
+            if src_type == 'application/x-mpegURL':
+                formats.extend(
+                    self._extract_m3u8_formats(
+                        src['src'], video_id, 'mp4', m3u8_id='hls', fatal=False))
+            elif src_type == 'video/mp4':
+                formats.append({'url': src['src'], 'format_id': 'http-mp4'})
+            else:
+                self.report_warning(f'Skipping unsupported format type: "{src_type}"')
+
         return {
             'id': video_id,
             'title': unescapeHTML(video_data['title']).strip(),
             'description': unescapeHTML(video_data.get('description')),
-            'url': video_data['clipList'][0]['srces'][0]['src'],
+            'formats': formats,
             'thumbnail': video_data.get('poster'),
             'duration': int_or_none(video_data.get('durationSec')),
         }

From 5be7e978867b5f66ad6786c674d79d40e950ae16 Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Sat, 16 Sep 2023 17:13:04 -0400
Subject: [PATCH 549/871] [ie/sohu] Fix extractor (#7628)

Closes #1667, Closes #7463
Authored by: c-basalt, bashonly
---
 yt_dlp/extractor/_extractors.py |   5 +-
 yt_dlp/extractor/sohu.py        | 107 ++++++++++++++++++++++++++++++--
 2 files changed, 105 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b836fe8a3..4fed6d66a 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1795,7 +1795,10 @@
 from .slutload import SlutloadIE
 from .smotrim import SmotrimIE
 from .snotr import SnotrIE
-from .sohu import SohuIE
+from .sohu import (
+    SohuIE,
+    SohuVIE,
+)
 from .sonyliv import (
     SonyLIVIE,
     SonyLIVSeriesIE,
diff --git a/yt_dlp/extractor/sohu.py b/yt_dlp/extractor/sohu.py
index a8f1e4623..c0ff4f9aa 100644
--- a/yt_dlp/extractor/sohu.py
+++ b/yt_dlp/extractor/sohu.py
@@ -1,3 +1,4 @@
+import base64
 import re
 
 from .common import InfoExtractor
@@ -8,7 +9,12 @@
 from ..utils import (
     ExtractorError,
     int_or_none,
+    float_or_none,
+    url_or_none,
+    unified_timestamp,
     try_get,
+    urljoin,
+    traverse_obj,
 )
 
 
@@ -31,13 +37,20 @@ class SohuIE(InfoExtractor):
             'id': '409385080',
             'ext': 'mp4',
             'title': '《2015湖南卫视羊年元宵晚会》唐嫣《花好月圆》',
-        }
+        },
+        'skip': 'no longer available',
     }, {
         'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
         'info_dict': {
             'id': '78693464',
             'ext': 'mp4',
             'title': '【爱范品】第31期：MWC见不到的奇葩手机',
+            'uploader': '爱范儿视频',
+            'duration': 213,
+            'timestamp': 1425519600,
+            'upload_date': '20150305',
+            'thumbnail': 'http://e3f49eaa46b57.cdn.sohucs.com//group1/M10/83/FA/MTAuMTAuODguODA=/6_14cbccdde5eg104SysCutcloud_78693464_7_0b.jpg',
+            'tags': ['爱范儿', '爱范品', 'MWC', '手机'],
         }
     }, {
         'note': 'Multipart video',
@@ -45,6 +58,12 @@ class SohuIE(InfoExtractor):
         'info_dict': {
             'id': '78910339',
             'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            'uploader': '小苍cany',
+            'duration': 744.0,
+            'timestamp': 1426269360,
+            'upload_date': '20150313',
+            'thumbnail': 'http://e3f49eaa46b57.cdn.sohucs.com//group1/M11/89/57/MTAuMTAuODguODA=/6_14cea022a1dg102SysCutcloud_78910339_8_0b.jpg',
+            'tags': ['小苍MM', '英雄联盟', '实战秘籍'],
         },
         'playlist': [{
             'info_dict': {
@@ -75,6 +94,11 @@ class SohuIE(InfoExtractor):
             'id': '78932792',
             'ext': 'mp4',
             'title': 'youtube-dl testing video',
+            'duration': 360,
+            'timestamp': 1426348620,
+            'upload_date': '20150314',
+            'thumbnail': 'http://e3f49eaa46b57.cdn.sohucs.com//group1/M02/8A/00/MTAuMTAuODguNzk=/6_14cee1be192g102SysCutcloud_78932792_7_7b.jpg',
+            'tags': [],
         },
         'params': {
             'skip_download': True
@@ -100,7 +124,7 @@ def _fetch_data(vid_id, mytv=False):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = re.sub(r' - 搜狐视频$', '', self._og_search_title(webpage))
+        title = re.sub(r'( - 高清正版在线观看)? - 搜狐视频$', '', self._og_search_title(webpage))
 
         vid = self._html_search_regex(
             r'var vid ?= ?["\'](\d+)["\']',
@@ -132,7 +156,9 @@ def _fetch_data(vid_id, mytv=False):
                 allot = format_data['allot']
 
                 data = format_data['data']
-                clips_url = data['clipsURL']
+                clip_url = traverse_obj(data, (('clipsURL', 'mp4PlayUrl'), i, {url_or_none}), get_all=False)
+                if not clip_url:
+                    raise ExtractorError(f'Unable to extract url for clip {i}')
                 su = data['su']
 
                 video_url = 'newflv.sohu.ccgslb.net'
@@ -142,9 +168,9 @@ def _fetch_data(vid_id, mytv=False):
                 while 'newflv.sohu.ccgslb.net' in video_url:
                     params = {
                         'prot': 9,
-                        'file': clips_url[i],
+                        'file': clip_url,
                         'new': su[i],
-                        'prod': 'flash',
+                        'prod': 'h5n',
                         'rb': 1,
                     }
 
@@ -193,6 +219,75 @@ def _fetch_data(vid_id, mytv=False):
                 'entries': playlist,
                 'id': video_id,
                 'title': title,
+                'duration': traverse_obj(vid_data, ('data', 'totalDuration', {float_or_none})),
             }
 
-        return info
+        if mytv:
+            publish_time = unified_timestamp(self._search_regex(
+                r'publishTime:\s*["\'](\d+-\d+-\d+ \d+:\d+)["\']', webpage, 'publish time', fatal=False))
+        else:
+            publish_time = traverse_obj(vid_data, ('tv_application_time', {unified_timestamp}))
+
+        return {
+            'timestamp': publish_time - 8 * 3600 if publish_time else None,
+            **traverse_obj(vid_data, {
+                'alt_title': ('data', 'subName', {str}),
+                'uploader': ('wm_data', 'wm_username', {str}),
+                'thumbnail': ('data', 'coverImg', {url_or_none}),
+                'tags': ('data', 'tag', {str.split}),
+            }),
+            **info,
+        }
+
+
+class SohuVIE(InfoExtractor):
+    _VALID_URL = r'https?://tv\.sohu\.com/v/(?P<id>[\w=-]+)\.html(?:$|[#?])'
+
+    _TESTS = [{
+        'note': 'Multipart video',
+        'url': 'https://tv.sohu.com/v/MjAyMzA2MTQvbjYwMTMxNTE5Mi5zaHRtbA==.html',
+        'info_dict': {
+            'id': '601315192',
+            'title': '《淬火丹心》第1集',
+            'alt_title': '“点天灯”发生事故',
+            'duration': 2701.692,
+            'timestamp': 1686758040,
+            'upload_date': '20230614',
+            'thumbnail': 'http://photocdn.tv.sohu.com/img/20230614/vrsa_hor_1686738763256_454010551.jpg',
+        },
+        'playlist_mincount': 9,
+        'skip': 'Only available in China',
+    }, {
+        'url': 'https://tv.sohu.com/v/dXMvMjMyNzk5ODg5Lzc4NjkzNDY0LnNodG1s.html',
+        'info_dict': {
+            'id': '78693464',
+            'ext': 'mp4',
+            'title': '【爱范品】第31期：MWC见不到的奇葩手机',
+            'uploader': '爱范儿视频',
+            'duration': 213,
+            'timestamp': 1425519600,
+            'upload_date': '20150305',
+            'thumbnail': 'http://e3f49eaa46b57.cdn.sohucs.com//group1/M10/83/FA/MTAuMTAuODguODA=/6_14cbccdde5eg104SysCutcloud_78693464_7_0b.jpg',
+            'tags': ['爱范儿', '爱范品', 'MWC', '手机'],
+        }
+    }, {
+        'note': 'Multipart video',
+        'url': 'https://tv.sohu.com/v/dXMvMjQyNTYyMTYzLzc4OTEwMzM5LnNodG1s.html?src=pl',
+        'info_dict': {
+            'id': '78910339',
+            'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            'uploader': '小苍cany',
+            'duration': 744.0,
+            'timestamp': 1426269360,
+            'upload_date': '20150313',
+            'thumbnail': 'http://e3f49eaa46b57.cdn.sohucs.com//group1/M11/89/57/MTAuMTAuODguODA=/6_14cea022a1dg102SysCutcloud_78910339_8_0b.jpg',
+            'tags': ['小苍MM', '英雄联盟', '实战秘籍'],
+        },
+        'playlist_mincount': 3,
+    }]
+
+    def _real_extract(self, url):
+        encoded_id = self._match_id(url)
+        path = base64.urlsafe_b64decode(encoded_id).decode()
+        subdomain = 'tv' if re.match(r'\d+/n\d+\.shtml', path) else 'my.tv'
+        return self.url_result(urljoin(f'http://{subdomain}.sohu.com/', path), SohuIE)

From 308936619c8a4f3a52d73c829c2006ff6c55fea2 Mon Sep 17 00:00:00 2001
From: fireattack <human.peng@gmail.com>
Date: Sun, 17 Sep 2023 05:18:04 +0800
Subject: [PATCH 550/871] [ie/facebook] Improve format sorting (#8074)

Authored by: fireattack
---
 yt_dlp/extractor/facebook.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index c30a6b06a..50a750d3b 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -505,7 +505,6 @@ def process_formats(info):
             # with non-browser User-Agent.
             for f in info['formats']:
                 f.setdefault('http_headers', {})['User-Agent'] = 'facebookexternalhit/1.1'
-            info['_format_sort_fields'] = ('res', 'quality')
 
         def extract_relay_data(_filter):
             return self._parse_json(self._search_regex(
@@ -552,7 +551,8 @@ def parse_graphql_video(video):
                         else:
                             formats.append({
                                 'format_id': format_id,
-                                'quality': q(format_id),
+                                # sd, hd formats w/o resolution info should be deprioritized below DASH
+                                'quality': q(format_id) - 3,
                                 'url': playable_url,
                             })
                     extract_dash_manifest(video, formats)
@@ -719,9 +719,11 @@ def parse_attachment(attachment, key='media'):
                 for src_type in ('src', 'src_no_ratelimit'):
                     src = f[0].get('%s_%s' % (quality, src_type))
                     if src:
-                        preference = -10 if format_id == 'progressive' else -1
+                        # sd, hd formats w/o resolution info should be deprioritized below DASH
+                        # TODO: investigate if progressive or src formats still exist
+                        preference = -10 if format_id == 'progressive' else -3
                         if quality == 'hd':
-                            preference += 5
+                            preference += 1
                         formats.append({
                             'format_id': '%s_%s_%s' % (format_id, quality, src_type),
                             'url': src,

From 53675852195d8dd859555d4789944a6887171ff8 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 16 Sep 2023 16:20:34 -0500
Subject: [PATCH 551/871] [ie/generic] Fix KVS thumbnail extraction

Closes #8045
Authored by: bashonly
---
 yt_dlp/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index f5c59a093..33e71d1c5 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -2370,7 +2370,7 @@ def _extract_kvs(self, url, webpage, video_id):
             'id': flashvars['video_id'],
             'display_id': display_id,
             'title': title,
-            'thumbnail': thumbnail,
+            'thumbnail': urljoin(url, thumbnail),
             'formats': formats,
         }
 

From 635ae31f68a3ac7f6393d59657ed711e34ee3552 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 16 Sep 2023 16:22:21 -0500
Subject: [PATCH 552/871] [ie/mediastream] Make embed extraction non-fatal

Authored by: bashonly
---
 yt_dlp/extractor/mediastream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/mediastream.py b/yt_dlp/extractor/mediastream.py
index cef769f29..d5c9aab8a 100644
--- a/yt_dlp/extractor/mediastream.py
+++ b/yt_dlp/extractor/mediastream.py
@@ -14,7 +14,7 @@ class MediaStreamBaseIE(InfoExtractor):
     _BASE_URL_RE = r'https?://mdstrm\.com/(?:embed|live-stream)'
 
     def _extract_mediastream_urls(self, webpage):
-        yield from traverse_obj(list(self._yield_json_ld(webpage, None)), (
+        yield from traverse_obj(list(self._yield_json_ld(webpage, None, fatal=False)), (
             lambda _, v: v['@type'] == 'VideoObject', ('embedUrl', 'contentUrl'),
             {lambda x: x if re.match(rf'{self._BASE_URL_RE}/\w+', x) else None}))
 

From 20c3c9b433dd47faf0dbde6b46e4e34eb76109a5 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 16 Sep 2023 16:23:54 -0500
Subject: [PATCH 553/871] [ie/reddit] Extract subtitles

Closes #7814
Authored by: bashonly
---
 yt_dlp/extractor/reddit.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/reddit.py b/yt_dlp/extractor/reddit.py
index 813e62874..62f669f35 100644
--- a/yt_dlp/extractor/reddit.py
+++ b/yt_dlp/extractor/reddit.py
@@ -319,16 +319,20 @@ def add_thumbnail(src):
                 'format_id': 'fallback',
                 'format_note': 'DASH video, mp4_dash',
             }]
-            formats.extend(self._extract_m3u8_formats(
-                hls_playlist_url, display_id, 'mp4', m3u8_id='hls', fatal=False))
-            formats.extend(self._extract_mpd_formats(
-                dash_playlist_url, display_id, mpd_id='dash', fatal=False))
+            hls_fmts, subtitles = self._extract_m3u8_formats_and_subtitles(
+                hls_playlist_url, display_id, 'mp4', m3u8_id='hls', fatal=False)
+            formats.extend(hls_fmts)
+            dash_fmts, dash_subs = self._extract_mpd_formats_and_subtitles(
+                dash_playlist_url, display_id, mpd_id='dash', fatal=False)
+            formats.extend(dash_fmts)
+            self._merge_subtitles(dash_subs, target=subtitles)
 
             return {
                 **info,
                 'id': video_id,
                 'display_id': display_id,
                 'formats': formats,
+                'subtitles': subtitles,
                 'duration': int_or_none(reddit_video.get('duration')),
             }
 

From eda0e415d26eb084e570cf5372d38ee1f616b70f Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Sat, 16 Sep 2023 23:47:49 +0100
Subject: [PATCH 554/871] [ie/bbc] Extract tracklist as chapters (#7788)

Authored by: garret1317
---
 yt_dlp/extractor/bbc.py | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/yt_dlp/extractor/bbc.py b/yt_dlp/extractor/bbc.py
index a55cdef2b..d1d6e04fa 100644
--- a/yt_dlp/extractor/bbc.py
+++ b/yt_dlp/extractor/bbc.py
@@ -15,11 +15,13 @@
     float_or_none,
     get_element_by_class,
     int_or_none,
+    join_nonempty,
     js_to_json,
     parse_duration,
     parse_iso8601,
     parse_qs,
     strip_or_none,
+    traverse_obj,
     try_get,
     unescapeHTML,
     unified_timestamp,
@@ -41,7 +43,6 @@ class BBCCoUkIE(InfoExtractor):
                             iplayer(?:/[^/]+)?/(?:episode/|playlist/)|
                             music/(?:clips|audiovideo/popular)[/#]|
                             radio/player/|
-                            sounds/play/|
                             events/[^/]+/play/[^/]+/
                         )
                         (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
@@ -218,20 +219,6 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-        }, {
-            'url': 'https://www.bbc.co.uk/sounds/play/m0007jzb',
-            'note': 'Audio',
-            'info_dict': {
-                'id': 'm0007jz9',
-                'ext': 'mp4',
-                'title': 'BBC Proms, 2019, Prom 34: West–Eastern Divan Orchestra',
-                'description': "Live BBC Proms. West–Eastern Divan Orchestra with Daniel Barenboim and Martha Argerich.",
-                'duration': 9840,
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            }
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,
@@ -844,6 +831,20 @@ class BBCIE(BBCCoUkIE):  # XXX: Do not subclass from concrete IE
             'upload_date': '20190604',
             'categories': ['Psychology'],
         },
+    }, {
+        # BBC Sounds
+        'url': 'https://www.bbc.co.uk/sounds/play/m001q78b',
+        'info_dict': {
+            'id': 'm001q789',
+            'ext': 'mp4',
+            'title': 'The Night Tracks Mix - Music for the darkling hour',
+            'thumbnail': 'https://ichef.bbci.co.uk/images/ic/raw/p0c00hym.jpg',
+            'chapters': 'count:8',
+            'description': 'md5:815fb51cbdaa270040aab8145b3f1d67',
+            'uploader': 'Radio 3',
+            'duration': 1800,
+            'uploader_id': 'bbc_radio_three',
+        },
     }, {  # onion routes
         'url': 'https://www.bbcnewsd73hkzno2ini43t4gblxvycyac5aw4gnv7t2rccijh7745uqd.onion/news/av/world-europe-63208576',
         'only_matching': True,
@@ -1128,6 +1129,13 @@ def _real_extract(self, url):
                     'uploader_id': network.get('id'),
                     'formats': formats,
                     'subtitles': subtitles,
+                    'chapters': traverse_obj(preload_state, (
+                        'tracklist', 'tracks', lambda _, v: float_or_none(v['offset']['start']), {
+                            'title': ('titles', {lambda x: join_nonempty(
+                                'primary', 'secondary', 'tertiary', delim=' - ', from_dict=x)}),
+                            'start_time': ('offset', 'start', {float_or_none}),
+                            'end_time': ('offset', 'end', {float_or_none}),
+                        })) or None,
                 }
 
         bbc3_config = self._parse_json(

From 2da7bcca16fdb40d4bdb2746643ba1a603771382 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 16 Sep 2023 18:57:14 -0500
Subject: [PATCH 555/871] Revert 9d376c4daeaf1279a011582f3f0e6ae42af520dd

Authored by: bashonly
---
 yt_dlp/extractor/amazonminitv.py | 63 +++++++++++++++++++++++++++-----
 1 file changed, 54 insertions(+), 9 deletions(-)

diff --git a/yt_dlp/extractor/amazonminitv.py b/yt_dlp/extractor/amazonminitv.py
index ad23b16bd..b57d985d1 100644
--- a/yt_dlp/extractor/amazonminitv.py
+++ b/yt_dlp/extractor/amazonminitv.py
@@ -37,7 +37,7 @@ def _call_api(self, asin, data=None, note=None):
         return resp['data'][data['operationName']]
 
 
-class AmazonMiniTVIE(InfoExtractor):
+class AmazonMiniTVIE(AmazonMiniTVBaseIE):
     _VALID_URL = r'(?:https?://(?:www\.)?amazon\.in/minitv/tp/|amazonminitv:(?:amzn1\.dv\.gti\.)?)(?P<id>[a-f0-9-]+)'
     _TESTS = [{
         'url': 'https://www.amazon.in/minitv/tp/75fe3a75-b8fe-4499-8100-5c9424344840?referrer=https%3A%2F%2Fwww.amazon.in%2Fminitv',
@@ -86,14 +86,56 @@ class AmazonMiniTVIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    _GRAPHQL_QUERY_CONTENT = '''
+query content($sessionIdToken: String!, $deviceLocale: String, $contentId: ID!, $contentType: ContentType!, $clientId: String) {
+  content(
+    applicationContextInput: {deviceLocale: $deviceLocale, sessionIdToken: $sessionIdToken, clientId: $clientId}
+    contentId: $contentId
+    contentType: $contentType
+  ) {
+    contentId
+    name
+    ... on Episode {
+      contentId
+      vodType
+      name
+      images
+      description {
+        synopsis
+        contentLengthInSeconds
+      }
+      publicReleaseDateUTC
+      audioTracks
+      seasonId
+      seriesId
+      seriesName
+      seasonNumber
+      episodeNumber
+      timecode {
+        endCreditsTime
+      }
+    }
+    ... on MovieContent {
+      contentId
+      vodType
+      name
+      description {
+        synopsis
+        contentLengthInSeconds
+      }
+      images
+      publicReleaseDateUTC
+      audioTracks
+    }
+  }
+}'''
+
     def _real_extract(self, url):
-        video_uuid = self._match_id(url)
-        asin = f'amzn1.dv.gti.{video_uuid}'
-        webpage = self._download_webpage(f'https://www.amazon.in/minitv/tp/{video_uuid}', asin)
-        data = self._search_nextjs_data(webpage, asin)['props']['pageProps']['ssrProps']
+        asin = f'amzn1.dv.gti.{self._match_id(url)}'
+        prs = self._call_api(asin, note='Downloading playback info')
 
         formats, subtitles = [], {}
-        for type_, asset in traverse_obj(data, ('playbackData', 'playbackAssets', {dict.items}, ...)):
+        for type_, asset in prs['playbackAssets'].items():
             if not traverse_obj(asset, 'manifestUrl'):
                 continue
             if type_ == 'hls':
@@ -110,7 +152,12 @@ def _real_extract(self, url):
             else:
                 self.report_warning(f'Unknown asset type: {type_}')
 
-        title_info = traverse_obj(data, ('contentData', {dict})) or {}
+        title_info = self._call_api(
+            asin, note='Downloading title info', data={
+                'operationName': 'content',
+                'variables': {'contentId': asin},
+                'query': self._GRAPHQL_QUERY_CONTENT,
+            })
         credits_time = try_get(title_info, lambda x: x['timecode']['endCreditsTime'] / 1000)
         is_episode = title_info.get('vodType') == 'EPISODE'
 
@@ -145,7 +192,6 @@ class AmazonMiniTVSeasonIE(AmazonMiniTVBaseIE):
     IE_NAME = 'amazonminitv:season'
     _VALID_URL = r'amazonminitv:season:(?:amzn1\.dv\.gti\.)?(?P<id>[a-f0-9-]+)'
     IE_DESC = 'Amazon MiniTV Season, "minitv:season:" prefix'
-    _WORKING = False
     _TESTS = [{
         'url': 'amazonminitv:season:amzn1.dv.gti.0aa996eb-6a1b-4886-a342-387fbd2f1db0',
         'playlist_mincount': 6,
@@ -205,7 +251,6 @@ class AmazonMiniTVSeriesIE(AmazonMiniTVBaseIE):
     IE_NAME = 'amazonminitv:series'
     _VALID_URL = r'amazonminitv:series:(?:amzn1\.dv\.gti\.)?(?P<id>[a-f0-9-]+)'
     IE_DESC = 'Amazon MiniTV Series, "minitv:series:" prefix'
-    _WORKING = False
     _TESTS = [{
         'url': 'amazonminitv:series:amzn1.dv.gti.56521d46-b040-4fd5-872e-3e70476a04b0',
         'playlist_mincount': 3,

From 538d37671a17e0782d17f08df17800e2e3bd57c8 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 16 Sep 2023 19:03:30 -0500
Subject: [PATCH 556/871] [ie/AmazonMiniTV] Fix extractors

Closes #7817
Authored by: GautamMKGarg, bashonly

Co-authored by: GautamMKGarg <GautamMKgarg@gmail.com>
---
 yt_dlp/extractor/amazonminitv.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/amazonminitv.py b/yt_dlp/extractor/amazonminitv.py
index b57d985d1..2c71c5ef5 100644
--- a/yt_dlp/extractor/amazonminitv.py
+++ b/yt_dlp/extractor/amazonminitv.py
@@ -22,8 +22,11 @@ def _call_api(self, asin, data=None, note=None):
 
         resp = self._download_json(
             f'https://www.amazon.in/minitv/api/web/{"graphql" if data else "prs"}',
-            asin, note=note, headers={'Content-Type': 'application/json'},
-            data=json.dumps(data).encode() if data else None,
+            asin, note=note, headers={
+                'Content-Type': 'application/json',
+                'currentpageurl': '/',
+                'currentplatform': 'dWeb'
+            }, data=json.dumps(data).encode() if data else None,
             query=None if data else {
                 'deviceType': 'A1WMMUXPCUJL4N',
                 'contentId': asin,
@@ -46,7 +49,7 @@ class AmazonMiniTVIE(AmazonMiniTVBaseIE):
             'ext': 'mp4',
             'title': 'May I Kiss You?',
             'language': 'Hindi',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
             'description': 'md5:a549bfc747973e04feb707833474e59d',
             'release_timestamp': 1644710400,
             'release_date': '20220213',
@@ -68,7 +71,7 @@ class AmazonMiniTVIE(AmazonMiniTVBaseIE):
             'ext': 'mp4',
             'title': 'Jahaan',
             'language': 'Hindi',
-            'thumbnail': r're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
             'description': 'md5:05eb765a77bf703f322f120ec6867339',
             'release_timestamp': 1647475200,
             'release_date': '20220317',

From 9652bca1bd02f6bc1b8cb1e186f2ccbf32225561 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 16 Sep 2023 19:38:09 -0500
Subject: [PATCH 557/871] [ie/web.archive:vlive] Remove extractor (#8132)

Closes #8122
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   1 -
 yt_dlp/extractor/archiveorg.py  | 235 --------------------------------
 yt_dlp/extractor/naver.py       |   2 +-
 3 files changed, 1 insertion(+), 237 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 4fed6d66a..bf0c67542 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -122,7 +122,6 @@
 from .archiveorg import (
     ArchiveOrgIE,
     YoutubeWebArchiveIE,
-    VLiveWebArchiveIE,
 )
 from .arcpublishing import ArcPublishingIE
 from .arkena import ArkenaIE
diff --git a/yt_dlp/extractor/archiveorg.py b/yt_dlp/extractor/archiveorg.py
index 2541cd6fd..a0b26ac5a 100644
--- a/yt_dlp/extractor/archiveorg.py
+++ b/yt_dlp/extractor/archiveorg.py
@@ -3,7 +3,6 @@
 import urllib.parse
 
 from .common import InfoExtractor
-from .naver import NaverBaseIE
 from .youtube import YoutubeBaseInfoExtractor, YoutubeIE
 from ..compat import compat_urllib_parse_unquote
 from ..networking import HEADRequest
@@ -947,237 +946,3 @@ def _real_extract(self, url):
         if not info.get('title'):
             info['title'] = video_id
         return info
-
-
-class VLiveWebArchiveIE(InfoExtractor):
-    IE_NAME = 'web.archive:vlive'
-    IE_DESC = 'web.archive.org saved vlive videos'
-    _VALID_URL = r'''(?x)
-            (?:https?://)?web\.archive\.org/
-            (?:web/)?(?:(?P<date>[0-9]{14})?[0-9A-Za-z_*]*/)?  # /web and the version index is optional
-            (?:https?(?::|%3[Aa])//)?(?:
-                (?:(?:www|m)\.)?vlive\.tv(?::(?:80|443))?/(?:video|embed)/(?P<id>[0-9]+)  # VLive URL
-            )
-        '''
-    _TESTS = [{
-        'url': 'https://web.archive.org/web/20221221144331/http://www.vlive.tv/video/1326',
-        'md5': 'cc7314812855ce56de70a06a27314983',
-        'info_dict': {
-            'id': '1326',
-            'ext': 'mp4',
-            'title': "Girl's Day's Broadcast",
-            'creator': "Girl's Day",
-            'view_count': int,
-            'uploader_id': 'muploader_a',
-            'uploader_url': None,
-            'uploader': None,
-            'upload_date': '20150817',
-            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
-            'timestamp': 1439816449,
-            'like_count': int,
-            'channel': 'Girl\'s Day',
-            'channel_id': 'FDF27',
-            'comment_count': int,
-            'release_timestamp': 1439818140,
-            'release_date': '20150817',
-            'duration': 1014,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://web.archive.org/web/20221221182103/http://www.vlive.tv/video/16937',
-        'info_dict': {
-            'id': '16937',
-            'ext': 'mp4',
-            'title': '첸백시 걍방',
-            'creator': 'EXO',
-            'view_count': int,
-            'subtitles': 'mincount:12',
-            'uploader_id': 'muploader_j',
-            'uploader_url': 'http://vlive.tv',
-            'uploader': None,
-            'upload_date': '20161112',
-            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
-            'timestamp': 1478923074,
-            'like_count': int,
-            'channel': 'EXO',
-            'channel_id': 'F94BD',
-            'comment_count': int,
-            'release_timestamp': 1478924280,
-            'release_date': '20161112',
-            'duration': 906,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://web.archive.org/web/20221127190050/http://www.vlive.tv/video/101870',
-        'info_dict': {
-            'id': '101870',
-            'ext': 'mp4',
-            'title': '[ⓓ xV] “레벨이들 매력에 반해? 안 반해?” 움직이는 HD 포토 (레드벨벳:Red Velvet)',
-            'creator': 'Dispatch',
-            'view_count': int,
-            'subtitles': 'mincount:6',
-            'uploader_id': 'V__FRA08071',
-            'uploader_url': 'http://vlive.tv',
-            'uploader': None,
-            'upload_date': '20181130',
-            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
-            'timestamp': 1543601327,
-            'like_count': int,
-            'channel': 'Dispatch',
-            'channel_id': 'C796F3',
-            'comment_count': int,
-            'release_timestamp': 1543601040,
-            'release_date': '20181130',
-            'duration': 279,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    # The wayback machine has special timestamp and "mode" values:
-    # timestamp:
-    #   1 = the first capture
-    #   2 = the last capture
-    # mode:
-    #   id_ = Identity - perform no alterations of the original resource, return it as it was archived.
-    _WAYBACK_BASE_URL = 'https://web.archive.org/web/2id_/'
-
-    def _download_archived_page(self, url, video_id, *, timestamp='2', **kwargs):
-        for retry in self.RetryManager():
-            try:
-                return self._download_webpage(f'https://web.archive.org/web/{timestamp}id_/{url}', video_id, **kwargs)
-            except ExtractorError as e:
-                if isinstance(e.cause, HTTPError) and e.cause.status == 404:
-                    raise ExtractorError('Page was not archived', expected=True)
-                retry.error = e
-                continue
-
-    def _download_archived_json(self, url, video_id, **kwargs):
-        page = self._download_archived_page(url, video_id, **kwargs)
-        if not page:
-            raise ExtractorError('Page was not archived', expected=True)
-        else:
-            return self._parse_json(page, video_id)
-
-    def _extract_formats_from_m3u8(self, m3u8_url, params, video_id):
-        m3u8_doc = self._download_archived_page(m3u8_url, video_id, note='Downloading m3u8', query=params, fatal=False)
-        if not m3u8_doc:
-            return
-
-        # M3U8 document should be changed to archive domain
-        m3u8_doc = m3u8_doc.splitlines()
-        url_base = m3u8_url.rsplit('/', 1)[0]
-        first_segment = None
-        for i, line in enumerate(m3u8_doc):
-            if not line.startswith('#'):
-                m3u8_doc[i] = f'{self._WAYBACK_BASE_URL}{url_base}/{line}?{urllib.parse.urlencode(params)}'
-                first_segment = first_segment or m3u8_doc[i]
-
-        # Segments may not have been archived. See https://web.archive.org/web/20221127190050/http://www.vlive.tv/video/101870
-        urlh = self._request_webpage(HEADRequest(first_segment), video_id, errnote=False,
-                                     fatal=False, note='Check first segment availablity')
-        if urlh:
-            formats, subtitles = self._parse_m3u8_formats_and_subtitles('\n'.join(m3u8_doc), ext='mp4', video_id=video_id)
-            if subtitles:
-                self._report_ignoring_subs('m3u8')
-            return formats
-
-    # Closely follows the logic of the ArchiveTeam grab script
-    # See: https://github.com/ArchiveTeam/vlive-grab/blob/master/vlive.lua
-    def _real_extract(self, url):
-        video_id, url_date = self._match_valid_url(url).group('id', 'date')
-
-        webpage = self._download_archived_page(f'https://www.vlive.tv/video/{video_id}', video_id, timestamp=url_date)
-
-        player_info = self._search_json(r'__PRELOADED_STATE__\s*=', webpage, 'player info', video_id)
-        user_country = traverse_obj(player_info, ('common', 'userCountry'))
-
-        main_script_url = self._search_regex(r'<script\s+src="([^"]+/js/main\.[^"]+\.js)"', webpage, 'main script url')
-        main_script = self._download_archived_page(main_script_url, video_id, note='Downloading main script')
-        app_id = self._search_regex(r'appId\s*=\s*"([^"]+)"', main_script, 'app id')
-
-        inkey = self._download_archived_json(
-            f'https://www.vlive.tv/globalv-web/vam-web/video/v1.0/vod/{video_id}/inkey', video_id, note='Fetching inkey', query={
-                'appId': app_id,
-                'platformType': 'PC',
-                'gcc': user_country,
-                'locale': 'en_US',
-            }, fatal=False)
-
-        vod_id = traverse_obj(player_info, ('postDetail', 'post', 'officialVideo', 'vodId'))
-
-        vod_data = self._download_archived_json(
-            f'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{vod_id}', video_id, note='Fetching vod data', query={
-                'key': inkey.get('inkey'),
-                'pid': 'rmcPlayer_16692457559726800',  # partially unix time and partially random. Fixed value used by archiveteam project
-                'sid': '2024',
-                'ver': '2.0',
-                'devt': 'html5_pc',
-                'doct': 'json',
-                'ptc': 'https',
-                'sptc': 'https',
-                'cpt': 'vtt',
-                'ctls': '%7B%22visible%22%3A%7B%22fullscreen%22%3Atrue%2C%22logo%22%3Afalse%2C%22playbackRate%22%3Afalse%2C%22scrap%22%3Afalse%2C%22playCount%22%3Atrue%2C%22commentCount%22%3Atrue%2C%22title%22%3Atrue%2C%22writer%22%3Atrue%2C%22expand%22%3Afalse%2C%22subtitles%22%3Atrue%2C%22thumbnails%22%3Atrue%2C%22quality%22%3Atrue%2C%22setting%22%3Atrue%2C%22script%22%3Afalse%2C%22logoDimmed%22%3Atrue%2C%22badge%22%3Atrue%2C%22seekingTime%22%3Atrue%2C%22muted%22%3Atrue%2C%22muteButton%22%3Afalse%2C%22viewerNotice%22%3Afalse%2C%22linkCount%22%3Afalse%2C%22createTime%22%3Afalse%2C%22thumbnail%22%3Atrue%7D%2C%22clicked%22%3A%7B%22expand%22%3Afalse%2C%22subtitles%22%3Afalse%7D%7D',
-                'pv': '4.26.9',
-                'dr': '1920x1080',
-                'cpl': 'en_US',
-                'lc': 'en_US',
-                'adi': '%5B%7B%22type%22%3A%22pre%22%2C%22exposure%22%3Afalse%2C%22replayExposure%22%3Afalse%7D%5D',
-                'adu': '%2F',
-                'videoId': vod_id,
-                'cc': user_country,
-            })
-
-        formats = []
-
-        streams = traverse_obj(vod_data, ('streams', ...))
-        if len(streams) > 1:
-            self.report_warning('Multiple streams found. Only the first stream will be downloaded.')
-        stream = streams[0]
-
-        max_stream = max(
-            stream.get('videos') or [],
-            key=lambda v: traverse_obj(v, ('bitrate', 'video'), default=0), default=None)
-        if max_stream is not None:
-            params = {arg.get('name'): arg.get('value') for arg in stream.get('keys', []) if arg.get('type') == 'param'}
-            formats = self._extract_formats_from_m3u8(max_stream.get('source'), params, video_id) or []
-
-        # For parts of the project MP4 files were archived
-        max_video = max(
-            traverse_obj(vod_data, ('videos', 'list', ...)),
-            key=lambda v: traverse_obj(v, ('bitrate', 'video'), default=0), default=None)
-        if max_video is not None:
-            video_url = self._WAYBACK_BASE_URL + max_video.get('source')
-            urlh = self._request_webpage(HEADRequest(video_url), video_id, errnote=False,
-                                         fatal=False, note='Check video availablity')
-            if urlh:
-                formats.append({'url': video_url})
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            **traverse_obj(player_info, ('postDetail', 'post', {
-                'title': ('officialVideo', 'title', {str}),
-                'creator': ('author', 'nickname', {str}),
-                'channel': ('channel', 'channelName', {str}),
-                'channel_id': ('channel', 'channelCode', {str}),
-                'duration': ('officialVideo', 'playTime', {int_or_none}),
-                'view_count': ('officialVideo', 'playCount', {int_or_none}),
-                'like_count': ('officialVideo', 'likeCount', {int_or_none}),
-                'comment_count': ('officialVideo', 'commentCount', {int_or_none}),
-                'timestamp': ('officialVideo', 'createdAt', {lambda x: int_or_none(x, scale=1000)}),
-                'release_timestamp': ('officialVideo', 'willStartAt', {lambda x: int_or_none(x, scale=1000)}),
-            })),
-            **traverse_obj(vod_data, ('meta', {
-                'uploader_id': ('user', 'id', {str}),
-                'uploader': ('user', 'name', {str}),
-                'uploader_url': ('user', 'url', {url_or_none}),
-                'thumbnail': ('cover', 'source', {url_or_none}),
-            }), expected_type=lambda x: x or None),
-            **NaverBaseIE.process_subtitles(vod_data, lambda x: [self._WAYBACK_BASE_URL + x]),
-        }
diff --git a/yt_dlp/extractor/naver.py b/yt_dlp/extractor/naver.py
index d79caf5f3..2d8459b02 100644
--- a/yt_dlp/extractor/naver.py
+++ b/yt_dlp/extractor/naver.py
@@ -21,7 +21,7 @@
 class NaverBaseIE(InfoExtractor):
     _CAPTION_EXT_RE = r'\.(?:ttml|vtt)'
 
-    @staticmethod  # NB: Used in VLiveWebArchiveIE, WeverseIE
+    @staticmethod  # NB: Used in WeverseIE
     def process_subtitles(vod_data, process_url):
         ret = {'subtitles': {}, 'automatic_captions': {}}
         for caption in traverse_obj(vod_data, ('captions', 'list', ...)):

From 94389b225d9bcf29aa7ba8afaf1bbd7c62204eae Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 16 Sep 2023 21:42:42 -0500
Subject: [PATCH 558/871] [ie/RTVSLO] Fix format extraction (#8131)

Closes #8020
Authored by: bashonly
---
 yt_dlp/extractor/rtvslo.py | 50 +++++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/yt_dlp/extractor/rtvslo.py b/yt_dlp/extractor/rtvslo.py
index 05942b6b4..39ace7cc6 100644
--- a/yt_dlp/extractor/rtvslo.py
+++ b/yt_dlp/extractor/rtvslo.py
@@ -1,6 +1,7 @@
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    int_or_none,
     parse_duration,
     traverse_obj,
     unified_timestamp,
@@ -25,7 +26,7 @@ class RTVSLOIE(InfoExtractor):
             'url': 'https://www.rtvslo.si/rtv365/arhiv/174842550?s=tv',
             'info_dict': {
                 'id': '174842550',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'release_timestamp': 1643140032,
                 'upload_date': '20220125',
                 'series': 'Dnevnik',
@@ -69,7 +70,21 @@ class RTVSLOIE(InfoExtractor):
                 'tbr': 128000,
                 'release_date': '20220201',
             },
-
+        }, {
+            'url': 'https://365.rtvslo.si/arhiv/razred-zase/148350750',
+            'info_dict': {
+                'id': '148350750',
+                'ext': 'mp4',
+                'title': 'Prvi šolski dan, mozaična oddaja za mlade',
+                'series': 'Razred zase',
+                'series_id': '148185730',
+                'duration': 1481,
+                'upload_date': '20121019',
+                'timestamp': 1350672122,
+                'release_date': '20121019',
+                'release_timestamp': 1350672122,
+                'thumbnail': 'https://img.rtvcdn.si/_up/ava/ava_misc/show_logos/148185730/razred_zase_2014_logo_4d_wide2.jpg',
+            },
         }, {
             'url': 'https://4d.rtvslo.si/arhiv/dnevnik/174842550',
             'only_matching': True
@@ -98,13 +113,14 @@ def _real_extract(self, url):
         media = self._download_json(self._API_BASE.format('getMedia', v_id), v_id, query={'jwt': jwt})['response']
 
         formats = []
+        skip_protocols = ['smil', 'f4m', 'dash']
         adaptive_url = traverse_obj(media, ('addaptiveMedia', 'hls_sec'), expected_type=url_or_none)
         if adaptive_url:
-            formats = self._extract_wowza_formats(adaptive_url, v_id, skip_protocols=['smil'])
+            formats = self._extract_wowza_formats(adaptive_url, v_id, skip_protocols=skip_protocols)
 
         adaptive_url = traverse_obj(media, ('addaptiveMedia_sl', 'hls_sec'), expected_type=url_or_none)
         if adaptive_url:
-            for f in self._extract_wowza_formats(adaptive_url, v_id, skip_protocols=['smil']):
+            for f in self._extract_wowza_formats(adaptive_url, v_id, skip_protocols=skip_protocols):
                 formats.append({
                     **f,
                     'format_id': 'sign-' + f['format_id'],
@@ -114,19 +130,19 @@ def _real_extract(self, url):
                         else f.get('language'))
                 })
 
-        formats.extend(
-            {
-                'url': f['streams'][strm],
-                'ext': traverse_obj(f, 'mediaType', expected_type=str.lower),
-                'width': f.get('width'),
-                'height': f.get('height'),
-                'tbr': f.get('bitrate'),
-                'filesize': f.get('filesize'),
-            }
-            for strm in ('http', 'https')
-            for f in media.get('mediaFiles') or []
-            if traverse_obj(f, ('streams', strm))
-        )
+        for mediafile in traverse_obj(media, ('mediaFiles', lambda _, v: url_or_none(v['streams']['https']))):
+            formats.append(traverse_obj(mediafile, {
+                'url': ('streams', 'https'),
+                'ext': ('mediaType', {str.lower}),
+                'width': ('width', {int_or_none}),
+                'height': ('height', {int_or_none}),
+                'tbr': ('bitrate', {int_or_none}),
+                'filesize': ('filesize', {int_or_none}),
+            }))
+
+        for mediafile in traverse_obj(media, ('mediaFiles', lambda _, v: url_or_none(v['streams']['hls_sec']))):
+            formats.extend(self._extract_wowza_formats(
+                mediafile['streams']['hls_sec'], v_id, skip_protocols=skip_protocols))
 
         if any('intermission.mp4' in x['url'] for x in formats):
             self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)

From 836e06d246512f286f30c1371b2c54b72c9ecd93 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 17 Sep 2023 12:56:50 +0200
Subject: [PATCH 559/871] [core] Fix support for upcoming Python 3.12 (#8130)

This also adds the following test runners:
- `3.12-dev` on `ubuntu-latest`
- `3.12-dev` on `windows-latest`
- `pypy-3.10` on `ubuntu-latest`

Authored by: Grub4K
---
 .github/workflows/core.yml      |  5 ++++-
 devscripts/update-version.py    |  4 ++--
 yt_dlp/YoutubeDL.py             |  2 +-
 yt_dlp/extractor/aws.py         |  2 +-
 yt_dlp/extractor/goplay.py      |  4 ++--
 yt_dlp/extractor/motherless.py  |  2 +-
 yt_dlp/extractor/panopto.py     |  4 ++--
 yt_dlp/networking/_urllib.py    |  2 +-
 yt_dlp/networking/exceptions.py |  2 +-
 yt_dlp/utils/_utils.py          | 12 ++++++++----
 10 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index dead444c0..689408c50 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -13,13 +13,16 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         # CPython 3.11 is in quick-test
-        python-version: ['3.8', '3.9', '3.10', pypy-3.7, pypy-3.8]
+        python-version: ['3.8', '3.9', '3.10', '3.12-dev', pypy-3.7, pypy-3.8, pypy-3.10]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
         - os: windows-latest
           python-version: '3.7'
           run-tests-ext: bat
+        - os: windows-latest
+          python-version: '3.12-dev'
+          run-tests-ext: bat
         - os: windows-latest
           python-version: pypy-3.9
           run-tests-ext: bat
diff --git a/devscripts/update-version.py b/devscripts/update-version.py
index c873d10a5..0144bd284 100644
--- a/devscripts/update-version.py
+++ b/devscripts/update-version.py
@@ -10,14 +10,14 @@
 import argparse
 import contextlib
 import sys
-from datetime import datetime
+from datetime import datetime, timezone
 
 from devscripts.utils import read_version, run_process, write_file
 
 
 def get_new_version(version, revision):
     if not version:
-        version = datetime.utcnow().strftime('%Y.%m.%d')
+        version = datetime.now(timezone.utc).strftime('%Y.%m.%d')
 
     if revision:
         assert revision.isdigit(), 'Revision must be a number'
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 666d89b46..1feed3052 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2591,7 +2591,7 @@ def _fill_common_fields(self, info_dict, final=True):
                 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
                 # see http://bugs.python.org/issue1646728)
                 with contextlib.suppress(ValueError, OverflowError, OSError):
-                    upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
+                    upload_date = datetime.datetime.fromtimestamp(info_dict[ts_key], datetime.timezone.utc)
                     info_dict[date_key] = upload_date.strftime('%Y%m%d')
 
         live_keys = ('is_live', 'was_live')
diff --git a/yt_dlp/extractor/aws.py b/yt_dlp/extractor/aws.py
index eb831a153..c4741a6a1 100644
--- a/yt_dlp/extractor/aws.py
+++ b/yt_dlp/extractor/aws.py
@@ -12,7 +12,7 @@ class AWSIE(InfoExtractor):  # XXX: Conventionally, base classes should end with
 
     def _aws_execute_api(self, aws_dict, video_id, query=None):
         query = query or {}
-        amz_date = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
+        amz_date = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ')
         date = amz_date[:8]
         headers = {
             'Accept': 'application/json',
diff --git a/yt_dlp/extractor/goplay.py b/yt_dlp/extractor/goplay.py
index 960d7d7bc..0a3c8340f 100644
--- a/yt_dlp/extractor/goplay.py
+++ b/yt_dlp/extractor/goplay.py
@@ -383,9 +383,9 @@ def __get_current_timestamp():
         months = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
         days = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
 
-        time_now = datetime.datetime.utcnow()
+        time_now = datetime.datetime.now(datetime.timezone.utc)
         format_string = "{} {} {} %H:%M:%S UTC %Y".format(days[time_now.weekday()], months[time_now.month], time_now.day)
-        time_string = datetime.datetime.utcnow().strftime(format_string)
+        time_string = time_now.strftime(format_string)
         return time_string
 
     def __str__(self):
diff --git a/yt_dlp/extractor/motherless.py b/yt_dlp/extractor/motherless.py
index 769b52ce6..e359c44e9 100644
--- a/yt_dlp/extractor/motherless.py
+++ b/yt_dlp/extractor/motherless.py
@@ -151,7 +151,7 @@ def _real_extract(self, url):
                     'd': 'days',
                 }
                 kwargs = {_AGO_UNITS.get(uploaded_ago[-1]): delta}
-                upload_date = (datetime.datetime.utcnow() - datetime.timedelta(**kwargs)).strftime('%Y%m%d')
+                upload_date = (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(**kwargs)).strftime('%Y%m%d')
 
         comment_count = len(re.findall(r'''class\s*=\s*['"]media-comment-contents\b''', webpage))
         uploader_id = self._html_search_regex(
diff --git a/yt_dlp/extractor/panopto.py b/yt_dlp/extractor/panopto.py
index 6e3c9f442..5ab2b2bce 100644
--- a/yt_dlp/extractor/panopto.py
+++ b/yt_dlp/extractor/panopto.py
@@ -1,7 +1,7 @@
 import calendar
 import json
 import functools
-from datetime import datetime
+from datetime import datetime, timezone
 from random import random
 
 from .common import InfoExtractor
@@ -243,7 +243,7 @@ def _mark_watched(self, base_url, video_id, delivery_info):
         invocation_id = delivery_info.get('InvocationId')
         stream_id = traverse_obj(delivery_info, ('Delivery', 'Streams', ..., 'PublicID'), get_all=False, expected_type=str)
         if invocation_id and stream_id and duration:
-            timestamp_str = f'/Date({calendar.timegm(datetime.utcnow().timetuple())}000)/'
+            timestamp_str = f'/Date({calendar.timegm(datetime.now(timezone.utc).timetuple())}000)/'
             data = {
                 'streamRequests': [
                     {
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index b3e705b84..3c0647ecf 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -429,7 +429,7 @@ def _send(self, request):
         except urllib.error.HTTPError as e:
             if isinstance(e.fp, (http.client.HTTPResponse, urllib.response.addinfourl)):
                 # Prevent file object from being closed when urllib.error.HTTPError is destroyed.
-                e._closer.file = None
+                e._closer.close_called = True
                 raise HTTPError(UrllibResponseAdapter(e.fp), redirect_loop='redirect error' in str(e)) from e
             raise  # unexpected
         except urllib.error.URLError as e:
diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py
index 10afc9ccb..465b18ba9 100644
--- a/yt_dlp/networking/exceptions.py
+++ b/yt_dlp/networking/exceptions.py
@@ -115,7 +115,7 @@ def __init__(self, http_error: HTTPError):
             hdrs=http_error.response.headers,
             fp=http_error.response
         )
-        self._closer.file = None  # Disable auto close
+        self._closer.close_called = True  # Disable auto close
         self._http_error = http_error
         HTTPError.__init__(self, http_error.response, redirect_loop=http_error.redirect_loop)
 
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 180bec245..ef26de116 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -669,6 +669,7 @@ def replace_insane(char):
 
 def sanitize_path(s, force=False):
     """Sanitizes and normalizes path on Windows"""
+    # XXX: this handles drive relative paths (c:sth) incorrectly
     if sys.platform == 'win32':
         force = False
         drive_or_unc, _ = os.path.splitdrive(s)
@@ -687,7 +688,10 @@ def sanitize_path(s, force=False):
         sanitized_path.insert(0, drive_or_unc + os.path.sep)
     elif force and s and s[0] == os.path.sep:
         sanitized_path.insert(0, os.path.sep)
-    return os.path.join(*sanitized_path)
+    # TODO: Fix behavioral differences <3.12
+    # The workaround using `normpath` only superficially passes tests
+    # Ref: https://github.com/python/cpython/pull/100351
+    return os.path.normpath(os.path.join(*sanitized_path))
 
 
 def sanitize_url(url, *, scheme='http'):
@@ -1256,7 +1260,7 @@ def datetime_from_str(date_str, precision='auto', format='%Y%m%d'):
     if precision == 'auto':
         auto_precision = True
         precision = 'microsecond'
-    today = datetime_round(datetime.datetime.utcnow(), precision)
+    today = datetime_round(datetime.datetime.now(datetime.timezone.utc), precision)
     if date_str in ('now', 'today'):
         return today
     if date_str == 'yesterday':
@@ -1319,8 +1323,8 @@ def datetime_round(dt, precision='day'):
         'second': 1,
     }
     roundto = lambda x, n: ((x + n / 2) // n) * n
-    timestamp = calendar.timegm(dt.timetuple())
-    return datetime.datetime.utcfromtimestamp(roundto(timestamp, unit_seconds[precision]))
+    timestamp = roundto(calendar.timegm(dt.timetuple()), unit_seconds[precision])
+    return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc)
 
 
 def hyphenate_date(date_str):

From 30ba233d4cee945756ed7344e7ddb3a90d2ae608 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 17 Sep 2023 13:22:04 +0200
Subject: [PATCH 560/871] [devscripts] `make_changelog`: Fix changelog grouping
 and add networking group (#8124)

Authored by: Grub4K
---
 devscripts/changelog_override.json | 21 ++++++-
 devscripts/make_changelog.py       | 96 ++++++++++++++++--------------
 2 files changed, 71 insertions(+), 46 deletions(-)

diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index d03db3f23..e7f453acf 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -68,6 +68,25 @@
     {
         "action": "change",
         "when": "b03fa7834579a01cc5fba48c0e73488a16683d48",
-        "short": "[ie/twitter] Revert 92315c03774cfabb3a921884326beb4b981f786b"
+        "short": "[ie/twitter] Revert 92315c03774cfabb3a921884326beb4b981f786b",
+        "authors": ["pukkandan"]
+    },
+    {
+        "action": "change",
+        "when": "fcd6a76adc49d5cd8783985c7ce35384b72e545f",
+        "short": "[test] Add tests for socks proxies (#7908)",
+        "authors": ["coletdjnz"]
+    },
+    {
+        "action": "change",
+        "when": "4bf912282a34b58b6b35d8f7e6be535770c89c76",
+        "short": "[rh:urllib] Remove dot segments during URL normalization (#7662)",
+        "authors": ["coletdjnz"]
+    },
+    {
+        "action": "change",
+        "when": "59e92b1f1833440bb2190f847eb735cf0f90bc85",
+        "short": "[rh:urllib] Simplify gzip decoding (#7611)",
+        "authors": ["Grub4K"]
     }
 ]
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index 84f72d52f..ac68dcd19 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -31,35 +31,27 @@ class CommitGroup(enum.Enum):
     EXTRACTOR = 'Extractor'
     DOWNLOADER = 'Downloader'
     POSTPROCESSOR = 'Postprocessor'
+    NETWORKING = 'Networking'
     MISC = 'Misc.'
 
-    @classmethod
-    @property
-    def ignorable_prefixes(cls):
-        return ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream')
-
     @classmethod
     @lru_cache
-    def commit_lookup(cls):
+    def subgroup_lookup(cls):
         return {
             name: group
             for group, names in {
-                cls.PRIORITY: {'priority'},
                 cls.CORE: {
                     'aes',
                     'cache',
                     'compat_utils',
                     'compat',
                     'cookies',
-                    'core',
                     'dependencies',
                     'formats',
                     'jsinterp',
-                    'networking',
                     'outtmpl',
                     'plugins',
                     'update',
-                    'upstream',
                     'utils',
                 },
                 cls.MISC: {
@@ -67,23 +59,40 @@ def commit_lookup(cls):
                     'cleanup',
                     'devscripts',
                     'docs',
-                    'misc',
                     'test',
                 },
-                cls.EXTRACTOR: {'extractor', 'ie'},
-                cls.DOWNLOADER: {'downloader', 'fd'},
-                cls.POSTPROCESSOR: {'postprocessor', 'pp'},
+                cls.NETWORKING: {
+                    'rh',
+                },
             }.items()
             for name in names
         }
 
     @classmethod
-    def get(cls, value):
-        result = cls.commit_lookup().get(value)
-        if result:
-            logger.debug(f'Mapped {value!r} => {result.name}')
+    @lru_cache
+    def group_lookup(cls):
+        result = {
+            'fd': cls.DOWNLOADER,
+            'ie': cls.EXTRACTOR,
+            'pp': cls.POSTPROCESSOR,
+            'upstream': cls.CORE,
+        }
+        result.update({item.name.lower(): item for item in iter(cls)})
         return result
 
+    @classmethod
+    def get(cls, value: str) -> tuple[CommitGroup | None, str | None]:
+        group, _, subgroup = (group.strip().lower() for group in value.partition('/'))
+
+        result = cls.group_lookup().get(group)
+        if not result:
+            if subgroup:
+                return None, value
+            subgroup = group
+            result = cls.subgroup_lookup().get(subgroup)
+
+        return result, subgroup or None
+
 
 @dataclass
 class Commit:
@@ -198,19 +207,23 @@ def _prepare_cleanup_misc_items(self, items):
         for commit_infos in cleanup_misc_items.values():
             sorted_items.append(CommitInfo(
                 'cleanup', ('Miscellaneous',), ', '.join(
-                    self._format_message_link(None, info.commit.hash).strip()
+                    self._format_message_link(None, info.commit.hash)
                     for info in sorted(commit_infos, key=lambda item: item.commit.hash or '')),
                 [], Commit(None, '', commit_infos[0].commit.authors), []))
 
         return sorted_items
 
-    def format_single_change(self, info):
-        message = self._format_message_link(info.message, info.commit.hash)
+    def format_single_change(self, info: CommitInfo):
+        message, sep, rest = info.message.partition('\n')
+        if '[' not in message:
+            # If the message doesn't already contain markdown links, try to add a link to the commit
+            message = self._format_message_link(message, info.commit.hash)
+
         if info.issues:
-            message = message.replace('\n', f' ({self._format_issues(info.issues)})\n', 1)
+            message = f'{message} ({self._format_issues(info.issues)})'
 
         if info.commit.authors:
-            message = message.replace('\n', f' by {self._format_authors(info.commit.authors)}\n', 1)
+            message = f'{message} by {self._format_authors(info.commit.authors)}'
 
         if info.fixes:
             fix_message = ', '.join(f'{self._format_message_link(None, fix.hash)}' for fix in info.fixes)
@@ -219,16 +232,14 @@ def format_single_change(self, info):
             if authors != info.commit.authors:
                 fix_message = f'{fix_message} by {self._format_authors(authors)}'
 
-            message = message.replace('\n', f' (With fixes in {fix_message})\n', 1)
+            message = f'{message} (With fixes in {fix_message})'
 
-        return message[:-1]
+        return message if not sep else f'{message}{sep}{rest}'
 
     def _format_message_link(self, message, hash):
         assert message or hash, 'Improperly defined commit message or override'
         message = message if message else hash[:HASH_LENGTH]
-        if not hash:
-            return f'{message}\n'
-        return f'[{message}\n'.replace('\n', f']({self.repo_url}/commit/{hash})\n', 1)
+        return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message
 
     def _format_issues(self, issues):
         return ', '.join(f'[#{issue}]({self.repo_url}/issues/{issue})' for issue in issues)
@@ -318,7 +329,7 @@ def _get_commits_and_fixes(self, default_author):
         for commitish, revert_commit in reverts.items():
             reverted = commits.pop(commitish, None)
             if reverted:
-                logger.debug(f'{commit} fully reverted {reverted}')
+                logger.debug(f'{commitish} fully reverted {reverted}')
             else:
                 commits[revert_commit.hash] = revert_commit
 
@@ -337,7 +348,7 @@ def apply_overrides(self, overrides):
         for override in overrides:
             when = override.get('when')
             if when and when not in self and when != self._start:
-                logger.debug(f'Ignored {when!r}, not in commits {self._start!r}')
+                logger.debug(f'Ignored {when!r} override')
                 continue
 
             override_hash = override.get('hash') or when
@@ -365,7 +376,7 @@ def groups(self):
         for commit in self:
             upstream_re = self.UPSTREAM_MERGE_RE.search(commit.short)
             if upstream_re:
-                commit.short = f'[core/upstream] Merged with youtube-dl {upstream_re.group(1)}'
+                commit.short = f'[upstream] Merged with youtube-dl {upstream_re.group(1)}'
 
             match = self.MESSAGE_RE.fullmatch(commit.short)
             if not match:
@@ -410,25 +421,20 @@ def details_from_prefix(prefix):
         if not prefix:
             return CommitGroup.CORE, None, ()
 
-        prefix, _, details = prefix.partition('/')
-        prefix = prefix.strip()
-        details = details.strip()
+        prefix, *sub_details = prefix.split(':')
 
-        group = CommitGroup.get(prefix.lower())
-        if group is CommitGroup.PRIORITY:
-            prefix, _, details = details.partition('/')
+        group, details = CommitGroup.get(prefix)
+        if group is CommitGroup.PRIORITY and details:
+            details = details.partition('/')[2].strip()
 
-        if not details and prefix and prefix not in CommitGroup.ignorable_prefixes:
-            logger.debug(f'Replaced details with {prefix!r}')
-            details = prefix or None
+        if details and '/' in details:
+            logger.error(f'Prefix is overnested, using first part: {prefix}')
+            details = details.partition('/')[0].strip()
 
         if details == 'common':
             details = None
-
-        if details:
-            details, *sub_details = details.split(':')
-        else:
-            sub_details = []
+        elif group is CommitGroup.NETWORKING and details == 'rh':
+            details = 'Request Handler'
 
         return group, details, sub_details
 

From 58493923e9b6f774947a2131e5258e9f3cf816be Mon Sep 17 00:00:00 2001
From: soundchaser128 <69268557+soundchaser128@users.noreply.github.com>
Date: Sun, 17 Sep 2023 17:09:42 +0200
Subject: [PATCH 561/871] [ie/rule34video] Extract tags (#7117)

Authored by: soundchaser128
---
 yt_dlp/extractor/rule34video.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/rule34video.py b/yt_dlp/extractor/rule34video.py
index 9d15f4d21..f3250b557 100644
--- a/yt_dlp/extractor/rule34video.py
+++ b/yt_dlp/extractor/rule34video.py
@@ -1,6 +1,6 @@
 import re
 
-from ..utils import parse_duration
+from ..utils import parse_duration, unescapeHTML
 from .common import InfoExtractor
 
 
@@ -16,7 +16,8 @@ class Rule34VideoIE(InfoExtractor):
                 'title': 'Shot It-(mmd hmv)',
                 'thumbnail': 'https://rule34video.com/contents/videos_screenshots/3065000/3065157/preview.jpg',
                 'duration': 347.0,
-                'age_limit': 18
+                'age_limit': 18,
+                'tags': 'count:14'
             }
         },
         {
@@ -28,7 +29,8 @@ class Rule34VideoIE(InfoExtractor):
                 'title': 'Lara in Trouble Ep. 7 [WildeerStudio]',
                 'thumbnail': 'https://rule34video.com/contents/videos_screenshots/3065000/3065296/preview.jpg',
                 'duration': 938.0,
-                'age_limit': 18
+                'age_limit': 18,
+                'tags': 'count:50'
             }
         },
     ]
@@ -57,5 +59,7 @@ def _real_extract(self, url):
             'title': title,
             'thumbnail': thumbnail,
             'duration': parse_duration(duration),
-            'age_limit': 18
+            'age_limit': 18,
+            'tags': list(map(unescapeHTML, re.findall(
+                r'<a class="tag_item"[^>]+\bhref="https://rule34video\.com/tags/\d+/"[^>]*>(?P<tag>[^>]*)</a>', webpage))),
         }

From efa2339502a37cf13ae7f143bd8b2c28f452d1cd Mon Sep 17 00:00:00 2001
From: Simon <simon30002021@icloud.com>
Date: Sun, 17 Sep 2023 17:11:22 +0200
Subject: [PATCH 562/871] [ie/lecturio] Improve `_VALID_URL` (#7649)

Authored by: simon300000
---
 yt_dlp/extractor/lecturio.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/lecturio.py b/yt_dlp/extractor/lecturio.py
index bb059d3a2..795012541 100644
--- a/yt_dlp/extractor/lecturio.py
+++ b/yt_dlp/extractor/lecturio.py
@@ -57,8 +57,8 @@ class LecturioIE(LecturioBaseIE):
     _VALID_URL = r'''(?x)
                     https://
                         (?:
-                            app\.lecturio\.com/([^/]+/(?P<nt>[^/?#&]+)\.lecture|(?:\#/)?lecture/c/\d+/(?P<id>\d+))|
-                            (?:www\.)?lecturio\.de/[^/]+/(?P<nt_de>[^/?#&]+)\.vortrag
+                            app\.lecturio\.com/([^/?#]+/(?P<nt>[^/?#&]+)\.lecture|(?:\#/)?lecture/c/\d+/(?P<id>\d+))|
+                            (?:www\.)?lecturio\.de/(?:[^/?#]+/)+(?P<nt_de>[^/?#&]+)\.vortrag
                         )
                     '''
     _TESTS = [{
@@ -73,6 +73,9 @@ class LecturioIE(LecturioBaseIE):
     }, {
         'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
         'only_matching': True,
+    }, {
+        'url': 'https://www.lecturio.de/jura/oeffentliches-recht-at-1-staatsexamen/oeffentliches-recht-staatsexamen.vortrag',
+        'only_matching': True,
     }, {
         'url': 'https://app.lecturio.com/#/lecture/c/6434/39634',
         'only_matching': True,

From 63e0c5748c0eb461a2ccca4181616eb930b4b750 Mon Sep 17 00:00:00 2001
From: aky-01 <65510015+aky-01@users.noreply.github.com>
Date: Sun, 17 Sep 2023 17:16:11 +0200
Subject: [PATCH 563/871] [ie/IndavideoEmbed] Fix extraction (#8129)

Closes #7190
Authored by: aky-01
---
 yt_dlp/extractor/indavideo.py | 73 +++++++++++++++++------------------
 1 file changed, 36 insertions(+), 37 deletions(-)

diff --git a/yt_dlp/extractor/indavideo.py b/yt_dlp/extractor/indavideo.py
index 4fa97d8bb..564bf8a02 100644
--- a/yt_dlp/extractor/indavideo.py
+++ b/yt_dlp/extractor/indavideo.py
@@ -1,9 +1,9 @@
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     int_or_none,
     parse_age_limit,
     parse_iso8601,
+    time_seconds,
     update_url_query,
 )
 
@@ -11,15 +11,14 @@
 class IndavideoEmbedIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:embed\.)?indavideo\.hu/player/video/|assets\.indavideo\.hu/swf/player\.swf\?.*\b(?:v(?:ID|id))=)(?P<id>[\da-f]+)'
     # Some example URLs covered by generic extractor:
-    #   http://indavideo.hu/video/Vicces_cica_1
-    #   http://index.indavideo.hu/video/2015_0728_beregszasz
-    #   http://auto.indavideo.hu/video/Sajat_utanfutoban_a_kis_tacsko
-    #   http://erotika.indavideo.hu/video/Amator_tini_punci
-    #   http://film.indavideo.hu/video/f_hrom_nagymamm_volt
-    #   http://palyazat.indavideo.hu/video/Embertelen_dal_Dodgem_egyuttes
-    _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//embed\.indavideo\.hu/player/video/[\da-f]+)']
+    #   https://indavideo.hu/video/Vicces_cica_1
+    #   https://index.indavideo.hu/video/Hod_Nemetorszagban
+    #   https://auto.indavideo.hu/video/Sajat_utanfutoban_a_kis_tacsko
+    #   https://film.indavideo.hu/video/f_farkaslesen
+    #   https://palyazat.indavideo.hu/video/Embertelen_dal_Dodgem_egyuttes
+    _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)//embed\.indavideo\.hu/player/video/[\da-f]+)']
     _TESTS = [{
-        'url': 'http://indavideo.hu/player/video/1bdc3c6d80/',
+        'url': 'https://indavideo.hu/player/video/1bdc3c6d80/',
         'md5': 'c8a507a1c7410685f83a06eaeeaafeab',
         'info_dict': {
             'id': '1837039',
@@ -36,21 +35,33 @@ class IndavideoEmbedIE(InfoExtractor):
             'tags': ['tánc', 'cica', 'cuki', 'cukiajanlo', 'newsroom'],
         },
     }, {
-        'url': 'http://embed.indavideo.hu/player/video/1bdc3c6d80?autostart=1&hide=1',
-        'only_matching': True,
-    }, {
-        'url': 'http://assets.indavideo.hu/swf/player.swf?v=fe25e500&vID=1bdc3c6d80&autostart=1&hide=1&i=1',
+        'url': 'https://embed.indavideo.hu/player/video/1bdc3c6d80?autostart=1&hide=1',
         'only_matching': True,
     }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://indavideo.hu/video/Vicces_cica_1',
+        'info_dict': {
+            'id': '1335611',
+            'ext': 'mp4',
+            'title': 'Vicces cica',
+            'description': 'Játszik a tablettel. :D',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'Jet_Pack',
+            'uploader_id': '491217',
+            'timestamp': 1390821212,
+            'upload_date': '20140127',
+            'duration': 7,
+            'age_limit': 0,
+            'tags': ['cica', 'Jet_Pack'],
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'https://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/%s' % video_id,
-            video_id)['data']
-
-        title = video['title']
+            f'https://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/{video_id}/',
+            video_id, query={'_': time_seconds()})['data']
 
         video_urls = []
 
@@ -60,33 +71,21 @@ def _real_extract(self, url):
         elif isinstance(video_files, dict):
             video_urls.extend(video_files.values())
 
-        video_file = video.get('video_file')
-        if video:
-            video_urls.append(video_file)
         video_urls = list(set(video_urls))
 
-        video_prefix = video_urls[0].rsplit('/', 1)[0]
-
-        for flv_file in video.get('flv_files', []):
-            flv_url = '%s/%s' % (video_prefix, flv_file)
-            if flv_url not in video_urls:
-                video_urls.append(flv_url)
-
-        filesh = video.get('filesh')
+        filesh = video.get('filesh') or {}
 
         formats = []
         for video_url in video_urls:
             height = int_or_none(self._search_regex(
                 r'\.(\d{3,4})\.mp4(?:\?|$)', video_url, 'height', default=None))
-            if filesh:
-                if not height:
-                    continue
-                token = filesh.get(compat_str(height))
-                if token is None:
-                    continue
-                video_url = update_url_query(video_url, {'token': token})
+            if not height and len(filesh) == 1:
+                height = int_or_none(list(filesh.keys())[0])
+            token = filesh.get(str(height))
+            if token is None:
+                continue
             formats.append({
-                'url': video_url,
+                'url': update_url_query(video_url, {'token': token}),
                 'height': height,
             })
 
@@ -103,7 +102,7 @@ def _real_extract(self, url):
 
         return {
             'id': video.get('id') or video_id,
-            'title': title,
+            'title': video.get('title'),
             'description': video.get('description'),
             'thumbnails': thumbnails,
             'uploader': video.get('user_name'),

From 81f46ac573dc443ad48560f308582a26784d3015 Mon Sep 17 00:00:00 2001
From: Sebastian Koch <sebastian@0py.de>
Date: Sun, 17 Sep 2023 22:54:00 +0200
Subject: [PATCH 564/871] [ie/massengeschmack.tv] Fix title extraction (#7813)

Authored by: sb0stn
---
 yt_dlp/extractor/massengeschmacktv.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/yt_dlp/extractor/massengeschmacktv.py b/yt_dlp/extractor/massengeschmacktv.py
index 7dacb43e0..1490e9b21 100644
--- a/yt_dlp/extractor/massengeschmacktv.py
+++ b/yt_dlp/extractor/massengeschmacktv.py
@@ -17,11 +17,12 @@ class MassengeschmackTVIE(InfoExtractor):
 
     _TEST = {
         'url': 'https://massengeschmack.tv/play/fktv202',
-        'md5': 'a9e054db9c2b5a08f0a0527cc201e8d3',
+        'md5': '9996f314994a49fefe5f39aa1b07ae21',
         'info_dict': {
             'id': 'fktv202',
             'ext': 'mp4',
-            'title': 'Fernsehkritik-TV - Folge 202',
+            'title': 'Fernsehkritik-TV #202',
+            'thumbnail': 'https://cache.massengeschmack.tv/img/mag/fktv202.jpg'
         },
     }
 
@@ -29,9 +30,6 @@ def _real_extract(self, url):
         episode = self._match_id(url)
 
         webpage = self._download_webpage(url, episode)
-        title = clean_html(self._html_search_regex(
-            '<h3>([^<]+)</h3>', webpage, 'title'))
-        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
         sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)
 
         formats = []
@@ -67,7 +65,8 @@ def _real_extract(self, url):
 
         return {
             'id': episode,
-            'title': title,
+            'title': clean_html(self._html_search_regex(
+                r'<span[^>]+\bid=["\']clip-title["\'][^>]*>([^<]+)', webpage, 'title', fatal=False)),
             'formats': formats,
-            'thumbnail': thumbnail,
+            'thumbnail': self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False),
         }

From 20fbbd9249a2f26c7ae579bde5ba5d69aa8fac69 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Mon, 18 Sep 2023 07:33:26 +0000
Subject: [PATCH 565/871] [networking] Fix various socks proxy bugs (#8065)

- Fixed support for IPv6 socks proxies
- Fixed support for IPv6 over socks5
- Fixed --source-address not being obeyed for socks4 and socks5
- Fixed socks4a when the destination address is an IPv4 address

Closes https://github.com/yt-dlp/yt-dlp/issues/7959
Fixes https://github.com/ytdl-org/youtube-dl/issues/15368

Authored by: coletdjnz
Co-authored-by: Simon Sawicki <accounts@grub4k.xyz>
Co-authored-by: bashonly <bashonly@bashonly.com>
---
 test/test_socks.py           | 38 +++++---------------
 yt_dlp/networking/_helper.py | 57 ++++++++++++++++++++++++++++++
 yt_dlp/networking/_urllib.py | 68 +++++++++++++-----------------------
 yt_dlp/socks.py              | 31 +++++++++-------
 4 files changed, 110 insertions(+), 84 deletions(-)

diff --git a/test/test_socks.py b/test/test_socks.py
index 95ffce275..211ee814d 100644
--- a/test/test_socks.py
+++ b/test/test_socks.py
@@ -281,17 +281,13 @@ def test_socks4_auth(self, handler, ctx):
                     rh, proxies={'all': f'socks4://user:@{server_address}'})
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [
-        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
-            reason='socks4a implementation currently broken when destination is not a domain name'))
-    ], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
     def test_socks4a_ipv4_target(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
                 response = ctx.socks_info_request(rh, target_domain='127.0.0.1')
                 assert response['version'] == 4
-                assert response['ipv4_address'] == '127.0.0.1'
-                assert response['domain_address'] is None
+                assert (response['ipv4_address'] == '127.0.0.1') != (response['domain_address'] == '127.0.0.1')
 
     @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
     def test_socks4a_domain_target(self, handler, ctx):
@@ -302,10 +298,7 @@ def test_socks4a_domain_target(self, handler, ctx):
                 assert response['ipv4_address'] is None
                 assert response['domain_address'] == 'localhost'
 
-    @pytest.mark.parametrize('handler,ctx', [
-        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
-            reason='source_address is not yet supported for socks4 proxies'))
-    ], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
@@ -327,10 +320,7 @@ def test_socks4_errors(self, handler, ctx, reply_code):
                 with pytest.raises(ProxyError):
                     ctx.socks_info_request(rh)
 
-    @pytest.mark.parametrize('handler,ctx', [
-        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
-            reason='IPv6 socks4 proxies are not yet supported'))
-    ], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
     def test_ipv6_socks4_proxy(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler, bind_ip='::1') as server_address:
             with handler(proxies={'all': f'socks4://{server_address}'}) as rh:
@@ -342,7 +332,7 @@ def test_ipv6_socks4_proxy(self, handler, ctx):
     @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
     def test_timeout(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler, sleep=2) as server_address:
-            with handler(proxies={'all': f'socks4://{server_address}'}, timeout=1) as rh:
+            with handler(proxies={'all': f'socks4://{server_address}'}, timeout=0.5) as rh:
                 with pytest.raises(TransportError):
                     ctx.socks_info_request(rh)
 
@@ -383,7 +373,7 @@ def test_socks5_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
                 response = ctx.socks_info_request(rh, target_domain='localhost')
-                assert response['ipv4_address'] == '127.0.0.1'
+                assert (response['ipv4_address'] == '127.0.0.1') != (response['ipv6_address'] == '::1')
                 assert response['version'] == 5
 
     @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
@@ -404,22 +394,15 @@ def test_socks5h_ip_target(self, handler, ctx):
                 assert response['domain_address'] is None
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [
-        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
-            reason='IPv6 destination addresses are not yet supported'))
-    ], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
     def test_socks5_ipv6_destination(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
                 response = ctx.socks_info_request(rh, target_domain='[::1]')
                 assert response['ipv6_address'] == '::1'
-                assert response['port'] == 80
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [
-        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
-            reason='IPv6 socks5 proxies are not yet supported'))
-    ], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
     def test_ipv6_socks5_proxy(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler, bind_ip='::1') as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -430,10 +413,7 @@ def test_ipv6_socks5_proxy(self, handler, ctx):
 
     # XXX: is there any feasible way of testing IPv6 source addresses?
     # Same would go for non-proxy source_address test...
-    @pytest.mark.parametrize('handler,ctx', [
-        pytest.param('Urllib', 'http', marks=pytest.mark.xfail(
-            reason='source_address is not yet supported for socks5 proxies'))
-    ], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
diff --git a/yt_dlp/networking/_helper.py b/yt_dlp/networking/_helper.py
index a43c57bb4..4c9dbf25d 100644
--- a/yt_dlp/networking/_helper.py
+++ b/yt_dlp/networking/_helper.py
@@ -2,6 +2,7 @@
 
 import contextlib
 import functools
+import socket
 import ssl
 import sys
 import typing
@@ -206,3 +207,59 @@ def wrapper(self, *args, **kwargs):
                 e.handler = self
             raise
     return wrapper
+
+
+def _socket_connect(ip_addr, timeout, source_address):
+    af, socktype, proto, canonname, sa = ip_addr
+    sock = socket.socket(af, socktype, proto)
+    try:
+        if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+            sock.settimeout(timeout)
+        if source_address:
+            sock.bind(source_address)
+        sock.connect(sa)
+        return sock
+    except socket.error:
+        sock.close()
+        raise
+
+
+def create_connection(
+    address,
+    timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+    source_address=None,
+    *,
+    _create_socket_func=_socket_connect
+):
+    # Work around socket.create_connection() which tries all addresses from getaddrinfo() including IPv6.
+    # This filters the addresses based on the given source_address.
+    # Based on: https://github.com/python/cpython/blob/main/Lib/socket.py#L810
+    host, port = address
+    ip_addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
+    if not ip_addrs:
+        raise socket.error('getaddrinfo returns an empty list')
+    if source_address is not None:
+        af = socket.AF_INET if ':' not in source_address[0] else socket.AF_INET6
+        ip_addrs = [addr for addr in ip_addrs if addr[0] == af]
+        if not ip_addrs:
+            raise OSError(
+                f'No remote IPv{4 if af == socket.AF_INET else 6} addresses available for connect. '
+                f'Can\'t use "{source_address[0]}" as source address')
+
+    err = None
+    for ip_addr in ip_addrs:
+        try:
+            sock = _create_socket_func(ip_addr, timeout, source_address)
+            # Explicitly break __traceback__ reference cycle
+            # https://bugs.python.org/issue36820
+            err = None
+            return sock
+        except socket.error as e:
+            err = e
+
+    try:
+        raise err
+    finally:
+        # Explicitly break __traceback__ reference cycle
+        # https://bugs.python.org/issue36820
+        err = None
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index 3c0647ecf..c327f7744 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -23,6 +23,7 @@
 from ._helper import (
     InstanceStoreMixin,
     add_accept_encoding_header,
+    create_connection,
     get_redirect_method,
     make_socks_proxy_opts,
     select_proxy,
@@ -54,44 +55,10 @@
 def _create_http_connection(http_class, source_address, *args, **kwargs):
     hc = http_class(*args, **kwargs)
 
+    if hasattr(hc, '_create_connection'):
+        hc._create_connection = create_connection
+
     if source_address is not None:
-        # This is to workaround _create_connection() from socket where it will try all
-        # address data from getaddrinfo() including IPv6. This filters the result from
-        # getaddrinfo() based on the source_address value.
-        # This is based on the cpython socket.create_connection() function.
-        # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
-        def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
-            host, port = address
-            err = None
-            addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
-            af = socket.AF_INET if '.' in source_address[0] else socket.AF_INET6
-            ip_addrs = [addr for addr in addrs if addr[0] == af]
-            if addrs and not ip_addrs:
-                ip_version = 'v4' if af == socket.AF_INET else 'v6'
-                raise OSError(
-                    "No remote IP%s addresses available for connect, can't use '%s' as source address"
-                    % (ip_version, source_address[0]))
-            for res in ip_addrs:
-                af, socktype, proto, canonname, sa = res
-                sock = None
-                try:
-                    sock = socket.socket(af, socktype, proto)
-                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
-                        sock.settimeout(timeout)
-                    sock.bind(source_address)
-                    sock.connect(sa)
-                    err = None  # Explicitly break reference cycle
-                    return sock
-                except OSError as _:
-                    err = _
-                    if sock is not None:
-                        sock.close()
-            if err is not None:
-                raise err
-            else:
-                raise OSError('getaddrinfo returns an empty list')
-        if hasattr(hc, '_create_connection'):
-            hc._create_connection = _create_connection
         hc.source_address = (source_address, 0)
 
     return hc
@@ -220,13 +187,28 @@ def make_socks_conn_class(base_class, socks_proxy):
     proxy_args = make_socks_proxy_opts(socks_proxy)
 
     class SocksConnection(base_class):
-        def connect(self):
-            self.sock = sockssocket()
-            self.sock.setproxy(**proxy_args)
-            if type(self.timeout) in (int, float):  # noqa: E721
-                self.sock.settimeout(self.timeout)
-            self.sock.connect((self.host, self.port))
+        _create_connection = create_connection
 
+        def connect(self):
+            def sock_socket_connect(ip_addr, timeout, source_address):
+                af, socktype, proto, canonname, sa = ip_addr
+                sock = sockssocket(af, socktype, proto)
+                try:
+                    connect_proxy_args = proxy_args.copy()
+                    connect_proxy_args.update({'addr': sa[0], 'port': sa[1]})
+                    sock.setproxy(**connect_proxy_args)
+                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:  # noqa: E721
+                        sock.settimeout(timeout)
+                    if source_address:
+                        sock.bind(source_address)
+                    sock.connect((self.host, self.port))
+                    return sock
+                except socket.error:
+                    sock.close()
+                    raise
+            self.sock = create_connection(
+                (proxy_args['addr'], proxy_args['port']), timeout=self.timeout,
+                source_address=self.source_address, _create_socket_func=sock_socket_connect)
             if isinstance(self, http.client.HTTPSConnection):
                 self.sock = self._context.wrap_socket(self.sock, server_hostname=self.host)
 
diff --git a/yt_dlp/socks.py b/yt_dlp/socks.py
index f93328f63..e7f41d7e2 100644
--- a/yt_dlp/socks.py
+++ b/yt_dlp/socks.py
@@ -134,26 +134,31 @@ def _check_response_version(self, expected_version, got_version):
             self.close()
             raise InvalidVersionError(expected_version, got_version)
 
-    def _resolve_address(self, destaddr, default, use_remote_dns):
-        try:
-            return socket.inet_aton(destaddr)
-        except OSError:
-            if use_remote_dns and self._proxy.remote_dns:
-                return default
-            else:
-                return socket.inet_aton(socket.gethostbyname(destaddr))
+    def _resolve_address(self, destaddr, default, use_remote_dns, family=None):
+        for f in (family,) if family else (socket.AF_INET, socket.AF_INET6):
+            try:
+                return f, socket.inet_pton(f, destaddr)
+            except OSError:
+                continue
+
+        if use_remote_dns and self._proxy.remote_dns:
+            return 0, default
+        else:
+            res = socket.getaddrinfo(destaddr, None, family=family or 0)
+            f, _, _, _, ipaddr = res[0]
+            return f, socket.inet_pton(f, ipaddr[0])
 
     def _setup_socks4(self, address, is_4a=False):
         destaddr, port = address
 
-        ipaddr = self._resolve_address(destaddr, SOCKS4_DEFAULT_DSTIP, use_remote_dns=is_4a)
+        _, ipaddr = self._resolve_address(destaddr, SOCKS4_DEFAULT_DSTIP, use_remote_dns=is_4a, family=socket.AF_INET)
 
         packet = struct.pack('!BBH', SOCKS4_VERSION, Socks4Command.CMD_CONNECT, port) + ipaddr
 
         username = (self._proxy.username or '').encode()
         packet += username + b'\x00'
 
-        if is_4a and self._proxy.remote_dns:
+        if is_4a and self._proxy.remote_dns and ipaddr == SOCKS4_DEFAULT_DSTIP:
             packet += destaddr.encode() + b'\x00'
 
         self.sendall(packet)
@@ -210,7 +215,7 @@ def _socks5_auth(self):
     def _setup_socks5(self, address):
         destaddr, port = address
 
-        ipaddr = self._resolve_address(destaddr, None, use_remote_dns=True)
+        family, ipaddr = self._resolve_address(destaddr, None, use_remote_dns=True)
 
         self._socks5_auth()
 
@@ -220,8 +225,10 @@ def _setup_socks5(self, address):
             destaddr = destaddr.encode()
             packet += struct.pack('!B', Socks5AddressType.ATYP_DOMAINNAME)
             packet += self._len_and_data(destaddr)
-        else:
+        elif family == socket.AF_INET:
             packet += struct.pack('!B', Socks5AddressType.ATYP_IPV4) + ipaddr
+        elif family == socket.AF_INET6:
+            packet += struct.pack('!B', Socks5AddressType.ATYP_IPV6) + ipaddr
         packet += struct.pack('!H', port)
 
         self.sendall(packet)

From ba8e9eb2c8bbb699f314169fab8e544437ad731e Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Mon, 18 Sep 2023 15:08:40 -0600
Subject: [PATCH 566/871] [ie/radiofrance] Add support for livestreams,
 podcasts, playlists (#7006)

Closes #4282
Authored by: elyse0
---
 yt_dlp/extractor/_extractors.py |   9 +-
 yt_dlp/extractor/radiofrance.py | 379 +++++++++++++++++++++++++++++++-
 2 files changed, 382 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index bf0c67542..ec3ae0e66 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1555,7 +1555,14 @@
 from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
-from .radiofrance import FranceCultureIE, RadioFranceIE
+from .radiofrance import (
+    FranceCultureIE,
+    RadioFranceIE,
+    RadioFranceLiveIE,
+    RadioFrancePodcastIE,
+    RadioFranceProfileIE,
+    RadioFranceProgramScheduleIE,
+)
 from .radiozet import RadioZetPodcastIE
 from .radiokapital import (
     RadioKapitalIE,
diff --git a/yt_dlp/extractor/radiofrance.py b/yt_dlp/extractor/radiofrance.py
index 92e51b7f4..35f4b91dd 100644
--- a/yt_dlp/extractor/radiofrance.py
+++ b/yt_dlp/extractor/radiofrance.py
@@ -1,7 +1,18 @@
+import itertools
 import re
+import urllib.parse
 
 from .common import InfoExtractor
-from ..utils import parse_duration, unified_strdate
+from ..utils import (
+    int_or_none,
+    join_nonempty,
+    js_to_json,
+    parse_duration,
+    strftime_or_none,
+    traverse_obj,
+    unified_strdate,
+    urljoin,
+)
 
 
 class RadioFranceIE(InfoExtractor):
@@ -56,8 +67,32 @@ def _real_extract(self, url):
         }
 
 
-class FranceCultureIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?radiofrance\.fr/(?:franceculture|fip|francemusique|mouv|franceinter)/podcasts/(?:[^?#]+/)?(?P<display_id>[^?#]+)-(?P<id>\d+)($|[?#])'
+class RadioFranceBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?radiofrance\.fr'
+
+    _STATIONS_RE = '|'.join(map(re.escape, (
+        'franceculture',
+        'franceinfo',
+        'franceinter',
+        'francemusique',
+        'fip',
+        'mouv',
+    )))
+
+    def _extract_data_from_webpage(self, webpage, display_id, key):
+        return traverse_obj(self._search_json(
+            r'\bconst\s+data\s*=', webpage, key, display_id,
+            contains_pattern=r'(\[\{.*?\}\]);', transform_source=js_to_json),
+            (..., 'data', key, {dict}), get_all=False) or {}
+
+
+class FranceCultureIE(RadioFranceBaseIE):
+    _VALID_URL = rf'''(?x)
+        {RadioFranceBaseIE._VALID_URL_BASE}
+        /(?:{RadioFranceBaseIE._STATIONS_RE})
+        /podcasts/(?:[^?#]+/)?(?P<display_id>[^?#]+)-(?P<id>\d{{6,}})(?:$|[?#])
+    '''
+
     _TESTS = [
         {
             'url': 'https://www.radiofrance.fr/franceculture/podcasts/science-en-questions/la-physique-d-einstein-aiderait-elle-a-comprendre-le-cerveau-8440487',
@@ -67,14 +102,30 @@ class FranceCultureIE(InfoExtractor):
                 'ext': 'mp3',
                 'title': 'La physique d’Einstein aiderait-elle à comprendre le cerveau ?',
                 'description': 'Existerait-il un pont conceptuel entre la physique de l’espace-temps et les neurosciences ?',
-                'thumbnail': 'https://cdn.radiofrance.fr/s3/cruiser-production/2022/05/d184e7a3-4827-4494-bf94-04ed7b120db4/1200x630_gettyimages-200171095-001.jpg',
+                'thumbnail': r're:^https?://.*\.(?:jpg|png)',
                 'upload_date': '20220514',
                 'duration': 2750,
             },
         },
+        {
+            'url': 'https://www.radiofrance.fr/franceinter/podcasts/le-7-9-30/le-7-9-30-du-vendredi-10-mars-2023-2107675',
+            'info_dict': {
+                'id': '2107675',
+                'display_id': 'le-7-9-30-du-vendredi-10-mars-2023',
+                'title': 'Inflation alimentaire : comment en sortir ? - Régis Debray et Claude Grange - Cybèle Idelot',
+                'description': 'md5:36ee74351ede77a314fdebb94026b916',
+                'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+                'upload_date': '20230310',
+                'duration': 8977,
+                'ext': 'mp3',
+            },
+        },
         {
             'url': 'https://www.radiofrance.fr/franceinter/podcasts/la-rafle-du-vel-d-hiv-une-affaire-d-etat/les-racines-du-crime-episode-1-3715507',
             'only_matching': True,
+        }, {
+            'url': 'https://www.radiofrance.fr/franceinfo/podcasts/le-billet-sciences/sante-bientot-un-vaccin-contre-l-asthme-allergique-3057200',
+            'only_matching': True,
         }
     ]
 
@@ -89,7 +140,6 @@ def _real_extract(self, url):
             'id': video_id,
             'display_id': display_id,
             'url': video_data['contentUrl'],
-            'ext': video_data.get('encodingFormat'),
             'vcodec': 'none' if video_data.get('encodingFormat') == 'mp3' else None,
             'duration': parse_duration(video_data.get('duration')),
             'title': self._html_search_regex(r'(?s)<h1[^>]*itemprop="[^"]*name[^"]*"[^>]*>(.+?)</h1>',
@@ -102,3 +152,322 @@ def _real_extract(self, url):
             'upload_date': unified_strdate(self._search_regex(
                 r'"datePublished"\s*:\s*"([^"]+)', webpage, 'timestamp', fatal=False))
         }
+
+
+class RadioFranceLiveIE(RadioFranceBaseIE):
+    _VALID_URL = rf'''(?x)
+        https?://(?:www\.)?radiofrance\.fr
+        /(?P<id>{RadioFranceBaseIE._STATIONS_RE})
+        /?(?P<substation_id>radio-[\w-]+)?(?:[#?]|$)
+    '''
+
+    _TESTS = [{
+        'url': 'https://www.radiofrance.fr/franceinter/',
+        'info_dict': {
+            'id': 'franceinter',
+            'title': str,
+            'live_status': 'is_live',
+            'ext': 'aac',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }, {
+        'url': 'https://www.radiofrance.fr/franceculture',
+        'info_dict': {
+            'id': 'franceculture',
+            'title': str,
+            'live_status': 'is_live',
+            'ext': 'aac',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }, {
+        'url': 'https://www.radiofrance.fr/mouv/radio-musique-kids-family',
+        'info_dict': {
+            'id': 'mouv-radio-musique-kids-family',
+            'title': str,
+            'live_status': 'is_live',
+            'ext': 'aac',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }, {
+        'url': 'https://www.radiofrance.fr/mouv/radio-rnb-soul',
+        'info_dict': {
+            'id': 'mouv-radio-rnb-soul',
+            'title': str,
+            'live_status': 'is_live',
+            'ext': 'aac',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }, {
+        'url': 'https://www.radiofrance.fr/mouv/radio-musique-mix',
+        'info_dict': {
+            'id': 'mouv-radio-musique-mix',
+            'title': str,
+            'live_status': 'is_live',
+            'ext': 'aac',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }, {
+        'url': 'https://www.radiofrance.fr/fip/radio-rock',
+        'info_dict': {
+            'id': 'fip-radio-rock',
+            'title': str,
+            'live_status': 'is_live',
+            'ext': 'aac',
+        },
+        'params': {
+            'skip_download': 'Livestream',
+        },
+    }, {
+        'url': 'https://www.radiofrance.fr/mouv',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        station_id, substation_id = self._match_valid_url(url).group('id', 'substation_id')
+
+        if substation_id:
+            webpage = self._download_webpage(url, station_id)
+            api_response = self._extract_data_from_webpage(webpage, station_id, 'webRadioData')
+        else:
+            api_response = self._download_json(
+                f'https://www.radiofrance.fr/{station_id}/api/live', station_id)
+
+        formats, subtitles = [], {}
+        for media_source in traverse_obj(api_response, (('now', None), 'media', 'sources', lambda _, v: v['url'])):
+            if media_source.get('format') == 'hls':
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(media_source['url'], station_id, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            else:
+                formats.append({
+                    'url': media_source['url'],
+                    'abr': media_source.get('bitrate'),
+                })
+
+        return {
+            'id': join_nonempty(station_id, substation_id),
+            'title': traverse_obj(api_response, ('visual', 'legend')) or join_nonempty(
+                ('now', 'firstLine', 'title'), ('now', 'secondLine', 'title'), from_dict=api_response, delim=' - '),
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': True,
+        }
+
+
+class RadioFrancePlaylistBase(RadioFranceBaseIE):
+    """Subclasses must set _METADATA_KEY"""
+
+    def _call_api(self, content_id, cursor, page_num):
+        raise NotImplementedError('This method must be implemented by subclasses')
+
+    def _generate_playlist_entries(self, content_id, content_response):
+        for page_num in itertools.count(2):
+            for entry in content_response['items']:
+                yield self.url_result(
+                    f'https://www.radiofrance.fr/{entry["path"]}', url_transparent=True, **traverse_obj(entry, {
+                        'title': 'title',
+                        'description': 'standFirst',
+                        'timestamp': ('publishedDate', {int_or_none}),
+                        'thumbnail': ('visual', 'src'),
+                    }))
+
+            next_cursor = traverse_obj(content_response, (('pagination', None), 'next'), get_all=False)
+            if not next_cursor:
+                break
+
+            content_response = self._call_api(content_id, next_cursor, page_num)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        metadata = self._download_json(
+            'https://www.radiofrance.fr/api/v2.1/path', display_id,
+            query={'value': urllib.parse.urlparse(url).path})['content']
+
+        content_id = metadata['id']
+
+        return self.playlist_result(
+            self._generate_playlist_entries(content_id, metadata[self._METADATA_KEY]), content_id,
+            display_id=display_id, **{**traverse_obj(metadata, {
+                'title': 'title',
+                'description': 'standFirst',
+                'thumbnail': ('visual', 'src'),
+            }), **traverse_obj(metadata, {
+                'title': 'name',
+                'description': 'role',
+            })})
+
+
+class RadioFrancePodcastIE(RadioFrancePlaylistBase):
+    _VALID_URL = rf'''(?x)
+        {RadioFranceBaseIE._VALID_URL_BASE}
+        /(?:{RadioFranceBaseIE._STATIONS_RE})
+        /podcasts/(?P<id>[\w-]+)/?(?:[?#]|$)
+    '''
+
+    _TESTS = [{
+        'url': 'https://www.radiofrance.fr/franceinfo/podcasts/le-billet-vert',
+        'info_dict': {
+            'id': 'eaf6ef81-a980-4f1c-a7d1-8a75ecd54b17',
+            'display_id': 'le-billet-vert',
+            'title': 'Le billet sciences',
+            'description': 'md5:eb1007b34b0c0a680daaa71525bbd4c1',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'playlist_mincount': 11,
+    }, {
+        'url': 'https://www.radiofrance.fr/franceinter/podcasts/jean-marie-le-pen-l-obsession-nationale',
+        'info_dict': {
+            'id': '566fd524-3074-4fbc-ac69-8696f2152a54',
+            'display_id': 'jean-marie-le-pen-l-obsession-nationale',
+            'title': 'Jean-Marie Le Pen, l\'obsession nationale',
+            'description': 'md5:a07c0cfb894f6d07a62d0ad12c4b7d73',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'playlist_count': 7,
+    }, {
+        'url': 'https://www.radiofrance.fr/franceculture/podcasts/serie-thomas-grjebine',
+        'info_dict': {
+            'id': '63c1ddc9-9f15-457a-98b2-411bac63f48d',
+            'display_id': 'serie-thomas-grjebine',
+            'title': 'Thomas Grjebine',
+        },
+        'playlist_count': 1,
+    }, {
+        'url': 'https://www.radiofrance.fr/fip/podcasts/certains-l-aiment-fip',
+        'info_dict': {
+            'id': '143dff38-e956-4a5d-8576-1c0b7242b99e',
+            'display_id': 'certains-l-aiment-fip',
+            'title': 'Certains l’aiment Fip',
+            'description': 'md5:ff974672ba00d4fd5be80fb001c5b27e',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'playlist_mincount': 321,
+    }, {
+        'url': 'https://www.radiofrance.fr/franceinter/podcasts/le-7-9',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.radiofrance.fr/mouv/podcasts/dirty-mix',
+        'only_matching': True,
+    }]
+
+    _METADATA_KEY = 'expressions'
+
+    def _call_api(self, podcast_id, cursor, page_num):
+        return self._download_json(
+            f'https://www.radiofrance.fr/api/v2.1/concepts/{podcast_id}/expressions', podcast_id,
+            note=f'Downloading page {page_num}', query={'pageCursor': cursor})
+
+
+class RadioFranceProfileIE(RadioFrancePlaylistBase):
+    _VALID_URL = rf'{RadioFranceBaseIE._VALID_URL_BASE}/personnes/(?P<id>[\w-]+)'
+
+    _TESTS = [{
+        'url': 'https://www.radiofrance.fr/personnes/thomas-pesquet?p=3',
+        'info_dict': {
+            'id': '86c62790-e481-11e2-9f7b-782bcb6744eb',
+            'display_id': 'thomas-pesquet',
+            'title': 'Thomas Pesquet',
+            'description': 'Astronaute à l\'agence spatiale européenne',
+        },
+        'playlist_mincount': 212,
+    }, {
+        'url': 'https://www.radiofrance.fr/personnes/eugenie-bastie',
+        'info_dict': {
+            'id': '9593050b-0183-4972-a0b5-d8f699079e02',
+            'display_id': 'eugenie-bastie',
+            'title': 'Eugénie Bastié',
+            'description': 'Journaliste et essayiste',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'playlist_mincount': 39,
+    }, {
+        'url': 'https://www.radiofrance.fr/personnes/lea-salame',
+        'only_matching': True,
+    }]
+
+    _METADATA_KEY = 'documents'
+
+    def _call_api(self, profile_id, cursor, page_num):
+        resp = self._download_json(
+            f'https://www.radiofrance.fr/api/v2.1/taxonomy/{profile_id}/documents', profile_id,
+            note=f'Downloading page {page_num}', query={
+                'relation': 'personality',
+                'cursor': cursor,
+            })
+
+        resp['next'] = traverse_obj(resp, ('pagination', 'next'))
+        return resp
+
+
+class RadioFranceProgramScheduleIE(RadioFranceBaseIE):
+    _VALID_URL = rf'''(?x)
+        {RadioFranceBaseIE._VALID_URL_BASE}
+        /(?P<station>{RadioFranceBaseIE._STATIONS_RE})
+        /grille-programmes(?:\?date=(?P<date>[\d-]+))?
+    '''
+
+    _TESTS = [{
+        'url': 'https://www.radiofrance.fr/franceinter/grille-programmes?date=17-02-2023',
+        'info_dict': {
+            'id': 'franceinter-program-20230217',
+            'upload_date': '20230217',
+        },
+        'playlist_count': 25,
+    }, {
+        'url': 'https://www.radiofrance.fr/franceculture/grille-programmes?date=01-02-2023',
+        'info_dict': {
+            'id': 'franceculture-program-20230201',
+            'upload_date': '20230201',
+        },
+        'playlist_count': 25,
+    }, {
+        'url': 'https://www.radiofrance.fr/mouv/grille-programmes?date=19-03-2023',
+        'info_dict': {
+            'id': 'mouv-program-20230319',
+            'upload_date': '20230319',
+        },
+        'playlist_count': 3,
+    }, {
+        'url': 'https://www.radiofrance.fr/francemusique/grille-programmes?date=18-03-2023',
+        'info_dict': {
+            'id': 'francemusique-program-20230318',
+            'upload_date': '20230318',
+        },
+        'playlist_count': 15,
+    }, {
+        'url': 'https://www.radiofrance.fr/franceculture/grille-programmes',
+        'only_matching': True,
+    }]
+
+    def _generate_playlist_entries(self, webpage_url, api_response):
+        for entry in traverse_obj(api_response, ('steps', lambda _, v: v['expression']['path'])):
+            yield self.url_result(
+                urljoin(webpage_url, f'/{entry["expression"]["path"]}'), ie=FranceCultureIE,
+                url_transparent=True, **traverse_obj(entry, {
+                    'title': ('expression', 'title'),
+                    'thumbnail': ('expression', 'visual', 'src'),
+                    'timestamp': ('startTime', {int_or_none}),
+                    'series_id': ('concept', 'id'),
+                    'series': ('concept', 'title'),
+                }))
+
+    def _real_extract(self, url):
+        station, date = self._match_valid_url(url).group('station', 'date')
+        webpage = self._download_webpage(url, station)
+        grid_data = self._extract_data_from_webpage(webpage, station, 'grid')
+        upload_date = strftime_or_none(grid_data.get('date'), '%Y%m%d')
+
+        return self.playlist_result(
+            self._generate_playlist_entries(url, grid_data),
+            join_nonempty(station, 'program', upload_date), upload_date=upload_date)

From 9e68747f9607f05e92bb7d9b6e79d678b50070e1 Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Mon, 18 Sep 2023 19:02:00 -0400
Subject: [PATCH 567/871] [ie/bilibili] Add support for series, favorites and
 watch later (#7518)

Closes #6719
Authored by: c-basalt
---
 yt_dlp/extractor/_extractors.py |   6 +-
 yt_dlp/extractor/bilibili.py    | 281 ++++++++++++++++++++++++++++++--
 2 files changed, 272 insertions(+), 15 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ec3ae0e66..a6a286766 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -223,7 +223,11 @@
     BiliBiliPlayerIE,
     BilibiliSpaceVideoIE,
     BilibiliSpaceAudioIE,
-    BilibiliSpacePlaylistIE,
+    BilibiliCollectionListIE,
+    BilibiliSeriesListIE,
+    BilibiliFavoritesListIE,
+    BilibiliWatchlaterIE,
+    BilibiliPlaylistIE,
     BiliIntlIE,
     BiliIntlSeriesIE,
     BiliLiveIE,
diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 290340078..5e7042dbb 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -15,6 +15,7 @@
     GeoRestrictedError,
     InAdvancePagedList,
     OnDemandPagedList,
+    bool_or_none,
     filter_dict,
     float_or_none,
     format_field,
@@ -35,6 +36,7 @@
     unsmuggle_url,
     url_or_none,
     urlencode_postdata,
+    variadic,
 )
 
 
@@ -156,7 +158,7 @@ def _get_episodes_from_season(self, ss_id, url):
 
 
 class BiliBiliIE(BilibiliBaseIE):
-    _VALID_URL = r'https?://www\.bilibili\.com/(?:video/|festival/\w+\?(?:[^#]*&)?bvid=)[aAbB][vV](?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/(?:video/|festival/\w+\?(?:[^#]*&)?bvid=)[aAbB][vV](?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'https://www.bilibili.com/video/BV13x41117TL',
@@ -252,7 +254,7 @@ class BiliBiliIE(BilibiliBaseIE):
             'description': 'md5:afde2b7ba9025c01d9e3dde10de221e4',
             'duration': 313.557,
             'upload_date': '20220709',
-            'uploader': '小夫Tech',
+            'uploader': '小夫太渴',
             'timestamp': 1657347907,
             'uploader_id': '1326814124',
             'comment_count': int,
@@ -509,7 +511,7 @@ def _real_extract(self, url):
 
 
 class BiliBiliBangumiMediaIE(BilibiliBaseIE):
-    _VALID_URL = r'https?://www\.bilibili\.com/bangumi/media/md(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/bangumi/media/md(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.bilibili.com/bangumi/media/md24097891',
         'info_dict': {
@@ -528,7 +530,7 @@ def _real_extract(self, url):
 
 
 class BiliBiliBangumiSeasonIE(BilibiliBaseIE):
-    _VALID_URL = r'(?x)https?://www\.bilibili\.com/bangumi/play/ss(?P<id>\d+)'
+    _VALID_URL = r'(?x)https?://(?:www\.)?bilibili\.com/bangumi/play/ss(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.bilibili.com/bangumi/play/ss26801',
         'info_dict': {
@@ -679,13 +681,35 @@ def get_entries(page_data):
         return self.playlist_result(paged_list, playlist_id)
 
 
-class BilibiliSpacePlaylistIE(BilibiliSpaceBaseIE):
-    _VALID_URL = r'https?://space.bilibili\.com/(?P<mid>\d+)/channel/collectiondetail\?sid=(?P<sid>\d+)'
+class BilibiliSpaceListBaseIE(BilibiliSpaceBaseIE):
+    def _get_entries(self, page_data, bvid_keys, ending_key='bvid'):
+        for bvid in traverse_obj(page_data, (*variadic(bvid_keys, (str, bytes, dict, set)), ..., ending_key, {str})):
+            yield self.url_result(f'https://www.bilibili.com/video/{bvid}', BiliBiliIE, bvid)
+
+    def _get_uploader(self, uid, playlist_id):
+        webpage = self._download_webpage(f'https://space.bilibili.com/{uid}', playlist_id, fatal=False)
+        return self._search_regex(r'(?s)<title\b[^>]*>([^<]+)的个人空间-', webpage, 'uploader', fatal=False)
+
+    def _extract_playlist(self, fetch_page, get_metadata, get_entries):
+        metadata, page_list = super()._extract_playlist(fetch_page, get_metadata, get_entries)
+        metadata.pop('page_count', None)
+        metadata.pop('page_size', None)
+        return metadata, page_list
+
+
+class BilibiliCollectionListIE(BilibiliSpaceListBaseIE):
+    _VALID_URL = r'https?://space\.bilibili\.com/(?P<mid>\d+)/channel/collectiondetail/?\?sid=(?P<sid>\d+)'
     _TESTS = [{
         'url': 'https://space.bilibili.com/2142762/channel/collectiondetail?sid=57445',
         'info_dict': {
             'id': '2142762_57445',
-            'title': '《底特律 变人》'
+            'title': '【完结】《底特律 变人》全结局流程解说',
+            'description': '',
+            'uploader': '老戴在此',
+            'uploader_id': '2142762',
+            'timestamp': int,
+            'upload_date': str,
+            'thumbnail': 'https://archive.biliimg.com/bfs/archive/e0e543ae35ad3df863ea7dea526bc32e70f4c091.jpg',
         },
         'playlist_mincount': 31,
     }]
@@ -706,22 +730,251 @@ def get_metadata(page_data):
             return {
                 'page_count': math.ceil(entry_count / page_size),
                 'page_size': page_size,
-                'title': traverse_obj(page_data, ('meta', 'name'))
+                'uploader': self._get_uploader(mid, playlist_id),
+                **traverse_obj(page_data, {
+                    'title': ('meta', 'name', {str}),
+                    'description': ('meta', 'description', {str}),
+                    'uploader_id': ('meta', 'mid', {str_or_none}),
+                    'timestamp': ('meta', 'ptime', {int_or_none}),
+                    'thumbnail': ('meta', 'cover', {url_or_none}),
+                })
             }
 
         def get_entries(page_data):
-            for entry in page_data.get('archives', []):
-                yield self.url_result(f'https://www.bilibili.com/video/{entry["bvid"]}',
-                                      BiliBiliIE, entry['bvid'])
+            return self._get_entries(page_data, 'archives')
 
         metadata, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries)
-        return self.playlist_result(paged_list, playlist_id, metadata['title'])
+        return self.playlist_result(paged_list, playlist_id, **metadata)
+
+
+class BilibiliSeriesListIE(BilibiliSpaceListBaseIE):
+    _VALID_URL = r'https?://space\.bilibili\.com/(?P<mid>\d+)/channel/seriesdetail/?\?\bsid=(?P<sid>\d+)'
+    _TESTS = [{
+        'url': 'https://space.bilibili.com/1958703906/channel/seriesdetail?sid=547718&ctype=0',
+        'info_dict': {
+            'id': '1958703906_547718',
+            'title': '直播回放',
+            'description': '直播回放',
+            'uploader': '靡烟miya',
+            'uploader_id': '1958703906',
+            'timestamp': 1637985853,
+            'upload_date': '20211127',
+            'modified_timestamp': int,
+            'modified_date': str,
+        },
+        'playlist_mincount': 513,
+    }]
+
+    def _real_extract(self, url):
+        mid, sid = self._match_valid_url(url).group('mid', 'sid')
+        playlist_id = f'{mid}_{sid}'
+        playlist_meta = traverse_obj(self._download_json(
+            f'https://api.bilibili.com/x/series/series?series_id={sid}', playlist_id, fatal=False
+        ), {
+            'title': ('data', 'meta', 'name', {str}),
+            'description': ('data', 'meta', 'description', {str}),
+            'uploader_id': ('data', 'meta', 'mid', {str_or_none}),
+            'timestamp': ('data', 'meta', 'ctime', {int_or_none}),
+            'modified_timestamp': ('data', 'meta', 'mtime', {int_or_none}),
+        })
+
+        def fetch_page(page_idx):
+            return self._download_json(
+                'https://api.bilibili.com/x/series/archives',
+                playlist_id, note=f'Downloading page {page_idx}',
+                query={'mid': mid, 'series_id': sid, 'pn': page_idx + 1, 'ps': 30})['data']
+
+        def get_metadata(page_data):
+            page_size = page_data['page']['size']
+            entry_count = page_data['page']['total']
+            return {
+                'page_count': math.ceil(entry_count / page_size),
+                'page_size': page_size,
+                'uploader': self._get_uploader(mid, playlist_id),
+                **playlist_meta
+            }
+
+        def get_entries(page_data):
+            return self._get_entries(page_data, 'archives')
+
+        metadata, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries)
+        return self.playlist_result(paged_list, playlist_id, **metadata)
+
+
+class BilibiliFavoritesListIE(BilibiliSpaceListBaseIE):
+    _VALID_URL = r'https?://(?:space\.bilibili\.com/\d+/favlist/?\?fid=|(?:www\.)?bilibili\.com/medialist/detail/ml)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://space.bilibili.com/84912/favlist?fid=1103407912&ftype=create',
+        'info_dict': {
+            'id': '1103407912',
+            'title': '【V2】（旧）',
+            'description': '',
+            'uploader': '晓月春日',
+            'uploader_id': '84912',
+            'timestamp': 1604905176,
+            'upload_date': '20201109',
+            'modified_timestamp': int,
+            'modified_date': str,
+            'thumbnail': r"re:http://i\d\.hdslb\.com/bfs/archive/14b83c62aa8871b79083df1e9ab4fbc699ad16fe\.jpg",
+            'view_count': int,
+            'like_count': int,
+        },
+        'playlist_mincount': 22,
+    }, {
+        'url': 'https://www.bilibili.com/medialist/detail/ml1103407912',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        fid = self._match_id(url)
+
+        list_info = self._download_json(
+            f'https://api.bilibili.com/x/v3/fav/resource/list?media_id={fid}&pn=1&ps=20',
+            fid, note='Downloading favlist metadata')
+        if list_info['code'] == -403:
+            self.raise_login_required(msg='This is a private favorites list. You need to log in as its owner')
+
+        entries = self._get_entries(self._download_json(
+            f'https://api.bilibili.com/x/v3/fav/resource/ids?media_id={fid}',
+            fid, note='Download favlist entries'), 'data')
+
+        return self.playlist_result(entries, fid, **traverse_obj(list_info, ('data', 'info', {
+            'title': ('title', {str}),
+            'description': ('intro', {str}),
+            'uploader': ('upper', 'name', {str}),
+            'uploader_id': ('upper', 'mid', {str_or_none}),
+            'timestamp': ('ctime', {int_or_none}),
+            'modified_timestamp': ('mtime', {int_or_none}),
+            'thumbnail': ('cover', {url_or_none}),
+            'view_count': ('cnt_info', 'play', {int_or_none}),
+            'like_count': ('cnt_info', 'thumb_up', {int_or_none}),
+        })))
+
+
+class BilibiliWatchlaterIE(BilibiliSpaceListBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/watchlater/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://www.bilibili.com/watchlater/#/list',
+        'info_dict': {'id': 'watchlater'},
+        'playlist_mincount': 0,
+        'skip': 'login required',
+    }]
+
+    def _real_extract(self, url):
+        list_id = getattr(self._get_cookies(url).get('DedeUserID'), 'value', 'watchlater')
+        watchlater_info = self._download_json(
+            'https://api.bilibili.com/x/v2/history/toview/web?jsonp=jsonp', list_id)
+        if watchlater_info['code'] == -101:
+            self.raise_login_required(msg='You need to login to access your watchlater list')
+        entries = self._get_entries(watchlater_info, ('data', 'list'))
+        return self.playlist_result(entries, id=list_id, title='稍后再看')
+
+
+class BilibiliPlaylistIE(BilibiliSpaceListBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/(?:medialist/play|list)/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://www.bilibili.com/list/1958703906?sid=547718',
+        'info_dict': {
+            'id': '5_547718',
+            'title': '直播回放',
+            'uploader': '靡烟miya',
+            'uploader_id': '1958703906',
+            'timestamp': 1637985853,
+            'upload_date': '20211127',
+        },
+        'playlist_mincount': 513,
+    }, {
+        'url': 'https://www.bilibili.com/medialist/play/1958703906?business=space_series&business_id=547718&desc=1',
+        'info_dict': {
+            'id': '5_547718',
+        },
+        'playlist_mincount': 513,
+        'skip': 'redirect url',
+    }, {
+        'url': 'https://www.bilibili.com/list/ml1103407912',
+        'info_dict': {
+            'id': '3_1103407912',
+            'title': '【V2】（旧）',
+            'uploader': '晓月春日',
+            'uploader_id': '84912',
+            'timestamp': 1604905176,
+            'upload_date': '20201109',
+            'thumbnail': r"re:http://i\d\.hdslb\.com/bfs/archive/14b83c62aa8871b79083df1e9ab4fbc699ad16fe\.jpg",
+        },
+        'playlist_mincount': 22,
+    }, {
+        'url': 'https://www.bilibili.com/medialist/play/ml1103407912',
+        'info_dict': {
+            'id': '3_1103407912',
+        },
+        'playlist_mincount': 22,
+        'skip': 'redirect url',
+    }, {
+        'url': 'https://www.bilibili.com/list/watchlater',
+        'info_dict': {'id': 'watchlater'},
+        'playlist_mincount': 0,
+        'skip': 'login required',
+    }, {
+        'url': 'https://www.bilibili.com/medialist/play/watchlater',
+        'info_dict': {'id': 'watchlater'},
+        'playlist_mincount': 0,
+        'skip': 'login required',
+    }]
+
+    def _extract_medialist(self, query, list_id):
+        for page_num in itertools.count(1):
+            page_data = self._download_json(
+                'https://api.bilibili.com/x/v2/medialist/resource/list',
+                list_id, query=query, note=f'getting playlist {query["biz_id"]} page {page_num}'
+            )['data']
+            yield from self._get_entries(page_data, 'media_list', ending_key='bv_id')
+            query['oid'] = traverse_obj(page_data, ('media_list', -1, 'id'))
+            if not page_data.get('has_more', False):
+                break
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        webpage = self._download_webpage(url, list_id)
+        initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', list_id)
+        if traverse_obj(initial_state, ('error', 'code', {int_or_none})) != 200:
+            error_code = traverse_obj(initial_state, ('error', 'trueCode', {int_or_none}))
+            error_message = traverse_obj(initial_state, ('error', 'message', {str_or_none}))
+            if error_code == -400 and list_id == 'watchlater':
+                self.raise_login_required('You need to login to access your watchlater playlist')
+            elif error_code == -403:
+                self.raise_login_required('This is a private playlist. You need to login as its owner')
+            elif error_code == 11010:
+                raise ExtractorError('Playlist is no longer available', expected=True)
+            raise ExtractorError(f'Could not access playlist: {error_code} {error_message}')
+
+        query = {
+            'ps': 20,
+            'with_current': False,
+            **traverse_obj(initial_state, {
+                'type': ('playlist', 'type', {int_or_none}),
+                'biz_id': ('playlist', 'id', {int_or_none}),
+                'tid': ('tid', {int_or_none}),
+                'sort_field': ('sortFiled', {int_or_none}),
+                'desc': ('desc', {bool_or_none}, {str_or_none}, {str.lower}),
+            })
+        }
+        metadata = {
+            'id': f'{query["type"]}_{query["biz_id"]}',
+            **traverse_obj(initial_state, ('mediaListInfo', {
+                'title': ('title', {str}),
+                'uploader': ('upper', 'name', {str}),
+                'uploader_id': ('upper', 'mid', {str_or_none}),
+                'timestamp': ('ctime', {int_or_none}),
+                'thumbnail': ('cover', {url_or_none}),
+            })),
+        }
+        return self.playlist_result(self._extract_medialist(query, list_id), **metadata)
 
 
 class BilibiliCategoryIE(InfoExtractor):
     IE_NAME = 'Bilibili category extractor'
     _MAX_RESULTS = 1000000
-    _VALID_URL = r'https?://www\.bilibili\.com/v/[a-zA-Z]+\/[a-zA-Z]+'
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/v/[a-zA-Z]+\/[a-zA-Z]+'
     _TESTS = [{
         'url': 'https://www.bilibili.com/v/kichiku/mad',
         'info_dict': {
@@ -1406,7 +1659,7 @@ def _real_extract(self, url):
 
 
 class BiliLiveIE(InfoExtractor):
-    _VALID_URL = r'https?://live.bilibili.com/(?:blanc/)?(?P<id>\d+)'
+    _VALID_URL = r'https?://live\.bilibili\.com/(?:blanc/)?(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'https://live.bilibili.com/196',

From 69b03f84f8378b0b5a2fbae56f9b7d860b2f529e Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Mon, 18 Sep 2023 19:06:36 -0400
Subject: [PATCH 568/871] [ie/weibo] Fix extractor and support user extraction
 (#7657)

Closes #3964, Closes #4673, Closes #6979
Authored by: c-basalt
---
 yt_dlp/extractor/_extractors.py |   3 +-
 yt_dlp/extractor/weibo.py       | 319 +++++++++++++++++++++-----------
 2 files changed, 215 insertions(+), 107 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index a6a286766..47d983c9c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2371,7 +2371,8 @@
 )
 from .weibo import (
     WeiboIE,
-    WeiboMobileIE
+    WeiboVideoIE,
+    WeiboUserIE,
 )
 from .weiqitv import WeiqiTVIE
 from .weverse import (
diff --git a/yt_dlp/extractor/weibo.py b/yt_dlp/extractor/weibo.py
index bc9a71abe..b0c3052b6 100644
--- a/yt_dlp/extractor/weibo.py
+++ b/yt_dlp/extractor/weibo.py
@@ -1,134 +1,241 @@
-from .common import InfoExtractor
-
-import json
 import random
-import re
+import itertools
+import urllib.parse
 
-from ..compat import (
-    compat_parse_qs,
-    compat_str,
-)
+from .common import InfoExtractor
 from ..utils import (
-    js_to_json,
+    int_or_none,
+    make_archive_id,
+    mimetype2ext,
+    parse_resolution,
+    str_or_none,
     strip_jsonp,
+    traverse_obj,
+    url_or_none,
     urlencode_postdata,
+    urljoin,
 )
 
 
-class WeiboIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?weibo\.com/[0-9]+/(?P<id>[a-zA-Z0-9]+)'
-    _TEST = {
-        'url': 'https://weibo.com/6275294458/Fp6RGfbff?type=comment',
-        'info_dict': {
-            'id': 'Fp6RGfbff',
-            'ext': 'mp4',
-            'title': 'You should have servants to massage you,... 来自Hosico_猫 - 微博',
-        }
-    }
+class WeiboBaseIE(InfoExtractor):
+    def _update_visitor_cookies(self, video_id):
+        visitor_data = self._download_json(
+            'https://passport.weibo.com/visitor/genvisitor', video_id,
+            note='Generating first-visit guest request',
+            transform_source=strip_jsonp,
+            data=urlencode_postdata({
+                'cb': 'gen_callback',
+                'fp': '{"os":"2","browser":"Gecko57,0,0,0","fonts":"undefined","screenInfo":"1440*900*24","plugins":""}',
+            }))
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        # to get Referer url for genvisitor
-        webpage, urlh = self._download_webpage_handle(url, video_id)
-
-        visitor_url = urlh.url
-
-        if 'passport.weibo.com' in visitor_url:
-            # first visit
-            visitor_data = self._download_json(
-                'https://passport.weibo.com/visitor/genvisitor', video_id,
-                note='Generating first-visit data',
-                transform_source=strip_jsonp,
-                headers={'Referer': visitor_url},
-                data=urlencode_postdata({
-                    'cb': 'gen_callback',
-                    'fp': json.dumps({
-                        'os': '2',
-                        'browser': 'Gecko57,0,0,0',
-                        'fonts': 'undefined',
-                        'screenInfo': '1440*900*24',
-                        'plugins': '',
-                    }),
-                }))
-
-            tid = visitor_data['data']['tid']
-            cnfd = '%03d' % visitor_data['data']['confidence']
-
-            self._download_webpage(
-                'https://passport.weibo.com/visitor/visitor', video_id,
-                note='Running first-visit callback',
-                query={
-                    'a': 'incarnate',
-                    't': tid,
-                    'w': 2,
-                    'c': cnfd,
-                    'cb': 'cross_domain',
-                    'from': 'weibo',
-                    '_rand': random.random(),
-                })
-
-            webpage = self._download_webpage(
-                url, video_id, note='Revisiting webpage')
-
-        title = self._html_extract_title(webpage)
-
-        video_formats = compat_parse_qs(self._search_regex(
-            r'video-sources=\\\"(.+?)\"', webpage, 'video_sources'))
-
-        formats = []
-        supported_resolutions = (480, 720)
-        for res in supported_resolutions:
-            vid_urls = video_formats.get(compat_str(res))
-            if not vid_urls or not isinstance(vid_urls, list):
-                continue
-
-            vid_url = vid_urls[0]
-            formats.append({
-                'url': vid_url,
-                'height': res,
+        self._download_webpage(
+            'https://passport.weibo.com/visitor/visitor', video_id,
+            note='Running first-visit callback to get guest cookies',
+            query={
+                'a': 'incarnate',
+                't': visitor_data['data']['tid'],
+                'w': 2,
+                'c': '%03d' % visitor_data['data']['confidence'],
+                'cb': 'cross_domain',
+                'from': 'weibo',
+                '_rand': random.random(),
             })
 
-        uploader = self._og_search_property(
-            'nick-name', webpage, 'uploader', default=None)
+    def _weibo_download_json(self, url, video_id, *args, fatal=True, note='Downloading JSON metadata', **kwargs):
+        webpage, urlh = self._download_webpage_handle(url, video_id, *args, fatal=fatal, note=note, **kwargs)
+        if urllib.parse.urlparse(urlh.url).netloc == 'passport.weibo.com':
+            self._update_visitor_cookies(video_id)
+            webpage = self._download_webpage(url, video_id, *args, fatal=fatal, note=note, **kwargs)
+        return self._parse_json(webpage, video_id, fatal=fatal)
 
+    def _extract_formats(self, video_info):
+        media_info = traverse_obj(video_info, ('page_info', 'media_info'))
+        formats = traverse_obj(media_info, (
+            'playback_list', lambda _, v: url_or_none(v['play_info']['url']), 'play_info', {
+                'url': 'url',
+                'format': ('quality_desc', {str}),
+                'format_id': ('label', {str}),
+                'ext': ('mime', {mimetype2ext}),
+                'tbr': ('bitrate', {int_or_none}, {lambda x: x or None}),
+                'vcodec': ('video_codecs', {str}),
+                'fps': ('fps', {int_or_none}),
+                'width': ('width', {int_or_none}),
+                'height': ('height', {int_or_none}),
+                'filesize': ('size', {int_or_none}),
+                'acodec': ('audio_codecs', {str}),
+                'asr': ('audio_sample_rate', {int_or_none}),
+                'audio_channels': ('audio_channels', {int_or_none}),
+            }))
+        if not formats:  # fallback, should be barely used
+            for url in set(traverse_obj(media_info, (..., {url_or_none}))):
+                if 'label=' in url:  # filter out non-video urls
+                    format_id, resolution = self._search_regex(
+                        r'label=(\w+)&template=(\d+x\d+)', url, 'format info',
+                        group=(1, 2), default=(None, None))
+                    formats.append({
+                        'url': url,
+                        'format_id': format_id,
+                        **parse_resolution(resolution),
+                        **traverse_obj(media_info, (
+                            'video_details', lambda _, v: v['label'].startswith(format_id), {
+                                'size': ('size', {int_or_none}),
+                                'tbr': ('bitrate', {int_or_none}),
+                            }
+                        ), get_all=False),
+                    })
+        return formats
+
+    def _parse_video_info(self, video_info, video_id=None):
         return {
             'id': video_id,
-            'title': title,
-            'uploader': uploader,
-            'formats': formats
+            'extractor_key': WeiboIE.ie_key(),
+            'extractor': WeiboIE.IE_NAME,
+            'formats': self._extract_formats(video_info),
+            'http_headers': {'Referer': 'https://weibo.com/'},
+            '_old_archive_ids': [make_archive_id('WeiboMobile', video_id)],
+            **traverse_obj(video_info, {
+                'id': (('id', 'id_str', 'mid'), {str_or_none}),
+                'display_id': ('mblogid', {str_or_none}),
+                'title': ('page_info', 'media_info', ('video_title', 'kol_title', 'name'), {str}, {lambda x: x or None}),
+                'description': ('text_raw', {str}),
+                'duration': ('page_info', 'media_info', 'duration', {int_or_none}),
+                'timestamp': ('page_info', 'media_info', 'video_publish_time', {int_or_none}),
+                'thumbnail': ('page_info', 'page_pic', {url_or_none}),
+                'uploader': ('user', 'screen_name', {str}),
+                'uploader_id': ('user', ('id', 'id_str'), {str_or_none}),
+                'uploader_url': ('user', 'profile_url', {lambda x: urljoin('https://weibo.com/', x)}),
+                'view_count': ('page_info', 'media_info', 'online_users_number', {int_or_none}),
+                'like_count': ('attitudes_count', {int_or_none}),
+                'repost_count': ('reposts_count', {int_or_none}),
+            }, get_all=False),
+            'tags': traverse_obj(video_info, ('topic_struct', ..., 'topic_title', {str})) or None,
         }
 
 
-class WeiboMobileIE(InfoExtractor):
-    _VALID_URL = r'https?://m\.weibo\.cn/status/(?P<id>[0-9]+)(\?.+)?'
-    _TEST = {
-        'url': 'https://m.weibo.cn/status/4189191225395228?wm=3333_2001&sourcetype=weixin&featurecode=newtitle&from=singlemessage&isappinstalled=0',
+class WeiboIE(WeiboBaseIE):
+    _VALID_URL = r'https?://(?:m\.weibo\.cn/status|(?:www\.)?weibo\.com/\d+)/(?P<id>[a-zA-Z0-9]+)'
+    _TESTS = [{
+        'url': 'https://weibo.com/7827771738/N4xlMvjhI',
+        'info_dict': {
+            'id': '4910815147462302',
+            'ext': 'mp4',
+            'display_id': 'N4xlMvjhI',
+            'title': '【睡前消息暑假版第一期：拉泰国一把  对中国有好处】',
+            'description': 'md5:e2637a7673980d68694ea7c43cf12a5f',
+            'duration': 918,
+            'timestamp': 1686312819,
+            'upload_date': '20230609',
+            'thumbnail': r're:https://.*\.jpg',
+            'uploader': '睡前视频基地',
+            'uploader_id': '7827771738',
+            'uploader_url': 'https://weibo.com/u/7827771738',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'tags': ['泰国大选远进党获胜', '睡前消息', '暑期版'],
+        },
+    }, {
+        'url': 'https://m.weibo.cn/status/4189191225395228',
         'info_dict': {
             'id': '4189191225395228',
             'ext': 'mp4',
-            'title': '午睡当然是要甜甜蜜蜜的啦',
-            'uploader': '柴犬柴犬'
+            'display_id': 'FBqgOmDxO',
+            'title': '柴犬柴犬的秒拍视频',
+            'description': 'md5:80f461ab5cdae6bbdb70efbf5a1db24f',
+            'duration': 53,
+            'timestamp': 1514264429,
+            'upload_date': '20171226',
+            'thumbnail': r're:https://.*\.jpg',
+            'uploader': '柴犬柴犬',
+            'uploader_id': '5926682210',
+            'uploader_url': 'https://weibo.com/u/5926682210',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
         }
-    }
+    }, {
+        'url': 'https://weibo.com/0/4224132150961381',
+        'note': 'no playback_list example',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        # to get Referer url for genvisitor
-        webpage = self._download_webpage(url, video_id, note='visit the page')
 
-        weibo_info = self._parse_json(self._search_regex(
-            r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\]\s*\|\|\s*{};',
-            webpage, 'js_code', flags=re.DOTALL),
-            video_id, transform_source=js_to_json)
+        return self._parse_video_info(self._weibo_download_json(
+            f'https://weibo.com/ajax/statuses/show?id={video_id}', video_id))
 
-        status_data = weibo_info.get('status', {})
-        page_info = status_data.get('page_info')
-        title = status_data['status_title']
-        uploader = status_data.get('user', {}).get('screen_name')
 
-        return {
-            'id': video_id,
-            'title': title,
-            'uploader': uploader,
-            'url': page_info['media_info']['stream_url']
+class WeiboVideoIE(WeiboBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?weibo\.com/tv/show/(?P<id>\d+:\d+)'
+    _TESTS = [{
+        'url': 'https://weibo.com/tv/show/1034:4797699866951785?from=old_pc_videoshow',
+        'info_dict': {
+            'id': '4797700463137878',
+            'ext': 'mp4',
+            'display_id': 'LEZDodaiW',
+            'title': '呃，稍微了解了一下靡烟miya，感觉这东西也太二了',
+            'description': '呃，稍微了解了一下靡烟miya，感觉这东西也太二了 http://t.cn/A6aerGsM ​​​',
+            'duration': 76,
+            'timestamp': 1659344278,
+            'upload_date': '20220801',
+            'thumbnail': r're:https://.*\.jpg',
+            'uploader': '君子爱财陈平安',
+            'uploader_id': '3905382233',
+            'uploader_url': 'https://weibo.com/u/3905382233',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
         }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        post_data = f'data={{"Component_Play_Playinfo":{{"oid":"{video_id}"}}}}'.encode()
+        video_info = self._weibo_download_json(
+            f'https://weibo.com/tv/api/component?page=%2Ftv%2Fshow%2F{video_id.replace(":", "%3A")}',
+            video_id, headers={'Referer': url}, data=post_data)['data']['Component_Play_Playinfo']
+        return self.url_result(f'https://weibo.com/0/{video_info["mid"]}', WeiboIE)
+
+
+class WeiboUserIE(WeiboBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?weibo\.com/u/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://weibo.com/u/2066652961?tabtype=video',
+        'info_dict': {
+            'id': '2066652961',
+            'title': '萧影殿下的视频',
+            'description': '萧影殿下的全部视频',
+            'uploader': '萧影殿下',
+        },
+        'playlist_mincount': 195,
+    }]
+
+    def _fetch_page(self, uid, cursor=0, page=1):
+        return self._weibo_download_json(
+            'https://weibo.com/ajax/profile/getWaterFallContent',
+            uid, note=f'Downloading videos page {page}',
+            query={'uid': uid, 'cursor': cursor})['data']
+
+    def _entries(self, uid, first_page):
+        cursor = 0
+        for page in itertools.count(1):
+            response = first_page if page == 1 else self._fetch_page(uid, cursor, page)
+            for video_info in traverse_obj(response, ('list', ..., {dict})):
+                yield self._parse_video_info(video_info)
+            cursor = response.get('next_cursor')
+            if (int_or_none(cursor) or -1) < 0:
+                break
+
+    def _real_extract(self, url):
+        uid = self._match_id(url)
+        first_page = self._fetch_page(uid)
+        uploader = traverse_obj(first_page, ('list', ..., 'user', 'screen_name', {str}), get_all=False)
+        metainfo = {
+            'title': f'{uploader}的视频',
+            'description': f'{uploader}的全部视频',
+            'uploader': uploader,
+        } if uploader else {}
+
+        return self.playlist_result(self._entries(uid, first_page), uid, **metainfo)

From 8ac5b6d96ae5c60cd5ae2495949e0068a6754c45 Mon Sep 17 00:00:00 2001
From: u-spec-png <srdjankalaba@protonmail.ch>
Date: Tue, 19 Sep 2023 01:36:10 +0200
Subject: [PATCH 569/871] [ie/N1Info:article] Fix extractor (#7373)

Authored by: u-spec-png
---
 yt_dlp/extractor/n1.py | 52 +++++++++++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 13 deletions(-)

diff --git a/yt_dlp/extractor/n1.py b/yt_dlp/extractor/n1.py
index 55345f398..edc41443a 100644
--- a/yt_dlp/extractor/n1.py
+++ b/yt_dlp/extractor/n1.py
@@ -33,7 +33,7 @@ def _real_extract(self, url):
 
 class N1InfoIIE(InfoExtractor):
     IE_NAME = 'N1Info:article'
-    _VALID_URL = r'https?://(?:(?:(?:ba|rs|hr)\.)?n1info\.(?:com|si)|nova\.rs)/(?:[^/]+/){1,2}(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:(?:\w+\.)?n1info\.\w+|nova\.rs)/(?:[^/?#]+/){1,2}(?P<id>[^/?#]+)'
     _TESTS = [{
         # Youtube embedded
         'url': 'https://rs.n1info.com/sport-klub/tenis/kako-je-djokovic-propustio-istorijsku-priliku-video/',
@@ -94,6 +94,16 @@ class N1InfoIIE(InfoExtractor):
             'upload_date': '20211102',
             'timestamp': 1635861677,
         },
+    }, {
+        'url': 'https://n1info.rs/vesti/cuta-biti-u-kosovskoj-mitrovici-znaci-da-te-docekaju-eksplozivnim-napravama/',
+        'info_dict': {
+            'id': '1332368',
+            'ext': 'mp4',
+            'title': 'Ćuta: Biti u Kosovskoj Mitrovici znači da te dočekaju eksplozivnim napravama',
+            'upload_date': '20230620',
+            'timestamp': 1687290536,
+            'thumbnail': 'https://cdn.brid.tv/live/partners/26827/snapshot/1332368_th_6492013a8356f_1687290170.jpg'
+        },
     }, {
         'url': 'https://hr.n1info.com/vijesti/pravobraniteljica-o-ubojstvu-u-zagrebu-radi-se-o-doista-nezapamcenoj-situaciji/',
         'only_matching': True,
@@ -105,19 +115,35 @@ def _real_extract(self, url):
 
         title = self._html_search_regex(r'<h1[^>]+>(.+?)</h1>', webpage, 'title')
         timestamp = unified_timestamp(self._html_search_meta('article:published_time', webpage))
-
-        videos = re.findall(r'(?m)(<video[^>]+>)', webpage)
+        plugin_data = self._html_search_meta('BridPlugin', webpage)
         entries = []
-        for video in videos:
-            video_data = extract_attributes(video)
-            entries.append({
-                '_type': 'url_transparent',
-                'url': video_data.get('data-url'),
-                'id': video_data.get('id'),
-                'title': title,
-                'thumbnail': video_data.get('data-thumbnail'),
-                'timestamp': timestamp,
-                'ie_key': 'N1InfoAsset'})
+        if plugin_data:
+            site_id = self._html_search_regex(r'site:(\d+)', webpage, 'site id')
+            for video_data in re.findall(r'\$bp\("Brid_\d+", (.+)\);', webpage):
+                video_id = self._parse_json(video_data, title)['video']
+                entries.append({
+                    'id': video_id,
+                    'title': title,
+                    'timestamp': timestamp,
+                    'thumbnail': self._html_search_meta('thumbnailURL', webpage),
+                    'formats': self._extract_m3u8_formats(
+                        f'https://cdn-uc.brid.tv/live/partners/{site_id}/streaming/{video_id}/{video_id}.m3u8',
+                        video_id, fatal=False),
+                })
+        else:
+            # Old player still present in older articles
+            videos = re.findall(r'(?m)(<video[^>]+>)', webpage)
+            for video in videos:
+                video_data = extract_attributes(video)
+                entries.append({
+                    '_type': 'url_transparent',
+                    'url': video_data.get('data-url'),
+                    'id': video_data.get('id'),
+                    'title': title,
+                    'thumbnail': video_data.get('data-thumbnail'),
+                    'timestamp': timestamp,
+                    'ie_key': 'N1InfoAsset',
+                })
 
         embedded_videos = re.findall(r'(<iframe[^>]+>)', webpage)
         for embedded_video in embedded_videos:

From 40999467f72db074a3f13057da9bf82a857530fe Mon Sep 17 00:00:00 2001
From: niemands <67282402+niemands@users.noreply.github.com>
Date: Tue, 19 Sep 2023 01:37:17 +0200
Subject: [PATCH 570/871] [ie/pornbox] Add extractor (#7386)

Authored by: niemands
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/pornbox.py     | 113 ++++++++++++++++++++++++++++++++
 2 files changed, 114 insertions(+)
 create mode 100644 yt_dlp/extractor/pornbox.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 47d983c9c..dd670d59c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1505,6 +1505,7 @@
 from .popcorntimes import PopcorntimesIE
 from .popcorntv import PopcornTVIE
 from .porn91 import Porn91IE
+from .pornbox import PornboxIE
 from .porncom import PornComIE
 from .pornflip import PornFlipIE
 from .pornhd import PornHdIE
diff --git a/yt_dlp/extractor/pornbox.py b/yt_dlp/extractor/pornbox.py
new file mode 100644
index 000000000..c381382e9
--- /dev/null
+++ b/yt_dlp/extractor/pornbox.py
@@ -0,0 +1,113 @@
+from .common import InfoExtractor
+from ..compat import functools
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+    qualities,
+    str_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class PornboxIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pornbox\.com/application/watch-page/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://pornbox.com/application/watch-page/212108',
+        'md5': '3ff6b6e206f263be4c5e987a3162ac6e',
+        'info_dict': {
+            'id': '212108',
+            'ext': 'mp4',
+            'title': 'md5:ececc5c6e6c9dd35d290c45fed05fd49',
+            'uploader': 'Lily Strong',
+            'timestamp': 1665871200,
+            'upload_date': '20221015',
+            'age_limit': 18,
+            'availability': 'needs_auth',
+            'duration': 1505,
+            'cast': ['Lily Strong', 'John Strong'],
+            'tags': 'count:11',
+            'description': 'md5:589c7f33e183aa8aa939537300efb859',
+            'thumbnail': r're:^https?://cdn-image\.gtflixtv\.com.*\.jpg.*$'
+        }
+    }, {
+        'url': 'https://pornbox.com/application/watch-page/216045',
+        'info_dict': {
+            'id': '216045',
+            'title': 'md5:3e48528e73a9a2b12f7a2772ed0b26a2',
+            'description': 'md5:3e631dcaac029f15ed434e402d1b06c7',
+            'uploader': 'VK Studio',
+            'timestamp': 1618264800,
+            'upload_date': '20210412',
+            'age_limit': 18,
+            'availability': 'premium_only',
+            'duration': 2710,
+            'cast': 'count:3',
+            'tags': 'count:29',
+            'thumbnail': r're:^https?://cdn-image\.gtflixtv\.com.*\.jpg.*$',
+            'subtitles': 'count:6'
+        },
+        'params': {
+            'skip_download': True,
+            'ignore_no_formats_error': True
+        },
+        'expected_warnings': [
+            'You are either not logged in or do not have access to this scene',
+            'No video formats found', 'Requested format is not available']
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        public_data = self._download_json(f'https://pornbox.com/contents/{video_id}', video_id)
+
+        subtitles = {country_code: [{
+            'url': f'https://pornbox.com/contents/{video_id}/subtitles/{country_code}',
+            'ext': 'srt'
+        }] for country_code in traverse_obj(public_data, ('subtitles', ..., {str}))}
+
+        is_free_scene = traverse_obj(
+            public_data, ('price', 'is_available_for_free', {bool}), default=False)
+
+        metadata = {
+            'id': video_id,
+            **traverse_obj(public_data, {
+                'title': ('scene_name', {str.strip}),
+                'description': ('small_description', {str.strip}),
+                'uploader': 'studio',
+                'duration': ('runtime', {parse_duration}),
+                'cast': (('models', 'male_models'), ..., 'model_name'),
+                'thumbnail': ('player_poster', {url_or_none}),
+                'tags': ('niches', ..., 'niche'),
+            }),
+            'age_limit': 18,
+            'timestamp': parse_iso8601(traverse_obj(
+                public_data, ('studios', 'release_date'), 'publish_date')),
+            'availability': self._availability(needs_auth=True, needs_premium=not is_free_scene),
+            'subtitles': subtitles,
+        }
+
+        if not public_data.get('is_purchased') or not is_free_scene:
+            self.raise_login_required(
+                'You are either not logged in or do not have access to this scene', metadata_available=True)
+            return metadata
+
+        media_id = traverse_obj(public_data, (
+            'medias', lambda _, v: v['title'] == 'Full video', 'media_id', {int}), get_all=False)
+        if not media_id:
+            self.raise_no_formats('Could not find stream id', video_id=video_id)
+
+        stream_data = self._download_json(
+            f'https://pornbox.com/media/{media_id}/stream', video_id=video_id, note='Getting manifest urls')
+
+        get_quality = qualities(['web', 'vga', 'hd', '1080p', '4k', '8k'])
+        metadata['formats'] = traverse_obj(stream_data, ('qualities', lambda _, v: v['src'], {
+            'url': 'src',
+            'vbr': ('bitrate', {functools.partial(int_or_none, scale=1000)}),
+            'format_id': ('quality', {str_or_none}),
+            'quality': ('quality', {get_quality}),
+            'width': ('size', {lambda x: int(x[:-1])}),
+        }))
+
+        return metadata

From cf11b40ac40e3d23a6352753296f3a732886efb9 Mon Sep 17 00:00:00 2001
From: Rohan Dey <142105763+Rohxn16@users.noreply.github.com>
Date: Mon, 18 Sep 2023 23:39:20 +0000
Subject: [PATCH 571/871] [ie/media.ccc.de:lists] Fix extraction (#8144)

Closes #8138
Authored by: Rohxn16
---
 yt_dlp/extractor/ccc.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/ccc.py b/yt_dlp/extractor/ccc.py
index 22e3a22ec..ca6b82c98 100644
--- a/yt_dlp/extractor/ccc.py
+++ b/yt_dlp/extractor/ccc.py
@@ -90,10 +90,17 @@ class CCCPlaylistIE(InfoExtractor):
             'id': '30c3',
         },
         'playlist_count': 135,
+    }, {
+        'url': 'https://media.ccc.de/c/DS2023',
+        'info_dict': {
+            'title': 'Datenspuren 2023',
+            'id': 'DS2023',
+        },
+        'playlist_count': 37
     }]
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url).lower()
+        playlist_id = self._match_id(url)
 
         conf = self._download_json(
             'https://media.ccc.de/public/conferences/' + playlist_id,

From b532556d0a85e7d76f8f0880861232fb706ddbc5 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 19 Sep 2023 21:52:44 +0200
Subject: [PATCH 572/871] [ie/pr0gramm] Rewrite extractor (#8151)

Authored by: Grub4K
---
 yt_dlp/extractor/_extractors.py |   2 +-
 yt_dlp/extractor/pr0gramm.py    | 218 ++++++++++++++++++++------------
 2 files changed, 139 insertions(+), 81 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index dd670d59c..490b010b8 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1524,7 +1524,7 @@
     PuhuTVIE,
     PuhuTVSerieIE,
 )
-from .pr0gramm import Pr0grammStaticIE, Pr0grammIE
+from .pr0gramm import Pr0grammIE
 from .prankcast import PrankCastIE
 from .premiershiprugby import PremiershipRugbyIE
 from .presstv import PressTVIE
diff --git a/yt_dlp/extractor/pr0gramm.py b/yt_dlp/extractor/pr0gramm.py
index 2eb327fba..c8e0bb493 100644
--- a/yt_dlp/extractor/pr0gramm.py
+++ b/yt_dlp/extractor/pr0gramm.py
@@ -1,97 +1,155 @@
-import re
+import json
+from datetime import date
+from urllib.parse import unquote
 
 from .common import InfoExtractor
-from ..utils import merge_dicts
+from ..compat import functools
+from ..utils import ExtractorError, make_archive_id, urljoin
+from ..utils.traversal import traverse_obj
 
 
-class Pr0grammStaticIE(InfoExtractor):
-    # Possible urls:
-    # https://pr0gramm.com/static/5466437
-    _VALID_URL = r'https?://pr0gramm\.com/static/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'https://pr0gramm.com/static/5466437',
-        'md5': '52fa540d70d3edc286846f8ca85938aa',
-        'info_dict': {
-            'id': '5466437',
-            'ext': 'mp4',
-            'title': 'pr0gramm-5466437 by g11st',
-            'uploader': 'g11st',
-            'upload_date': '20221221',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        # Fetch media sources
-        entries = self._parse_html5_media_entries(url, webpage, video_id)
-        media_info = entries[0]
-
-        # Fetch author
-        uploader = self._html_search_regex(r'by\W+([\w-]+)\W+', webpage, 'uploader')
-
-        # Fetch approx upload timestamp from filename
-        # Have None-defaults in case the extraction fails
-        uploadDay = None
-        uploadMon = None
-        uploadYear = None
-        uploadTimestr = None
-        # (//img.pr0gramm.com/2022/12/21/62ae8aa5e2da0ebf.mp4)
-        m = re.search(r'//img\.pr0gramm\.com/(?P<year>[\d]+)/(?P<mon>[\d]+)/(?P<day>[\d]+)/\w+\.\w{,4}', webpage)
-
-        if (m):
-            # Up to a day of accuracy should suffice...
-            uploadDay = m.groupdict().get('day')
-            uploadMon = m.groupdict().get('mon')
-            uploadYear = m.groupdict().get('year')
-            uploadTimestr = uploadYear + uploadMon + uploadDay
-
-        return merge_dicts({
-            'id': video_id,
-            'title': 'pr0gramm-%s%s' % (video_id, (' by ' + uploader) if uploader else ''),
-            'uploader': uploader,
-            'upload_date': uploadTimestr
-        }, media_info)
-
-
-# This extractor is for the primary url (used for sharing, and appears in the
-# location bar) Since this page loads the DOM via JS, yt-dl can't find any
-# video information here. So let's redirect to a compatibility version of
-# the site, which does contain the <video>-element  by itself,  without requiring
-# js to be ran.
 class Pr0grammIE(InfoExtractor):
-    # Possible urls:
-    # https://pr0gramm.com/new/546637
-    # https://pr0gramm.com/new/video/546637
-    # https://pr0gramm.com/top/546637
-    # https://pr0gramm.com/top/video/546637
-    # https://pr0gramm.com/user/g11st/uploads/5466437
-    # https://pr0gramm.com/user/froschler/dafur-ist-man-hier/5091290
-    # https://pr0gramm.com/user/froschler/reinziehen-1elf/5232030
-    # https://pr0gramm.com/user/froschler/1elf/5232030
-    # https://pr0gramm.com/new/5495710:comment62621020 <- this is not the id!
-    # https://pr0gramm.com/top/fruher war alles damals/5498175
-
-    _VALID_URL = r'https?:\/\/pr0gramm\.com\/(?!static/\d+).+?\/(?P<id>[\d]+)(:|$)'
-    _TEST = {
+    _VALID_URL = r'https?://pr0gramm\.com\/(?:[^/?#]+/)+(?P<id>[\d]+)(?:[/?#:]|$)'
+    _TESTS = [{
+        # Tags require account
         'url': 'https://pr0gramm.com/new/video/5466437',
         'info_dict': {
             'id': '5466437',
             'ext': 'mp4',
             'title': 'pr0gramm-5466437 by g11st',
+            'tags': ['Neon Genesis Evangelion', 'Touhou Project', 'Fly me to the Moon', 'Marisad', 'Marisa Kirisame', 'video', 'sound', 'Marisa', 'Anime'],
             'uploader': 'g11st',
+            'uploader_id': 394718,
+            'upload_timestamp': 1671590240,
             'upload_date': '20221221',
-        }
-    }
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 0,
+            'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
+        },
+    }, {
+        # Tags require account
+        'url': 'https://pr0gramm.com/new/3052805:comment28391322',
+        'info_dict': {
+            'id': '3052805',
+            'ext': 'mp4',
+            'title': 'pr0gramm-3052805 by Hansking1',
+            'tags': 'count:15',
+            'uploader': 'Hansking1',
+            'uploader_id': 385563,
+            'upload_timestamp': 1552930408,
+            'upload_date': '20190318',
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 0,
+            'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
+        },
+    }, {
+        # Requires verified account
+        'url': 'https://pr0gramm.com/new/Gianna%20Michaels/5848332',
+        'info_dict': {
+            'id': '5848332',
+            'ext': 'mp4',
+            'title': 'pr0gramm-5848332 by erd0pfel',
+            'tags': 'count:18',
+            'uploader': 'erd0pfel',
+            'uploader_id': 349094,
+            'upload_timestamp': 1694489652,
+            'upload_date': '20230912',
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 18,
+            'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
+        },
+    }, {
+        'url': 'https://pr0gramm.com/static/5466437',
+        'only_matching': True,
+    }, {
+        'url': 'https://pr0gramm.com/new/rowan%20atkinson%20herr%20bohne/3052805',
+        'only_matching': True,
+    }, {
+        'url': 'https://pr0gramm.com/user/froschler/dafur-ist-man-hier/5091290',
+        'only_matching': True,
+    }]
 
-    def _generic_title():
-        return "oof"
+    BASE_URL = 'https://pr0gramm.com'
+
+    @functools.cached_property
+    def _is_logged_in(self):
+        return 'pp' in self._get_cookies(self.BASE_URL)
+
+    @functools.cached_property
+    def _maximum_flags(self):
+        # We need to guess the flags for the content otherwise the api will raise an error
+        # We can guess the maximum allowed flags for the account from the cookies
+        # Bitflags are (msbf): nsfp, nsfl, nsfw, sfw
+        flags = 0b0001
+        if self._is_logged_in:
+            flags |= 0b1000
+            cookies = self._get_cookies(self.BASE_URL)
+            if 'me' not in cookies:
+                self._download_webpage(self.BASE_URL, None, 'Refreshing verification information')
+            if traverse_obj(cookies, ('me', {lambda x: x.value}, {unquote}, {json.loads}, 'verified')):
+                flags |= 0b0110
+
+        return flags
+
+    def _call_api(self, endpoint, video_id, query={}, note='Downloading API json'):
+        data = self._download_json(
+            f'https://pr0gramm.com/api/items/{endpoint}',
+            video_id, note, query=query, expected_status=403)
+
+        error = traverse_obj(data, ('error', {str}))
+        if error in ('nsfwRequired', 'nsflRequired', 'nsfpRequired', 'verificationRequired'):
+            if not self._is_logged_in:
+                self.raise_login_required()
+            raise ExtractorError(f'Unverified account cannot access NSFW/NSFL ({error})', expected=True)
+        elif error:
+            message = traverse_obj(data, ('msg', {str})) or error
+            raise ExtractorError(f'API returned error: {message}', expected=True)
+
+        return data
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        video_info = traverse_obj(
+            self._call_api('get', video_id, {'id': video_id, 'flags': self._maximum_flags}),
+            ('items', 0, {dict}))
 
-        return self.url_result(
-            'https://pr0gramm.com/static/' + video_id,
-            video_id=video_id,
-            ie=Pr0grammStaticIE.ie_key())
+        source = urljoin('https://img.pr0gramm.com', video_info.get('image'))
+        if not source or not source.endswith('mp4'):
+            self.raise_no_formats('Could not extract a video', expected=bool(source), video_id=video_id)
+
+        tags = None
+        if self._is_logged_in:
+            metadata = self._call_api('info', video_id, {'itemId': video_id})
+            tags = traverse_obj(metadata, ('tags', ..., 'tag', {str}))
+            # Sorted by "confidence", higher confidence = earlier in list
+            confidences = traverse_obj(metadata, ('tags', ..., 'confidence', ({int}, {float})))
+            if confidences:
+                tags = [tag for _, tag in sorted(zip(confidences, tags), reverse=True)]
+
+        return {
+            'id': video_id,
+            'title': f'pr0gramm-{video_id} by {video_info.get("user")}',
+            'formats': [{
+                'url': source,
+                'ext': 'mp4',
+                **traverse_obj(video_info, {
+                    'width': ('width', {int}),
+                    'height': ('height', {int}),
+                }),
+            }],
+            'tags': tags,
+            'age_limit': 18 if traverse_obj(video_info, ('flags', {0b110.__and__})) else 0,
+            '_old_archive_ids': [make_archive_id('Pr0grammStatic', video_id)],
+            **traverse_obj(video_info, {
+                'uploader': ('user', {str}),
+                'uploader_id': ('userId', {int}),
+                'like_count': ('up', {int}),
+                'dislike_count': ('down', {int}),
+                'upload_timestamp': ('created', {int}),
+                'upload_date': ('created', {int}, {date.fromtimestamp}, {lambda x: x.strftime('%Y%m%d')}),
+                'thumbnail': ('thumb', {lambda x: urljoin('https://thumb.pr0gramm.com', x)})
+            }),
+        }

From 9d6254069c75877bc88bc3584f4326fb1853a543 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Wed, 20 Sep 2023 19:14:10 +0000
Subject: [PATCH 573/871] Update to ytdl-commit-66ab08 (#8128)

[utils] Revert bbd3e7e, updating docstring, test instead
 https://github.com/ytdl-org/youtube-dl/commit/66ab0814c4baa2dc79c2dd5287bc0ad61a37c5b9

Authored by: coletdjnz
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c7b73f4fd..d94d8ea82 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,7 @@
 
 # NEW FEATURES
 
-* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@42f2d4**](https://github.com/ytdl-org/youtube-dl/commit/07af47960f3bb262ead02490ce65c8c45c01741e) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
+* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@66ab08**](https://github.com/ytdl-org/youtube-dl/commit/66ab0814c4baa2dc79c2dd5287bc0ad61a37c5b9) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 

From 35f9a306e6934793cff100200cd03f288ec33f11 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 21 Sep 2023 10:58:53 -0500
Subject: [PATCH 574/871] [dependencies] Handle deprecation of
 `sqlite3.version` (#8167)

Closes #8152
Authored by: bashonly
---
 yt_dlp/compat/compat_utils.py   | 2 +-
 yt_dlp/dependencies/__init__.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/compat/compat_utils.py b/yt_dlp/compat/compat_utils.py
index 3ca46d270..d62b7d048 100644
--- a/yt_dlp/compat/compat_utils.py
+++ b/yt_dlp/compat/compat_utils.py
@@ -15,7 +15,7 @@ def get_package_info(module):
         name=getattr(module, '_yt_dlp__identifier', module.__name__),
         version=str(next(filter(None, (
             getattr(module, attr, None)
-            for attr in ('__version__', 'version_string', 'version')
+            for attr in ('_yt_dlp__version', '__version__', 'version_string', 'version')
         )), None)))
 
 
diff --git a/yt_dlp/dependencies/__init__.py b/yt_dlp/dependencies/__init__.py
index 6e7d29c5c..b56e4f5cc 100644
--- a/yt_dlp/dependencies/__init__.py
+++ b/yt_dlp/dependencies/__init__.py
@@ -43,6 +43,8 @@
 
 try:
     import sqlite3
+    # We need to get the underlying `sqlite` version, see https://github.com/yt-dlp/yt-dlp/issues/8152
+    sqlite3._yt_dlp__version = sqlite3.sqlite_version
 except ImportError:
     # although sqlite3 is part of the standard library, it is possible to compile python without
     # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544

From 295fbb3ae3a7d0dd50e286be5c487cf145ed5778 Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Fri, 22 Sep 2023 01:28:20 +0800
Subject: [PATCH 575/871] [ie/eplus:inbound] Add extractor (#5782)

Authored by: pzhlkj6612
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/eplus.py       | 96 +++++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100644 yt_dlp/extractor/eplus.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 490b010b8..3ce6baef2 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -565,6 +565,7 @@
     EpiconIE,
     EpiconSeriesIE,
 )
+from .eplus import EplusIbIE
 from .epoch import EpochIE
 from .eporner import EpornerIE
 from .eroprofile import (
diff --git a/yt_dlp/extractor/eplus.py b/yt_dlp/extractor/eplus.py
new file mode 100644
index 000000000..3ebdcf5fb
--- /dev/null
+++ b/yt_dlp/extractor/eplus.py
@@ -0,0 +1,96 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    try_call,
+    unified_timestamp,
+)
+
+
+class EplusIbIE(InfoExtractor):
+    IE_NAME = 'eplus:inbound'
+    IE_DESC = 'e+ (イープラス) overseas'
+    _VALID_URL = r'https?://live\.eplus\.jp/ex/player\?ib=(?P<id>(?:\w|%2B|%2F){86}%3D%3D)'
+    _TESTS = [{
+        'url': 'https://live.eplus.jp/ex/player?ib=YEFxb3Vyc2Dombnjg7blkrLlrablnJLjgrnjgq%2Fjg7zjg6vjgqLjgqTjg4njg6vlkIzlpb3kvJpgTGllbGxhIQ%3D%3D',
+        'info_dict': {
+            'id': '354502-0001-002',
+            'title': 'LoveLive!Series Presents COUNTDOWN LoveLive! 2021→2022～LIVE with a smile!～【Streaming+(配信)】',
+            'live_status': 'was_live',
+            'release_date': '20211231',
+            'release_timestamp': 1640952000,
+            'description': str,
+        },
+        'params': {
+            'skip_download': True,
+            'ignore_no_formats_error': True,
+        },
+        'expected_warnings': [
+            'Could not find the playlist URL. This event may not be accessible',
+            'No video formats found!',
+            'Requested format is not available',
+        ],
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        data_json = self._search_json(r'<script>\s*var app\s*=', webpage, 'data json', video_id)
+
+        delivery_status = data_json.get('delivery_status')
+        archive_mode = data_json.get('archive_mode')
+        release_timestamp = try_call(lambda: unified_timestamp(data_json['event_datetime']) - 32400)
+        release_timestamp_str = data_json.get('event_datetime_text')  # JST
+
+        self.write_debug(f'delivery_status = {delivery_status}, archive_mode = {archive_mode}')
+
+        if delivery_status == 'PREPARING':
+            live_status = 'is_upcoming'
+        elif delivery_status == 'STARTED':
+            live_status = 'is_live'
+        elif delivery_status == 'STOPPED':
+            if archive_mode != 'ON':
+                raise ExtractorError(
+                    'This event has ended and there is no archive for this event', expected=True)
+            live_status = 'post_live'
+        elif delivery_status == 'WAIT_CONFIRM_ARCHIVED':
+            live_status = 'post_live'
+        elif delivery_status == 'CONFIRMED_ARCHIVE':
+            live_status = 'was_live'
+        else:
+            self.report_warning(f'Unknown delivery_status {delivery_status}, treat it as a live')
+            live_status = 'is_live'
+
+        formats = []
+
+        m3u8_playlist_urls = self._search_json(
+            r'var listChannels\s*=', webpage, 'hls URLs', video_id, contains_pattern=r'\[.+\]', default=[])
+        if not m3u8_playlist_urls:
+            if live_status == 'is_upcoming':
+                self.raise_no_formats(
+                    f'Could not find the playlist URL. This live event will begin at {release_timestamp_str} JST', expected=True)
+            else:
+                self.raise_no_formats(
+                    'Could not find the playlist URL. This event may not be accessible', expected=True)
+        elif live_status == 'is_upcoming':
+            self.raise_no_formats(f'This live event will begin at {release_timestamp_str} JST', expected=True)
+        elif live_status == 'post_live':
+            self.raise_no_formats('This event has ended, and the archive will be available shortly', expected=True)
+        else:
+            for m3u8_playlist_url in m3u8_playlist_urls:
+                formats.extend(self._extract_m3u8_formats(m3u8_playlist_url, video_id))
+            # FIXME: HTTP request headers need to be updated to continue download
+            warning = 'Due to technical limitations, the download will be interrupted after one hour'
+            if live_status == 'is_live':
+                self.report_warning(warning)
+            elif live_status == 'was_live':
+                self.report_warning(f'{warning}. You can restart to continue the download')
+
+        return {
+            'id': data_json['app_id'],
+            'title': data_json.get('app_name'),
+            'formats': formats,
+            'live_status': live_status,
+            'description': data_json.get('content'),
+            'release_timestamp': release_timestamp,
+        }

From b3febedbeb662dfdf9b5c1d5799039ad4fc969de Mon Sep 17 00:00:00 2001
From: Elyse <26639800+elyse0@users.noreply.github.com>
Date: Thu, 21 Sep 2023 11:30:32 -0600
Subject: [PATCH 576/871] [ie/Canal1,CaracolTvPlay] Add extractors (#7151)

Closes #5826
Authored by: elyse0
---
 yt_dlp/extractor/_extractors.py |   2 +
 yt_dlp/extractor/canal1.py      |  39 +++++++++
 yt_dlp/extractor/caracoltv.py   | 136 ++++++++++++++++++++++++++++++++
 yt_dlp/extractor/mediastream.py |   8 +-
 4 files changed, 183 insertions(+), 2 deletions(-)
 create mode 100644 yt_dlp/extractor/canal1.py
 create mode 100644 yt_dlp/extractor/caracoltv.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 3ce6baef2..632d6720e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -296,9 +296,11 @@
 from .camsoda import CamsodaIE
 from .camtasia import CamtasiaEmbedIE
 from .camwithher import CamWithHerIE
+from .canal1 import Canal1IE
 from .canalalpha import CanalAlphaIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
+from .caracoltv import CaracolTvPlayIE
 from .carambatv import (
     CarambaTVIE,
     CarambaTVPageIE,
diff --git a/yt_dlp/extractor/canal1.py b/yt_dlp/extractor/canal1.py
new file mode 100644
index 000000000..587a11ab8
--- /dev/null
+++ b/yt_dlp/extractor/canal1.py
@@ -0,0 +1,39 @@
+from .common import InfoExtractor
+
+
+class Canal1IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.|noticias\.)?canal1\.com\.co/(?:[^?#&])+/(?P<id>[\w-]+)'
+
+    _TESTS = [{
+        'url': 'https://canal1.com.co/noticias/napa-i-una-cadena-de-produccion-de-arroz-que-se-quedo-en-veremos-y-abandonada-en-el-departamento-del-choco/',
+        'info_dict': {
+            'id': '63b39f6b354977084b85ab54',
+            'display_id': 'napa-i-una-cadena-de-produccion-de-arroz-que-se-quedo-en-veremos-y-abandonada-en-el-departamento-del-choco',
+            'title': 'Ñapa I Una cadena de producción de arroz que se quedó en veremos y abandonada en el departamento del Chocó',
+            'description': 'md5:bc49c6d64d20610ea1e7daf079a0d013',
+            'thumbnail': r're:^https?://[^?#]+63b39f6b354977084b85ab54',
+            'ext': 'mp4',
+        },
+    }, {
+        'url': 'https://noticias.canal1.com.co/noticias/tres-i-el-triste-record-que-impuso-elon-musk-el-dueno-de-tesla-y-de-twitter/',
+        'info_dict': {
+            'id': '63b39e93f5fd223aa32250fb',
+            'display_id': 'tres-i-el-triste-record-que-impuso-elon-musk-el-dueno-de-tesla-y-de-twitter',
+            'title': 'Tres I El triste récord que impuso Elon Musk, el dueño de Tesla y de Twitter',
+            'description': 'md5:d9f691f131a21ce6767ca6c05d17d791',
+            'thumbnail': r're:^https?://[^?#]+63b39e93f5fd223aa32250fb',
+            'ext': 'mp4',
+        },
+    }, {
+        # Geo-restricted to Colombia
+        'url': 'https://canal1.com.co/programas/guerreros-canal-1/video-inedito-guerreros-despedida-kewin-zarate/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        return self.url_result(
+            self._search_regex(r'"embedUrl"\s*:\s*"([^"]+)', webpage, 'embed url'),
+            display_id=display_id, url_transparent=True)
diff --git a/yt_dlp/extractor/caracoltv.py b/yt_dlp/extractor/caracoltv.py
new file mode 100644
index 000000000..79f7752fe
--- /dev/null
+++ b/yt_dlp/extractor/caracoltv.py
@@ -0,0 +1,136 @@
+import base64
+import json
+import uuid
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    traverse_obj,
+    urljoin,
+)
+
+
+class CaracolTvPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://play\.caracoltv\.com/videoDetails/(?P<id>[^/?#]+)'
+    _NETRC_MACHINE = 'caracoltv-play'
+
+    _TESTS = [{
+        'url': 'https://play.caracoltv.com/videoDetails/OTo4NGFmNjUwOWQ2ZmM0NTg2YWRiOWU0MGNhOWViOWJkYQ==',
+        'info_dict': {
+            'id': 'OTo4NGFmNjUwOWQ2ZmM0NTg2YWRiOWU0MGNhOWViOWJkYQ==',
+            'title': 'La teoría del promedio',
+            'description': 'md5:1cdd6d2c13f19ef0d9649ab81a023ac3',
+        },
+        'playlist_count': 6,
+    }, {
+        'url': 'https://play.caracoltv.com/videoDetails/OTo3OWM4ZTliYzQxMmM0MTMxYTk4Mjk2YjdjNGQ4NGRkOQ==/ella?season=0',
+        'info_dict': {
+            'id': 'OTo3OWM4ZTliYzQxMmM0MTMxYTk4Mjk2YjdjNGQ4NGRkOQ==',
+            'title': 'Ella',
+            'description': 'md5:a639b1feb5ddcc0cff92a489b4e544b8',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'https://play.caracoltv.com/videoDetails/OTpiYTY1YTVmOTI5MzI0ZWJhOGZiY2Y3MmRlOWZlYmJkOA==/la-vuelta-al-mundo-en-80-risas-2022?season=0',
+        'info_dict': {
+            'id': 'OTpiYTY1YTVmOTI5MzI0ZWJhOGZiY2Y3MmRlOWZlYmJkOA==',
+            'title': 'La vuelta al mundo en 80 risas 2022',
+            'description': 'md5:e97aac36106e5c37ebf947b3350106a4',
+        },
+        'playlist_count': 17,
+    }, {
+        'url': 'https://play.caracoltv.com/videoDetails/MzoxX3BwbjRmNjB1',
+        'only_matching': True,
+    }]
+
+    _USER_TOKEN = None
+
+    def _extract_app_token(self, webpage):
+        config_js_path = self._search_regex(
+            r'<script[^>]+src\s*=\s*"([^"]+coreConfig.js[^"]+)', webpage, 'config js url', fatal=False)
+
+        mediation_config = {} if not config_js_path else self._search_json(
+            r'mediation\s*:', self._download_webpage(
+                urljoin('https://play.caracoltv.com/', config_js_path), None, fatal=False, note='Extracting JS config'),
+            'mediation_config', None, transform_source=js_to_json, fatal=False)
+
+        key = traverse_obj(
+            mediation_config, ('live', 'key')) or '795cd9c089a1fc48094524a5eba85a3fca1331817c802f601735907c8bbb4f50'
+        secret = traverse_obj(
+            mediation_config, ('live', 'secret')) or '64dec00a6989ba83d087621465b5e5d38bdac22033b0613b659c442c78976fa0'
+
+        return base64.b64encode(f'{key}:{secret}'.encode()).decode()
+
+    def _perform_login(self, email, password):
+        webpage = self._download_webpage('https://play.caracoltv.com/', None, fatal=False)
+        app_token = self._extract_app_token(webpage)
+
+        bearer_token = self._download_json(
+            'https://eu-gateway.inmobly.com/applications/oauth', None, data=b'', note='Retrieving bearer token',
+            headers={'Authorization': f'Basic {app_token}'})['token']
+
+        self._USER_TOKEN = self._download_json(
+            'https://eu-gateway.inmobly.com/user/login', None, note='Performing login', headers={
+                'Content-Type': 'application/json',
+                'Authorization': f'Bearer {bearer_token}',
+            }, data=json.dumps({
+                'device_data': {
+                    'device_id': str(uuid.uuid4()),
+                    'device_token': '',
+                    'device_type': 'web'
+                },
+                'login_data': {
+                    'enabled': True,
+                    'email': email,
+                    'password': password,
+                }
+            }).encode())['user_token']
+
+    def _extract_video(self, video_data, series_id=None, season_id=None, season_number=None):
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_data['stream_url'], series_id, 'mp4')
+
+        return {
+            'id': video_data['id'],
+            'title': video_data.get('name'),
+            'description': video_data.get('description'),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnails': traverse_obj(
+                video_data, ('extra_thumbs', ..., {'url': 'thumb_url', 'height': 'height', 'width': 'width'})),
+            'series_id': series_id,
+            'season_id': season_id,
+            'season_number': int_or_none(season_number),
+            'episode_number': int_or_none(video_data.get('item_order')),
+            'is_live': video_data.get('entry_type') == 3,
+        }
+
+    def _extract_series_seasons(self, seasons, series_id):
+        for season in seasons:
+            api_response = self._download_json(
+                'https://eu-gateway.inmobly.com/feed', series_id, query={'season_id': season['id']},
+                headers={'Authorization': f'Bearer {self._USER_TOKEN}'})
+
+            season_number = season.get('order')
+            for episode in api_response['items']:
+                yield self._extract_video(episode, series_id, season['id'], season_number)
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+
+        if self._USER_TOKEN is None:
+            self._perform_login('guest@inmobly.com', 'Test@gus1')
+
+        api_response = self._download_json(
+            'https://eu-gateway.inmobly.com/feed', series_id, query={'include_ids': series_id},
+            headers={'Authorization': f'Bearer {self._USER_TOKEN}'})['items'][0]
+
+        if not api_response.get('seasons'):
+            return self._extract_video(api_response)
+
+        return self.playlist_result(
+            self._extract_series_seasons(api_response['seasons'], series_id),
+            series_id, **traverse_obj(api_response, {
+                'title': 'name',
+                'description': 'description',
+            }))
diff --git a/yt_dlp/extractor/mediastream.py b/yt_dlp/extractor/mediastream.py
index d5c9aab8a..b8cb5a691 100644
--- a/yt_dlp/extractor/mediastream.py
+++ b/yt_dlp/extractor/mediastream.py
@@ -106,8 +106,12 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        if 'Debido a tu ubicación no puedes ver el contenido' in webpage:
-            self.raise_geo_restricted()
+        for message in [
+            'Debido a tu ubicación no puedes ver el contenido',
+            'You are not allowed to watch this video: Geo Fencing Restriction'
+        ]:
+            if message in webpage:
+                self.raise_geo_restricted()
 
         player_config = self._search_json(r'window\.MDSTRM\.OPTIONS\s*=', webpage, 'metadata', video_id)
 

From 21f40e75dfc0055ea9cdbd7fe2c46c6f9b561afd Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Thu, 21 Sep 2023 13:34:35 -0400
Subject: [PATCH 577/871] [ie/douyutv] Fix extractors (#7652)

Closes #2494, Closes #7295
Authored by: c-basalt
---
 yt_dlp/extractor/douyutv.py | 273 ++++++++++++++++++++++++------------
 1 file changed, 184 insertions(+), 89 deletions(-)

diff --git a/yt_dlp/extractor/douyutv.py b/yt_dlp/extractor/douyutv.py
index fa40844df..ee8893d5a 100644
--- a/yt_dlp/extractor/douyutv.py
+++ b/yt_dlp/extractor/douyutv.py
@@ -1,31 +1,72 @@
 import time
 import hashlib
-import re
 import urllib
+import uuid
 
 from .common import InfoExtractor
+from .openload import PhantomJSwrapper
 from ..utils import (
     ExtractorError,
+    UserNotLive,
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    parse_resolution,
+    str_or_none,
+    traverse_obj,
     unescapeHTML,
-    unified_strdate,
+    url_or_none,
+    urlencode_postdata,
     urljoin,
 )
 
 
-class DouyuTVIE(InfoExtractor):
-    IE_DESC = '斗鱼'
+class DouyuBaseIE(InfoExtractor):
+    def _download_cryptojs_md5(self, video_id):
+        for url in [
+            'https://cdnjs.cloudflare.com/ajax/libs/crypto-js/3.1.2/rollups/md5.js',
+            'https://cdn.bootcdn.net/ajax/libs/crypto-js/3.1.2/rollups/md5.js',
+        ]:
+            js_code = self._download_webpage(
+                url, video_id, note='Downloading signing dependency', fatal=False)
+            if js_code:
+                self.cache.store('douyu', 'crypto-js-md5', js_code)
+                return js_code
+        raise ExtractorError('Unable to download JS dependency (crypto-js/md5)')
+
+    def _get_cryptojs_md5(self, video_id):
+        return self.cache.load('douyu', 'crypto-js-md5') or self._download_cryptojs_md5(video_id)
+
+    def _calc_sign(self, sign_func, video_id, a):
+        b = uuid.uuid4().hex
+        c = round(time.time())
+        js_script = f'{self._get_cryptojs_md5(video_id)};{sign_func};console.log(ub98484234("{a}","{b}","{c}"))'
+        phantom = PhantomJSwrapper(self)
+        result = phantom.execute(js_script, video_id,
+                                 note='Executing JS signing script').strip()
+        return {i: v[0] for i, v in urllib.parse.parse_qs(result).items()}
+
+    def _search_js_sign_func(self, webpage, fatal=True):
+        # The greedy look-behind ensures last possible script tag is matched
+        return self._search_regex(
+            r'(?:<script.*)?<script[^>]*>(.*?ub98484234.*?)</script>', webpage, 'JS sign func', fatal=fatal)
+
+
+class DouyuTVIE(DouyuBaseIE):
+    IE_DESC = '斗鱼直播'
     _VALID_URL = r'https?://(?:www\.)?douyu(?:tv)?\.com/(topic/\w+\?rid=|(?:[^/]+/))*(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
-        'url': 'http://www.douyutv.com/iseven',
+        'url': 'https://www.douyu.com/pigff',
         'info_dict': {
-            'id': '17732',
-            'display_id': 'iseven',
-            'ext': 'flv',
-            'title': 're:^清晨醒脑！根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': r're:.*m7show@163\.com.*',
-            'thumbnail': r're:^https?://.*\.png',
-            'uploader': '7师傅',
+            'id': '24422',
+            'display_id': 'pigff',
+            'ext': 'mp4',
+            'title': 're:^【PIGFF】.* [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': r'≥15级牌子看鱼吧置顶帖进粉丝vx群',
+            'thumbnail': str,
+            'uploader': 'pigff',
             'is_live': True,
+            'live_status': 'is_live',
         },
         'params': {
             'skip_download': True,
@@ -85,15 +126,43 @@ class DouyuTVIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _get_sign_func(self, room_id, video_id):
+        return self._download_json(
+            f'https://www.douyu.com/swf_api/homeH5Enc?rids={room_id}', video_id,
+            note='Getting signing script')['data'][f'room{room_id}']
+
+    def _extract_stream_formats(self, stream_formats):
+        formats = []
+        for stream_info in traverse_obj(stream_formats, (..., 'data')):
+            stream_url = urljoin(
+                traverse_obj(stream_info, 'rtmp_url'), traverse_obj(stream_info, 'rtmp_live'))
+            if stream_url:
+                rate_id = traverse_obj(stream_info, ('rate', {int_or_none}))
+                rate_info = traverse_obj(stream_info, ('multirates', lambda _, v: v['rate'] == rate_id), get_all=False)
+                ext = determine_ext(stream_url)
+                formats.append({
+                    'url': stream_url,
+                    'format_id': str_or_none(rate_id),
+                    'ext': 'mp4' if ext == 'm3u8' else ext,
+                    'protocol': 'm3u8_native' if ext == 'm3u8' else 'https',
+                    'quality': rate_id % -10000 if rate_id is not None else None,
+                    **traverse_obj(rate_info, {
+                        'format': ('name', {str_or_none}),
+                        'tbr': ('bit', {int_or_none}),
+                    }),
+                })
+        return formats
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        if video_id.isdigit():
-            room_id = video_id
-        else:
-            page = self._download_webpage(url, video_id)
-            room_id = self._html_search_regex(
-                r'"room_id\\?"\s*:\s*(\d+),', page, 'room id')
+        webpage = self._download_webpage(url, video_id)
+        room_id = self._search_regex(r'\$ROOM\.room_id\s*=\s*(\d+)', webpage, 'room id')
+
+        if self._search_regex(r'"videoLoop"\s*:\s*(\d+)', webpage, 'loop', default='') == '1':
+            raise UserNotLive('The channel is auto-playing VODs', video_id=video_id)
+        if self._search_regex(r'\$ROOM\.show_status\s*=\s*(\d+)', webpage, 'status', default='') == '2':
+            raise UserNotLive(video_id=video_id)
 
         # Grab metadata from API
         params = {
@@ -102,110 +171,136 @@ def _real_extract(self, url):
             'time': int(time.time()),
         }
         params['auth'] = hashlib.md5(
-            f'room/{video_id}?{urllib.parse.urlencode(params)}zNzMV1y4EMxOHS6I5WKm'.encode()).hexdigest()
-        room = self._download_json(
+            f'room/{room_id}?{urllib.parse.urlencode(params)}zNzMV1y4EMxOHS6I5WKm'.encode()).hexdigest()
+        room = traverse_obj(self._download_json(
             f'http://www.douyutv.com/api/v1/room/{room_id}', video_id,
-            note='Downloading room info', query=params)['data']
+            note='Downloading room info', query=params, fatal=False), 'data')
 
         # 1 = live, 2 = offline
-        if room.get('show_status') == '2':
-            raise ExtractorError('Live stream is offline', expected=True)
+        if traverse_obj(room, 'show_status') == '2':
+            raise UserNotLive(video_id=video_id)
 
-        video_url = urljoin('https://hls3-akm.douyucdn.cn/', self._search_regex(r'(live/.*)', room['hls_url'], 'URL'))
-        formats, subs = self._extract_m3u8_formats_and_subtitles(video_url, room_id)
+        js_sign_func = self._search_js_sign_func(webpage, fatal=False) or self._get_sign_func(room_id, video_id)
+        form_data = {
+            'rate': 0,
+            **self._calc_sign(js_sign_func, video_id, room_id),
+        }
+        stream_formats = [self._download_json(
+            f'https://www.douyu.com/lapi/live/getH5Play/{room_id}',
+            video_id, note="Downloading livestream format",
+            data=urlencode_postdata(form_data))]
 
-        title = unescapeHTML(room['room_name'])
-        description = room.get('show_details')
-        thumbnail = room.get('room_src')
-        uploader = room.get('nickname')
+        for rate_id in traverse_obj(stream_formats[0], ('data', 'multirates', ..., 'rate')):
+            if rate_id != traverse_obj(stream_formats[0], ('data', 'rate')):
+                form_data['rate'] = rate_id
+                stream_formats.append(self._download_json(
+                    f'https://www.douyu.com/lapi/live/getH5Play/{room_id}',
+                    video_id, note=f'Downloading livestream format {rate_id}',
+                    data=urlencode_postdata(form_data)))
 
         return {
             'id': room_id,
-            'display_id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
+            'formats': self._extract_stream_formats(stream_formats),
             'is_live': True,
-            'subtitles': subs,
-            'formats': formats,
+            **traverse_obj(room, {
+                'display_id': ('url', {str}, {lambda i: i[1:]}),
+                'title': ('room_name', {unescapeHTML}),
+                'description': ('show_details', {str}),
+                'uploader': ('nickname', {str}),
+                'thumbnail': ('room_src', {url_or_none}),
+            })
         }
 
 
-class DouyuShowIE(InfoExtractor):
+class DouyuShowIE(DouyuBaseIE):
     _VALID_URL = r'https?://v(?:mobile)?\.douyu\.com/show/(?P<id>[0-9a-zA-Z]+)'
 
     _TESTS = [{
-        'url': 'https://v.douyu.com/show/rjNBdvnVXNzvE2yw',
-        'md5': '0c2cfd068ee2afe657801269b2d86214',
+        'url': 'https://v.douyu.com/show/mPyq7oVNe5Yv1gLY',
         'info_dict': {
-            'id': 'rjNBdvnVXNzvE2yw',
+            'id': 'mPyq7oVNe5Yv1gLY',
             'ext': 'mp4',
-            'title': '陈一发儿：砒霜 我有个室友系列！04-01 22点场',
-            'duration': 7150.08,
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': '陈一发儿',
-            'uploader_id': 'XrZwYelr5wbK',
-            'uploader_url': 'https://v.douyu.com/author/XrZwYelr5wbK',
-            'upload_date': '20170402',
+            'title': '四川人小时候的味道“蒜苗回锅肉”，传统菜不能丢，要常做来吃',
+            'duration': 633,
+            'thumbnail': str,
+            'uploader': '美食作家王刚V',
+            'uploader_id': 'OVAO4NVx1m7Q',
+            'timestamp': 1661850002,
+            'upload_date': '20220830',
+            'view_count': int,
+            'tags': ['美食', '美食综合'],
         },
     }, {
         'url': 'https://vmobile.douyu.com/show/rjNBdvnVXNzvE2yw',
         'only_matching': True,
     }]
 
+    _FORMATS = {
+        'super': '原画',
+        'high': '超清',
+        'normal': '高清',
+    }
+
+    _QUALITIES = {
+        'super': -1,
+        'high': -2,
+        'normal': -3,
+    }
+
+    _RESOLUTIONS = {
+        'super': '1920x1080',
+        'high': '1280x720',
+        'normal': '852x480',
+    }
+
     def _real_extract(self, url):
         url = url.replace('vmobile.', 'v.')
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        room_info = self._parse_json(self._search_regex(
-            r'var\s+\$ROOM\s*=\s*({.+});', webpage, 'room info'), video_id)
+        video_info = self._search_json(
+            r'<script>\s*window\.\$DATA\s*=', webpage,
+            'video info', video_id, transform_source=js_to_json)
 
-        video_info = None
+        js_sign_func = self._search_js_sign_func(webpage)
+        form_data = {
+            'vid': video_id,
+            **self._calc_sign(js_sign_func, video_id, video_info['ROOM']['point_id']),
+        }
+        url_info = self._download_json(
+            'https://v.douyu.com/api/stream/getStreamUrl', video_id,
+            data=urlencode_postdata(form_data), note="Downloading video formats")
 
-        for trial in range(5):
-            # Sometimes Douyu rejects our request. Let's try it more times
-            try:
-                video_info = self._download_json(
-                    'https://vmobile.douyu.com/video/getInfo', video_id,
-                    query={'vid': video_id},
-                    headers={
-                        'Referer': url,
-                        'x-requested-with': 'XMLHttpRequest',
-                    })
-                break
-            except ExtractorError:
-                self._sleep(1, video_id)
-
-        if not video_info:
-            raise ExtractorError('Can\'t fetch video info')
-
-        formats = self._extract_m3u8_formats(
-            video_info['data']['video_url'], video_id,
-            entry_protocol='m3u8_native', ext='mp4')
-
-        upload_date = unified_strdate(self._html_search_regex(
-            r'<em>上传时间：</em><span>([^<]+)</span>', webpage,
-            'upload date', fatal=False))
-
-        uploader = uploader_id = uploader_url = None
-        mobj = re.search(
-            r'(?m)<a[^>]+href="/author/([0-9a-zA-Z]+)".+?<strong[^>]+title="([^"]+)"',
-            webpage)
-        if mobj:
-            uploader_id, uploader = mobj.groups()
-            uploader_url = urljoin(url, '/author/' + uploader_id)
+        formats = []
+        for name, url in traverse_obj(url_info, ('data', 'thumb_video', {dict.items}, ...)):
+            video_url = traverse_obj(url, ('url', {url_or_none}))
+            if video_url:
+                ext = determine_ext(video_url)
+                formats.append({
+                    'format': self._FORMATS.get(name),
+                    'format_id': name,
+                    'url': video_url,
+                    'quality': self._QUALITIES.get(name),
+                    'ext': 'mp4' if ext == 'm3u8' else ext,
+                    'protocol': 'm3u8_native' if ext == 'm3u8' else 'https',
+                    **parse_resolution(self._RESOLUTIONS.get(name))
+                })
+            else:
+                self.to_screen(
+                    f'"{self._FORMATS.get(name, name)}" format may require logging in. {self._login_hint()}')
 
         return {
             'id': video_id,
-            'title': room_info['name'],
             'formats': formats,
-            'duration': room_info.get('duration'),
-            'thumbnail': room_info.get('pic'),
-            'upload_date': upload_date,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'uploader_url': uploader_url,
+            **traverse_obj(video_info, ('DATA', {
+                'title': ('content', 'title', {str}),
+                'uploader': ('content', 'author', {str}),
+                'uploader_id': ('content', 'up_id', {str_or_none}),
+                'duration': ('content', 'video_duration', {int_or_none}),
+                'thumbnail': ('content', 'video_pic', {url_or_none}),
+                'timestamp': ('content', 'create_time', {int_or_none}),
+                'view_count': ('content', 'view_num', {int_or_none}),
+                'tags': ('videoTag', ..., 'tagName', {str}),
+            }))
         }

From 5fccabac27ca3c1165ade1b0df6fbadc24258dc2 Mon Sep 17 00:00:00 2001
From: Simon <simon30002021@icloud.com>
Date: Thu, 21 Sep 2023 19:37:58 +0200
Subject: [PATCH 578/871] [ie/rbgtum] Fix extraction and support new URL format
 (#7690)

Authored by: simon300000
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/rbgtum.py      | 79 ++++++++++++++++++++++++++-------
 2 files changed, 65 insertions(+), 15 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 632d6720e..9cda06d8f 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1601,6 +1601,7 @@
 from .rbgtum import (
     RbgTumIE,
     RbgTumCourseIE,
+    RbgTumNewCourseIE,
 )
 from .rcs import (
     RCSIE,
diff --git a/yt_dlp/extractor/rbgtum.py b/yt_dlp/extractor/rbgtum.py
index 47649cfc5..c8a331f3e 100644
--- a/yt_dlp/extractor/rbgtum.py
+++ b/yt_dlp/extractor/rbgtum.py
@@ -1,10 +1,11 @@
 import re
 
 from .common import InfoExtractor
+from ..utils import parse_qs, remove_start, traverse_obj, ExtractorError
 
 
 class RbgTumIE(InfoExtractor):
-    _VALID_URL = r'https://live\.rbg\.tum\.de/w/(?P<id>.+)'
+    _VALID_URL = r'https://(?:live\.rbg\.tum\.de|tum\.live)/w/(?P<id>[^?#]+)'
     _TESTS = [{
         # Combined view
         'url': 'https://live.rbg.tum.de/w/cpp/22128',
@@ -35,16 +36,18 @@ class RbgTumIE(InfoExtractor):
             'title': 'Fachschaftsvollversammlung',
             'series': 'Fachschaftsvollversammlung Informatik',
         }
+    }, {
+        'url': 'https://tum.live/w/linalginfo/27102',
+        'only_matching': True,
     }, ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        m3u8 = self._html_search_regex(r'(https://.+?\.m3u8)', webpage, 'm3u8')
-        lecture_title = self._html_search_regex(r'(?si)<h1.*?>(.*)</h1>', webpage, 'title')
-        lecture_series_title = self._html_search_regex(
-            r'(?s)<title\b[^>]*>\s*(?:TUM-Live\s\|\s?)?([^:]+):?.*?</title>', webpage, 'series')
+        m3u8 = self._html_search_regex(r'"(https://[^"]+\.m3u8[^"]*)', webpage, 'm3u8')
+        lecture_title = self._html_search_regex(r'<h1[^>]*>([^<]+)</h1>', webpage, 'title', fatal=False)
+        lecture_series_title = remove_start(self._html_extract_title(webpage), 'TUM-Live | ')
 
         formats = self._extract_m3u8_formats(m3u8, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
 
@@ -57,9 +60,9 @@ def _real_extract(self, url):
 
 
 class RbgTumCourseIE(InfoExtractor):
-    _VALID_URL = r'https://live\.rbg\.tum\.de/course/(?P<id>.+)'
+    _VALID_URL = r'https://(?P<hostname>(?:live\.rbg\.tum\.de|tum\.live))/old/course/(?P<id>(?P<year>\d+)/(?P<term>\w+)/(?P<slug>[^/?#]+))'
     _TESTS = [{
-        'url': 'https://live.rbg.tum.de/course/2022/S/fpv',
+        'url': 'https://live.rbg.tum.de/old/course/2022/S/fpv',
         'info_dict': {
             'title': 'Funktionale Programmierung und Verifikation (IN0003)',
             'id': '2022/S/fpv',
@@ -69,7 +72,7 @@ class RbgTumCourseIE(InfoExtractor):
         },
         'playlist_count': 13,
     }, {
-        'url': 'https://live.rbg.tum.de/course/2022/W/set',
+        'url': 'https://live.rbg.tum.de/old/course/2022/W/set',
         'info_dict': {
             'title': 'SET FSMPIC',
             'id': '2022/W/set',
@@ -78,16 +81,62 @@ class RbgTumCourseIE(InfoExtractor):
             'noplaylist': False,
         },
         'playlist_count': 6,
+    }, {
+        'url': 'https://tum.live/old/course/2023/S/linalginfo',
+        'only_matching': True,
     }, ]
 
     def _real_extract(self, url):
-        course_id = self._match_id(url)
-        webpage = self._download_webpage(url, course_id)
+        course_id, hostname, year, term, slug = self._match_valid_url(url).group('id', 'hostname', 'year', 'term', 'slug')
+        meta = self._download_json(
+            f'https://{hostname}/api/courses/{slug}/', course_id, fatal=False,
+            query={'year': year, 'term': term}) or {}
+        lecture_series_title = meta.get('Name')
+        lectures = [self.url_result(f'https://{hostname}/w/{slug}/{stream_id}', RbgTumIE)
+                    for stream_id in traverse_obj(meta, ('Streams', ..., 'ID'))]
 
-        lecture_series_title = self._html_search_regex(r'(?si)<h1.*?>(.*)</h1>', webpage, 'title')
+        if not lectures:
+            webpage = self._download_webpage(url, course_id)
+            lecture_series_title = remove_start(self._html_extract_title(webpage), 'TUM-Live | ')
+            lectures = [self.url_result(f'https://{hostname}{lecture_path}', RbgTumIE)
+                        for lecture_path in re.findall(r'href="(/w/[^/"]+/[^/"]+)"', webpage)]
 
-        lecture_urls = []
-        for lecture_url in re.findall(r'(?i)href="/w/(.+)(?<!/cam)(?<!/pres)(?<!/chat)"', webpage):
-            lecture_urls.append(self.url_result('https://live.rbg.tum.de/w/' + lecture_url, ie=RbgTumIE.ie_key()))
+        return self.playlist_result(lectures, course_id, lecture_series_title)
 
-        return self.playlist_result(lecture_urls, course_id, lecture_series_title)
+
+class RbgTumNewCourseIE(InfoExtractor):
+    _VALID_URL = r'https://(?P<hostname>(?:live\.rbg\.tum\.de|tum\.live))/\?'
+    _TESTS = [{
+        'url': 'https://live.rbg.tum.de/?year=2022&term=S&slug=fpv&view=3',
+        'info_dict': {
+            'title': 'Funktionale Programmierung und Verifikation (IN0003)',
+            'id': '2022/S/fpv',
+        },
+        'params': {
+            'noplaylist': False,
+        },
+        'playlist_count': 13,
+    }, {
+        'url': 'https://live.rbg.tum.de/?year=2022&term=W&slug=set&view=3',
+        'info_dict': {
+            'title': 'SET FSMPIC',
+            'id': '2022/W/set',
+        },
+        'params': {
+            'noplaylist': False,
+        },
+        'playlist_count': 6,
+    }, {
+        'url': 'https://tum.live/?year=2023&term=S&slug=linalginfo&view=3',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        query = parse_qs(url)
+        errors = [key for key in ('year', 'term', 'slug') if not query.get(key)]
+        if errors:
+            raise ExtractorError(f'Input URL is missing query parameters: {", ".join(errors)}')
+        year, term, slug = query['year'][0], query['term'][0], query['slug'][0]
+        hostname = self._match_valid_url(url).group('hostname')
+
+        return self.url_result(f'https://{hostname}/old/course/{year}/{term}/{slug}', RbgTumCourseIE)

From b84fda7388dd20d38921e23b469147f3957c1812 Mon Sep 17 00:00:00 2001
From: ClosedPort22 <44864697+ClosedPort22@users.noreply.github.com>
Date: Thu, 21 Sep 2023 17:45:18 +0000
Subject: [PATCH 579/871] [ie/bilibili] Extract Dolby audio formats (#8142)

Closes #4050
Authored by: ClosedPort22
---
 yt_dlp/extractor/bilibili.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 5e7042dbb..9119f396b 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -49,14 +49,14 @@ def extract_formats(self, play_info):
             for r in traverse_obj(play_info, ('support_formats', lambda _, v: v['quality']))
         }
 
-        audios = traverse_obj(play_info, ('dash', 'audio', ...))
+        audios = traverse_obj(play_info, ('dash', (None, 'dolby'), 'audio', ..., {dict}))
         flac_audio = traverse_obj(play_info, ('dash', 'flac', 'audio'))
         if flac_audio:
             audios.append(flac_audio)
         formats = [{
             'url': traverse_obj(audio, 'baseUrl', 'base_url', 'url'),
             'ext': mimetype2ext(traverse_obj(audio, 'mimeType', 'mime_type')),
-            'acodec': audio.get('codecs'),
+            'acodec': traverse_obj(audio, ('codecs', {str.lower})),
             'vcodec': 'none',
             'tbr': float_or_none(audio.get('bandwidth'), scale=1000),
             'filesize': int_or_none(audio.get('size')),
@@ -71,6 +71,7 @@ def extract_formats(self, play_info):
             'height': int_or_none(video.get('height')),
             'vcodec': video.get('codecs'),
             'acodec': 'none' if audios else None,
+            'dynamic_range': {126: 'DV', 125: 'HDR10'}.get(int_or_none(video.get('id'))),
             'tbr': float_or_none(video.get('bandwidth'), scale=1000),
             'filesize': int_or_none(video.get('size')),
             'quality': int_or_none(video.get('id')),

From a5e264d74b4bd60c6e7ec4e38f1a23af4e420531 Mon Sep 17 00:00:00 2001
From: kylegustavo <kysalves@yahoo.com>
Date: Thu, 21 Sep 2023 10:46:49 -0700
Subject: [PATCH 580/871] [ie/Expressen] Improve `_VALID_URL` (#8153)

Closes #8141
Authored by: kylegustavo
---
 yt_dlp/extractor/expressen.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/expressen.py b/yt_dlp/extractor/expressen.py
index 86967b631..b96f2e4cb 100644
--- a/yt_dlp/extractor/expressen.py
+++ b/yt_dlp/extractor/expressen.py
@@ -11,8 +11,8 @@ class ExpressenIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.)?(?:expressen|di)\.se/
-                        (?:(?:tvspelare/video|videoplayer/embed)/)?
-                        tv/(?:[^/]+/)*
+                        (?:(?:tvspelare/video|video-?player/embed)/)?
+                        (?:tv|nyheter)/(?:[^/?#]+/)*
                         (?P<id>[^/?#&]+)
                     '''
     _EMBED_REGEX = [r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?(?:expressen|di)\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1']
@@ -42,6 +42,12 @@ class ExpressenIE(InfoExtractor):
     }, {
         'url': 'https://www.di.se/videoplayer/embed/tv/ditv/borsmorgon/implantica-rusar-70--under-borspremiaren-hor-styrelsemedlemmen/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di',
         'only_matching': True,
+    }, {
+        'url': 'https://www.expressen.se/video-player/embed/tv/nyheter/ekero-fodda-olof-gustafsson-forvaltar-knarkbaronen-pablo-escobars-namn',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.expressen.se/nyheter/efter-egna-telefonbluffen-escobar-stammer-klarna/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 2269065ad60cb0ab62408ae6a7b20283e5252232 Mon Sep 17 00:00:00 2001
From: std-move <26625259+std-move@users.noreply.github.com>
Date: Thu, 21 Sep 2023 20:19:52 +0200
Subject: [PATCH 581/871] [ie/NovaEmbed] Fix extractor (#7910)

Closes #8025
Authored by: std-move
---
 yt_dlp/extractor/nova.py | 116 +++++++++++++++------------------------
 1 file changed, 45 insertions(+), 71 deletions(-)

diff --git a/yt_dlp/extractor/nova.py b/yt_dlp/extractor/nova.py
index 8bd3fd472..bd0c4ebe3 100644
--- a/yt_dlp/extractor/nova.py
+++ b/yt_dlp/extractor/nova.py
@@ -6,7 +6,6 @@
     determine_ext,
     int_or_none,
     js_to_json,
-    qualities,
     traverse_obj,
     unified_strdate,
     url_or_none,
@@ -49,77 +48,52 @@ def _real_extract(self, url):
         duration = None
         formats = []
 
-        player = self._parse_json(
-            self._search_regex(
-                (r'(?:(?:replacePlaceholders|processAdTagModifier).*?:\s*)?(?:replacePlaceholders|processAdTagModifier)\s*\(\s*(?P<json>{.*?})\s*\)(?:\s*\))?\s*,',
-                    r'Player\.init\s*\([^,]+,(?P<cndn>\s*\w+\s*\?)?\s*(?P<json>{(?(cndn).+?|.+)})\s*(?(cndn):|,\s*{.+?}\s*\)\s*;)'),
-                webpage, 'player', default='{}', group='json'), video_id, fatal=False)
-        if player:
-            for format_id, format_list in player['tracks'].items():
-                if not isinstance(format_list, list):
-                    format_list = [format_list]
-                for format_dict in format_list:
-                    if not isinstance(format_dict, dict):
-                        continue
-                    if (not self.get_param('allow_unplayable_formats')
-                            and traverse_obj(format_dict, ('drm', 'keySystem'))):
-                        has_drm = True
-                        continue
-                    format_url = url_or_none(format_dict.get('src'))
-                    format_type = format_dict.get('type')
-                    ext = determine_ext(format_url)
-                    if (format_type == 'application/x-mpegURL'
-                            or format_id == 'HLS' or ext == 'm3u8'):
-                        formats.extend(self._extract_m3u8_formats(
-                            format_url, video_id, 'mp4',
-                            entry_protocol='m3u8_native', m3u8_id='hls',
-                            fatal=False))
-                    elif (format_type == 'application/dash+xml'
-                          or format_id == 'DASH' or ext == 'mpd'):
-                        formats.extend(self._extract_mpd_formats(
-                            format_url, video_id, mpd_id='dash', fatal=False))
-                    else:
-                        formats.append({
-                            'url': format_url,
-                        })
-            duration = int_or_none(player.get('duration'))
-        else:
-            # Old path, not actual as of 08.04.2020
-            bitrates = self._parse_json(
-                self._search_regex(
-                    r'(?s)(?:src|bitrates)\s*=\s*({.+?})\s*;', webpage, 'formats'),
-                video_id, transform_source=js_to_json)
-
-            QUALITIES = ('lq', 'mq', 'hq', 'hd')
-            quality_key = qualities(QUALITIES)
-
-            for format_id, format_list in bitrates.items():
-                if not isinstance(format_list, list):
-                    format_list = [format_list]
-                for format_url in format_list:
-                    format_url = url_or_none(format_url)
-                    if not format_url:
-                        continue
-                    if format_id == 'hls':
-                        formats.extend(self._extract_m3u8_formats(
-                            format_url, video_id, ext='mp4',
-                            entry_protocol='m3u8_native', m3u8_id='hls',
-                            fatal=False))
-                        continue
-                    f = {
+        def process_format_list(format_list, format_id=""):
+            nonlocal formats, has_drm
+            if not isinstance(format_list, list):
+                format_list = [format_list]
+            for format_dict in format_list:
+                if not isinstance(format_dict, dict):
+                    continue
+                if (not self.get_param('allow_unplayable_formats')
+                        and traverse_obj(format_dict, ('drm', 'keySystem'))):
+                    has_drm = True
+                    continue
+                format_url = url_or_none(format_dict.get('src'))
+                format_type = format_dict.get('type')
+                ext = determine_ext(format_url)
+                if (format_type == 'application/x-mpegURL'
+                        or format_id == 'HLS' or ext == 'm3u8'):
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, video_id, 'mp4',
+                        entry_protocol='m3u8_native', m3u8_id='hls',
+                        fatal=False))
+                elif (format_type == 'application/dash+xml'
+                      or format_id == 'DASH' or ext == 'mpd'):
+                    formats.extend(self._extract_mpd_formats(
+                        format_url, video_id, mpd_id='dash', fatal=False))
+                else:
+                    formats.append({
                         'url': format_url,
-                    }
-                    f_id = format_id
-                    for quality in QUALITIES:
-                        if '%s.mp4' % quality in format_url:
-                            f_id += '-%s' % quality
-                            f.update({
-                                'quality': quality_key(quality),
-                                'format_note': quality.upper(),
-                            })
-                            break
-                    f['format_id'] = f_id
-                    formats.append(f)
+                    })
+
+        player = self._search_json(
+            r'player:', webpage, 'player', video_id, fatal=False, end_pattern=r';\s*</script>')
+        if player:
+            for src in traverse_obj(player, ('lib', 'source', 'sources', ...)):
+                process_format_list(src)
+            duration = traverse_obj(player, ('sourceInfo', 'duration', {int_or_none}))
+        if not formats and not has_drm:
+            # older code path, in use before August 2023
+            player = self._parse_json(
+                self._search_regex(
+                    (r'(?:(?:replacePlaceholders|processAdTagModifier).*?:\s*)?(?:replacePlaceholders|processAdTagModifier)\s*\(\s*(?P<json>{.*?})\s*\)(?:\s*\))?\s*,',
+                     r'Player\.init\s*\([^,]+,(?P<cndn>\s*\w+\s*\?)?\s*(?P<json>{(?(cndn).+?|.+)})\s*(?(cndn):|,\s*{.+?}\s*\)\s*;)'),
+                    webpage, 'player', group='json'), video_id)
+            if player:
+                for format_id, format_list in player['tracks'].items():
+                    process_format_list(format_list, format_id)
+                duration = int_or_none(player.get('duration'))
 
         if not formats and has_drm:
             self.report_drm(video_id)

From 52414d64ca7b92d3f83964cdd68247989b0c4625 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Thu, 21 Sep 2023 16:51:57 -0500
Subject: [PATCH 582/871] [utils] `js_to_json`: Handle `Array` objects

Authored by: Grub4K, std-move

Co-authored-by: std-move <26625259+std-move@users.noreply.github.com>
Co-authored-by: Simon Sawicki <accounts@grub4k.xyz>
---
 test/test_utils.py     | 6 ++++++
 yt_dlp/utils/_utils.py | 1 +
 2 files changed, 7 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 91e3ffd39..47d1f71bf 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1218,6 +1218,12 @@ def test_js_to_json_template_literal(self):
         self.assertEqual(js_to_json('`${name}"${name}"`', {'name': '5'}), '"5\\"5\\""')
         self.assertEqual(js_to_json('`${name}`', {}), '"name"')
 
+    def test_js_to_json_map_array_constructors(self):
+        self.assertEqual(json.loads(js_to_json('new Map([["a", 5]])')), {'a': 5})
+        self.assertEqual(json.loads(js_to_json('Array(5, 10)')), [5, 10])
+        self.assertEqual(json.loads(js_to_json('new Array(15,5)')), [15, 5])
+        self.assertEqual(json.loads(js_to_json('new Map([Array(5, 10),new Array(15,5)])')), {'5': 10, '15': 5})
+
     def test_extract_attributes(self):
         self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
         self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index ef26de116..213ccc636 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -2727,6 +2727,7 @@ def fix_kv(m):
     def create_map(mobj):
         return json.dumps(dict(json.loads(js_to_json(mobj.group(1) or '[]', vars=vars))))
 
+    code = re.sub(r'(?:new\s+)?Array\((.*?)\)', r'[\g<1>]', code)
     code = re.sub(r'new Map\((\[.*?\])?\)', create_map, code)
     if not strict:
         code = re.sub(r'new Date\((".+")\)', r'\g<1>', code)

From 904a19ee93195ce0bd4b08bd22b186120afb5b17 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Thu, 21 Sep 2023 16:54:57 -0500
Subject: [PATCH 583/871] [ie] Make `_search_nuxt_data` more lenient

Authored by: std-move

Co-authored-by: std-move <26625259+std-move@users.noreply.github.com>
---
 yt_dlp/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 7deab995c..c94b4abdc 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -1687,7 +1687,7 @@ def _search_nextjs_data(self, webpage, video_id, *, transform_source=None, fatal
     def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', *, fatal=True, traverse=('data', 0)):
         """Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
         rectx = re.escape(context_name)
-        FUNCTION_RE = r'\(function\((?P<arg_keys>.*?)\){return\s+(?P<js>{.*?})\s*;?\s*}\((?P<arg_vals>.*?)\)'
+        FUNCTION_RE = r'\(function\((?P<arg_keys>.*?)\){(?:.*?)return\s+(?P<js>{.*?})\s*;?\s*}\((?P<arg_vals>.*?)\)'
         js, arg_keys, arg_vals = self._search_regex(
             (rf'<script>\s*window\.{rectx}={FUNCTION_RE}\s*\)\s*;?\s*</script>', rf'{rectx}\(.*?{FUNCTION_RE}'),
             webpage, context_name, group=('js', 'arg_keys', 'arg_vals'),

From 568f08051841aedea968258889539741e26009e9 Mon Sep 17 00:00:00 2001
From: std-move <26625259+std-move@users.noreply.github.com>
Date: Fri, 22 Sep 2023 00:20:52 +0200
Subject: [PATCH 584/871] [ie/iprima] Fix extractor (#7216)

Closes #7229
Authored by: std-move
---
 yt_dlp/extractor/iprima.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/iprima.py b/yt_dlp/extractor/iprima.py
index 6dec1510d..f7aa579b3 100644
--- a/yt_dlp/extractor/iprima.py
+++ b/yt_dlp/extractor/iprima.py
@@ -134,10 +134,17 @@ def _real_extract(self, url):
         ), webpage, 'real id', group='id', default=None)
 
         if not video_id:
-            nuxt_data = self._search_nuxt_data(webpage, video_id, traverse='data')
+            nuxt_data = self._search_nuxt_data(webpage, video_id, traverse='data', fatal=False)
             video_id = traverse_obj(
                 nuxt_data, (..., 'content', 'additionals', 'videoPlayId', {str}), get_all=False)
 
+        if not video_id:
+            nuxt_data = self._search_json(
+                r'<script[^>]+\bid=["\']__NUXT_DATA__["\'][^>]*>',
+                webpage, 'nuxt data', None, end_pattern=r'</script>', contains_pattern=r'\[(?s:.+)\]')
+
+            video_id = traverse_obj(nuxt_data, lambda _, v: re.fullmatch(r'p\d+', v), get_all=False)
+
         if not video_id:
             self.raise_no_formats('Unable to extract video ID from webpage')
 

From 661c9a1d029296b28e0b2f8be8a72a43abaf6536 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Thu, 21 Sep 2023 17:48:57 -0500
Subject: [PATCH 585/871] [test:download] Test for `expected_exception`

Authored by: at-wat

Co-authored-by: Atsushi Watanabe <atsushi.w@ieee.org>
---
 test/test_download.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/test/test_download.py b/test/test_download.py
index 6f00a4ded..253079249 100755
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -31,6 +31,7 @@
     DownloadError,
     ExtractorError,
     UnavailableVideoError,
+    YoutubeDLError,
     format_bytes,
     join_nonempty,
 )
@@ -100,6 +101,8 @@ def print_skipping(reason):
             print_skipping('IE marked as not _WORKING')
 
         for tc in test_cases:
+            if tc.get('expected_exception'):
+                continue
             info_dict = tc.get('info_dict', {})
             params = tc.get('params', {})
             if not info_dict.get('id'):
@@ -139,6 +142,17 @@ def get_tc_filename(tc):
 
         res_dict = None
 
+        def match_exception(err):
+            expected_exception = test_case.get('expected_exception')
+            if not expected_exception:
+                return False
+            if err.__class__.__name__ == expected_exception:
+                return True
+            for exc in err.exc_info:
+                if exc.__class__.__name__ == expected_exception:
+                    return True
+            return False
+
         def try_rm_tcs_files(tcs=None):
             if tcs is None:
                 tcs = test_cases
@@ -161,6 +175,8 @@ def try_rm_tcs_files(tcs=None):
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
                     if not isinstance(err.exc_info[1], (TransportError, UnavailableVideoError)) or (isinstance(err.exc_info[1], HTTPError) and err.exc_info[1].status == 503):
+                        if match_exception(err):
+                            return
                         err.msg = f'{getattr(err, "msg", err)} ({tname})'
                         raise
 
@@ -171,6 +187,10 @@ def try_rm_tcs_files(tcs=None):
                     print(f'Retrying: {try_num} failed tries\n\n##########\n\n')
 
                     try_num += 1
+                except YoutubeDLError as err:
+                    if match_exception(err):
+                        return
+                    raise
                 else:
                     break
 

From c1d71d0d9f41db5e4306c86af232f5f6220a130b Mon Sep 17 00:00:00 2001
From: Atsushi Watanabe <atsushi.w@ieee.org>
Date: Fri, 22 Sep 2023 08:04:05 +0900
Subject: [PATCH 586/871] [ie/twitcasting] Support `--wait-for-video` (#7975)

Authored by: at-wat
---
 yt_dlp/extractor/twitcasting.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py
index 3890d5d8f..540e217fd 100644
--- a/yt_dlp/extractor/twitcasting.py
+++ b/yt_dlp/extractor/twitcasting.py
@@ -5,8 +5,9 @@
 from .common import InfoExtractor
 from ..dependencies import websockets
 from ..utils import (
-    clean_html,
     ExtractorError,
+    UserNotLive,
+    clean_html,
     float_or_none,
     get_element_by_class,
     get_element_by_id,
@@ -235,6 +236,9 @@ class TwitCastingLiveIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://twitcasting.tv/ivetesangalo',
         'only_matching': True,
+    }, {
+        'url': 'https://twitcasting.tv/c:unusedlive',
+        'expected_exception': 'UserNotLive',
     }]
 
     def _real_extract(self, url):
@@ -260,7 +264,7 @@ def _real_extract(self, url):
                     r'(?s)<a\s+class="tw-movie-thumbnail"\s*href="/[^/]+/movie/(?P<video_id>\d+)"\s*>.+?</a>',
                     webpage, 'current live ID 2', default=None, group='video_id')
         if not current_live:
-            raise ExtractorError('The user is not currently live')
+            raise UserNotLive(video_id=uploader_id)
         return self.url_result('https://twitcasting.tv/%s/movie/%s' % (uploader_id, current_live))
 
 

From c2da0b5ea215298135f76e3dc14b972a3c4afacb Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sat, 23 Sep 2023 14:54:00 -0500
Subject: [PATCH 587/871] [ie/ArteTV] Fix HLS formats extraction

Closes #8156
Authored by: bashonly
---
 yt_dlp/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/arte.py b/yt_dlp/extractor/arte.py
index e3cc5afb0..a19cd2a3a 100644
--- a/yt_dlp/extractor/arte.py
+++ b/yt_dlp/extractor/arte.py
@@ -169,7 +169,7 @@ def _real_extract(self, url):
                 )))
 
             short_label = traverse_obj(stream_version, 'shortLabel', expected_type=str, default='?')
-            if stream['protocol'].startswith('HLS'):
+            if 'HLS' in stream['protocol']:
                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
                     stream['url'], video_id=video_id, ext='mp4', m3u8_id=stream_version_code, fatal=False)
                 for fmt in fmts:

From 5ca095cbcde3e32642a4fe5b2d69e8e3c785a021 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 23 Sep 2023 15:00:31 -0500
Subject: [PATCH 588/871] [cleanup] Misc (#8182)

Closes #7796, Closes #8028
Authored by: barsnick, sqrtNOT, gamer191, coletdjnz, Grub4K, bashonly
---
 CONTRIBUTING.md                   | 8 ++++----
 README.md                         | 2 +-
 devscripts/make_changelog.py      | 2 +-
 test/test_YoutubeDL.py            | 1 -
 test/test_networking_utils.py     | 6 +++---
 yt_dlp/YoutubeDL.py               | 6 +++---
 yt_dlp/compat/urllib/__init__.py  | 2 +-
 yt_dlp/extractor/abc.py           | 1 -
 yt_dlp/extractor/ign.py           | 4 ----
 yt_dlp/extractor/nebula.py        | 1 -
 yt_dlp/extractor/peekvids.py      | 1 -
 yt_dlp/extractor/radiofrance.py   | 2 +-
 yt_dlp/extractor/rcs.py           | 6 +++---
 yt_dlp/extractor/rokfin.py        | 1 -
 yt_dlp/extractor/s4c.py           | 2 --
 yt_dlp/extractor/sovietscloset.py | 1 -
 yt_dlp/extractor/youtube.py       | 2 +-
 yt_dlp/networking/__init__.py     | 2 +-
 yt_dlp/networking/_urllib.py      | 2 +-
 yt_dlp/networking/exceptions.py   | 4 ++--
 20 files changed, 22 insertions(+), 34 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a8587fe92..90e7faf7c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -217,7 +217,7 @@ ## Adding support for a new site
 1. Add an import in [`yt_dlp/extractor/_extractors.py`](yt_dlp/extractor/_extractors.py). Note that the class name must end with `IE`.
 1. Run `python test/test_download.py TestDownload.test_YourExtractor` (note that `YourExtractor` doesn't end with `IE`). This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, the tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. You can also run all the tests in one go with `TestDownload.test_YourExtractor_all`
 1. Make sure you have atleast one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
-1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L91-L426). Add tests and code for as many as you want.
+1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L119-L440). Add tests and code for as many as you want.
 1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
 
         $ flake8 yt_dlp/extractor/yourextractor.py
@@ -251,7 +251,7 @@ ## yt-dlp coding conventions
 
 ### Mandatory and optional metafields
 
-For extraction to work yt-dlp relies on metadata your extractor extracts and provides to yt-dlp expressed by an [information dictionary](yt_dlp/extractor/common.py#L91-L426) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by yt-dlp:
+For extraction to work yt-dlp relies on metadata your extractor extracts and provides to yt-dlp expressed by an [information dictionary](yt_dlp/extractor/common.py#L119-L440) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by yt-dlp:
 
  - `id` (media identifier)
  - `title` (media title)
@@ -696,7 +696,7 @@ #### Examples
 
 ### Use convenience conversion and parsing functions
 
-Wrap all extracted numeric data into safe functions from [`yt_dlp/utils.py`](yt_dlp/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
+Wrap all extracted numeric data into safe functions from [`yt_dlp/utils/`](yt_dlp/utils/): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
 
 Use `url_or_none` for safe URL processing.
 
@@ -704,7 +704,7 @@ ### Use convenience conversion and parsing functions
 
 Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. 
 
-Explore [`yt_dlp/utils.py`](yt_dlp/utils.py) for more useful convenience functions.
+Explore [`yt_dlp/utils/`](yt_dlp/utils/) for more useful convenience functions.
 
 #### Examples
 
diff --git a/README.md b/README.md
index d94d8ea82..d9b11952d 100644
--- a/README.md
+++ b/README.md
@@ -1800,7 +1800,7 @@ # EXTRACTOR ARGUMENTS
 #### youtube
 * `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube.py](https://github.com/yt-dlp/yt-dlp/blob/c26f9b991a0681fd3ea548d535919cec1fbbd430/yt_dlp/extractor/youtube.py#L381-L390) for list of supported content language codes
 * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
-* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb` and `tv_embedded` (agegate bypass) with no variants. By default, `ios,android,web` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly, the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
+* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb`, `mweb_embedscreen` and `tv_embedded` (agegate bypass) with no variants. By default, `ios,android,web` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly, the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
 * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
 * `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp.
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index ac68dcd19..9ff65db14 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -260,7 +260,7 @@ class CommitRange:
     AUTHOR_INDICATOR_RE = re.compile(r'Authored by:? ', re.IGNORECASE)
     MESSAGE_RE = re.compile(r'''
         (?:\[(?P<prefix>[^\]]+)\]\ )?
-        (?:(?P<sub_details>`?[^:`]+`?): )?
+        (?:(?P<sub_details>`?[\w.-]+`?): )?
         (?P<message>.+?)
         (?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
         ''', re.VERBOSE | re.DOTALL)
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 3cfb61fb2..916ee48b9 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -631,7 +631,6 @@ def test_add_extra_info(self):
         self.assertEqual(test_dict['playlist'], 'funny videos')
 
     outtmpl_info = {
-        'id': '1234',
         'id': '1234',
         'ext': 'mp4',
         'width': None,
diff --git a/test/test_networking_utils.py b/test/test_networking_utils.py
index dbf656090..419aae1e4 100644
--- a/test/test_networking_utils.py
+++ b/test/test_networking_utils.py
@@ -269,14 +269,14 @@ def test_compat_http_error_autoclose(self):
         assert not response.closed
 
     def test_incomplete_read_error(self):
-        error = IncompleteRead(b'test', 3, cause='test')
+        error = IncompleteRead(4, 3, cause='test')
         assert isinstance(error, IncompleteRead)
         assert repr(error) == '<IncompleteRead: 4 bytes read, 3 more expected>'
         assert str(error) == error.msg == '4 bytes read, 3 more expected'
-        assert error.partial == b'test'
+        assert error.partial == 4
         assert error.expected == 3
         assert error.cause == 'test'
 
-        error = IncompleteRead(b'aaa')
+        error = IncompleteRead(3)
         assert repr(error) == '<IncompleteRead: 3 bytes read>'
         assert str(error) == '3 bytes read'
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 1feed3052..39aaf2c2e 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -239,9 +239,9 @@ class YoutubeDL:
                        'selected' (check selected formats),
                        or None (check only if requested by extractor)
     paths:             Dictionary of output paths. The allowed keys are 'home'
-                       'temp' and the keys of OUTTMPL_TYPES (in utils.py)
+                       'temp' and the keys of OUTTMPL_TYPES (in utils/_utils.py)
     outtmpl:           Dictionary of templates for output names. Allowed keys
-                       are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
+                       are 'default' and the keys of OUTTMPL_TYPES (in utils/_utils.py).
                        For compatibility with youtube-dl, a single string can also be used
     outtmpl_na_placeholder: Placeholder for unavailable meta fields.
     restrictfilenames: Do not allow "&" and spaces in file names
@@ -422,7 +422,7 @@ class YoutubeDL:
                          asked whether to download the video.
                        - Raise utils.DownloadCancelled(msg) to abort remaining
                          downloads when a video is rejected.
-                       match_filter_func in utils.py is one example for this.
+                       match_filter_func in utils/_utils.py is one example for this.
     color:             A Dictionary with output stream names as keys
                        and their respective color policy as values.
                        Can also just be a single color policy,
diff --git a/yt_dlp/compat/urllib/__init__.py b/yt_dlp/compat/urllib/__init__.py
index b27cc6133..9084b3c2b 100644
--- a/yt_dlp/compat/urllib/__init__.py
+++ b/yt_dlp/compat/urllib/__init__.py
@@ -1,7 +1,7 @@
 # flake8: noqa: F405
 from urllib import *  # noqa: F403
 
-del request
+del request  # noqa: F821
 from . import request  # noqa: F401
 
 from ..compat_utils import passthrough_module
diff --git a/yt_dlp/extractor/abc.py b/yt_dlp/extractor/abc.py
index f56133eb3..d2cf5f7c5 100644
--- a/yt_dlp/extractor/abc.py
+++ b/yt_dlp/extractor/abc.py
@@ -180,7 +180,6 @@ class ABCIViewIE(InfoExtractor):
     _VALID_URL = r'https?://iview\.abc\.net\.au/(?:[^/]+/)*video/(?P<id>[^/?#]+)'
     _GEO_COUNTRIES = ['AU']
 
-    # ABC iview programs are normally available for 14 days only.
     _TESTS = [{
         'url': 'https://iview.abc.net.au/show/gruen/series/11/video/LE1927H001S00',
         'md5': '67715ce3c78426b11ba167d875ac6abf',
diff --git a/yt_dlp/extractor/ign.py b/yt_dlp/extractor/ign.py
index 64875f8ce..1c4f105e9 100644
--- a/yt_dlp/extractor/ign.py
+++ b/yt_dlp/extractor/ign.py
@@ -197,10 +197,6 @@ class IGNVideoIE(IGNBaseIE):
             'thumbnail': 'https://sm.ign.com/ign_me/video/h/how-hitman/how-hitman-aims-to-be-different-than-every-other-s_8z14.jpg',
             'duration': 298,
             'tags': 'count:13',
-            'display_id': '112203',
-            'thumbnail': 'https://sm.ign.com/ign_me/video/h/how-hitman/how-hitman-aims-to-be-different-than-every-other-s_8z14.jpg',
-            'duration': 298,
-            'tags': 'count:13',
         },
         'expected_warnings': ['HTTP Error 400: Bad Request'],
     }, {
diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py
index 4f3e691b7..8fba2bcf7 100644
--- a/yt_dlp/extractor/nebula.py
+++ b/yt_dlp/extractor/nebula.py
@@ -127,7 +127,6 @@ class NebulaIE(NebulaBaseIE):
                 'channel_id': 'lindsayellis',
                 'uploader': 'Lindsay Ellis',
                 'uploader_id': 'lindsayellis',
-                'timestamp': 1533009600,
                 'uploader_url': 'https://nebula.tv/lindsayellis',
                 'series': 'Lindsay Ellis',
                 'display_id': 'that-time-disney-remade-beauty-and-the-beast',
diff --git a/yt_dlp/extractor/peekvids.py b/yt_dlp/extractor/peekvids.py
index d1fc058b9..41f591b09 100644
--- a/yt_dlp/extractor/peekvids.py
+++ b/yt_dlp/extractor/peekvids.py
@@ -146,7 +146,6 @@ class PlayVidsIE(PeekVidsBaseIE):
             'uploader': 'Brazzers',
             'age_limit': 18,
             'view_count': int,
-            'age_limit': 18,
             'categories': list,
             'tags': list,
         },
diff --git a/yt_dlp/extractor/radiofrance.py b/yt_dlp/extractor/radiofrance.py
index 35f4b91dd..ec1b97631 100644
--- a/yt_dlp/extractor/radiofrance.py
+++ b/yt_dlp/extractor/radiofrance.py
@@ -82,7 +82,7 @@ class RadioFranceBaseIE(InfoExtractor):
     def _extract_data_from_webpage(self, webpage, display_id, key):
         return traverse_obj(self._search_json(
             r'\bconst\s+data\s*=', webpage, key, display_id,
-            contains_pattern=r'(\[\{.*?\}\]);', transform_source=js_to_json),
+            contains_pattern=r'\[\{(?s:.+)\}\]', transform_source=js_to_json),
             (..., 'data', key, {dict}), get_all=False) or {}
 
 
diff --git a/yt_dlp/extractor/rcs.py b/yt_dlp/extractor/rcs.py
index 028d3d90b..b865f63fb 100644
--- a/yt_dlp/extractor/rcs.py
+++ b/yt_dlp/extractor/rcs.py
@@ -239,10 +239,10 @@ class RCSEmbedsIE(RCSBaseIE):
         }
     }, {
         'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789',
-        'match_only': True
+        'only_matching': True
     }, {
         'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140',
-        'match_only': True
+        'only_matching': True
     }]
     _WEBPAGE_TESTS = [{
         'url': 'https://www.iodonna.it/video-iodonna/personaggi-video/monica-bellucci-piu-del-lavoro-oggi-per-me-sono-importanti-lamicizia-e-la-famiglia/',
@@ -325,7 +325,7 @@ class RCSIE(RCSBaseIE):
         }
     }, {
         'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945',
-        'match_only': True
+        'only_matching': True
     }]
 
 
diff --git a/yt_dlp/extractor/rokfin.py b/yt_dlp/extractor/rokfin.py
index 4a4d40bef..cad76f0c9 100644
--- a/yt_dlp/extractor/rokfin.py
+++ b/yt_dlp/extractor/rokfin.py
@@ -40,7 +40,6 @@ class RokfinIE(InfoExtractor):
             'channel': 'Jimmy Dore',
             'channel_id': 65429,
             'channel_url': 'https://rokfin.com/TheJimmyDoreShow',
-            'duration': 213.0,
             'availability': 'public',
             'live_status': 'not_live',
             'dislike_count': int,
diff --git a/yt_dlp/extractor/s4c.py b/yt_dlp/extractor/s4c.py
index 990ea2b44..67eff723b 100644
--- a/yt_dlp/extractor/s4c.py
+++ b/yt_dlp/extractor/s4c.py
@@ -78,7 +78,6 @@ class S4CSeriesIE(InfoExtractor):
         'info_dict': {
             'id': '864982911',
             'title': 'Iaith ar Daith',
-            'description': 'md5:e878ebf660dce89bd2ef521d7ce06397'
         },
     }, {
         'url': 'https://www.s4c.cymru/clic/series/866852587',
@@ -86,7 +85,6 @@ class S4CSeriesIE(InfoExtractor):
         'info_dict': {
             'id': '866852587',
             'title': 'FFIT Cymru',
-            'description': 'md5:abcb3c129cb68dbb6cd304fd33b07e96'
         },
     }]
 
diff --git a/yt_dlp/extractor/sovietscloset.py b/yt_dlp/extractor/sovietscloset.py
index 453016ccb..493eea2a6 100644
--- a/yt_dlp/extractor/sovietscloset.py
+++ b/yt_dlp/extractor/sovietscloset.py
@@ -76,7 +76,6 @@ class SovietsClosetIE(SovietsClosetBaseIE):
                 'title': 'Arma 3 - Zeus Games #5',
                 'uploader': 'SovietWomble',
                 'thumbnail': r're:^https?://.*\.b-cdn\.net/c0e5e76f-3a93-40b4-bf01-12343c2eec5d/thumbnail\.jpg$',
-                'uploader': 'SovietWomble',
                 'creator': 'SovietWomble',
                 'release_timestamp': 1461157200,
                 'release_date': '20160420',
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 023d8fd8c..a39d17cf1 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -902,7 +902,7 @@ def extract_relative_time(relative_time_text):
         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today', '8 yr ago'
         """
 
-        # XXX: this could be moved to a general function in utils.py
+        # XXX: this could be moved to a general function in utils/_utils.py
         # The relative time text strings are roughly the same as what
         # Javascript's Intl.RelativeTimeFormat function generates.
         # See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/RelativeTimeFormat
diff --git a/yt_dlp/networking/__init__.py b/yt_dlp/networking/__init__.py
index 5e8876484..5b1599a6d 100644
--- a/yt_dlp/networking/__init__.py
+++ b/yt_dlp/networking/__init__.py
@@ -1,4 +1,4 @@
-# flake8: noqa: 401
+# flake8: noqa: F401
 from .common import (
     HEADRequest,
     PUTRequest,
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index c327f7744..9e2bf33e4 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -337,7 +337,7 @@ def handle_sslerror(e: ssl.SSLError):
 
 def handle_response_read_exceptions(e):
     if isinstance(e, http.client.IncompleteRead):
-        raise IncompleteRead(partial=e.partial, cause=e, expected=e.expected) from e
+        raise IncompleteRead(partial=len(e.partial), cause=e, expected=e.expected) from e
     elif isinstance(e, ssl.SSLError):
         handle_sslerror(e)
     elif isinstance(e, (OSError, EOFError, http.client.HTTPException, *CONTENT_DECODE_ERRORS)):
diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py
index 465b18ba9..f58dc246e 100644
--- a/yt_dlp/networking/exceptions.py
+++ b/yt_dlp/networking/exceptions.py
@@ -75,10 +75,10 @@ def __repr__(self):
 
 
 class IncompleteRead(TransportError):
-    def __init__(self, partial, expected=None, **kwargs):
+    def __init__(self, partial: int, expected: int = None, **kwargs):
         self.partial = partial
         self.expected = expected
-        msg = f'{len(partial)} bytes read'
+        msg = f'{partial} bytes read'
         if expected is not None:
             msg += f', {expected} more expected'
 

From eaee21bf71889d495076037cbe590c8c0b21ef3a Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Sat, 23 Sep 2023 23:13:48 +0100
Subject: [PATCH 589/871] [ie/Monstercat] Add extractor (#8133)

Closes #8067
Authored by: garret1317
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/monstercat.py  | 79 +++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 yt_dlp/extractor/monstercat.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 9cda06d8f..691cac339 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1126,6 +1126,7 @@
     MofosexEmbedIE,
 )
 from .mojvideo import MojvideoIE
+from .monstercat import MonstercatIE
 from .morningstar import MorningstarIE
 from .motherless import (
     MotherlessIE,
diff --git a/yt_dlp/extractor/monstercat.py b/yt_dlp/extractor/monstercat.py
new file mode 100644
index 000000000..7f04825fc
--- /dev/null
+++ b/yt_dlp/extractor/monstercat.py
@@ -0,0 +1,79 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    extract_attributes,
+    get_element_by_class,
+    get_element_html_by_class,
+    get_element_text_and_html_by_tag,
+    int_or_none,
+    unified_strdate,
+    strip_or_none,
+    traverse_obj,
+    try_call,
+)
+
+
+class MonstercatIE(InfoExtractor):
+    _VALID_URL = r'https://www\.monstercat\.com/release/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.monstercat.com/release/742779548009',
+        'playlist_count': 20,
+        'info_dict': {
+            'title': 'The Secret Language of Trees',
+            'id': '742779548009',
+            'thumbnail': 'https://www.monstercat.com/release/742779548009/cover',
+            'release_year': 2023,
+            'release_date': '20230711',
+            'album': 'The Secret Language of Trees',
+            'album_artist': 'BT',
+        }
+    }]
+
+    def _extract_tracks(self, table, album_meta):
+        for td in re.findall(r'<tr[^<]*>((?:(?!</tr>)[\w\W])+)', table):  # regex by chatgpt due to lack of get_elements_by_tag
+            title = clean_html(try_call(
+                lambda: get_element_by_class('d-inline-flex flex-column', td).partition(' <span')[0]))
+            ids = extract_attributes(try_call(lambda: get_element_html_by_class('btn-play cursor-pointer mr-small', td)) or '')
+            track_id = ids.get('data-track-id')
+            release_id = ids.get('data-release-id')
+
+            track_number = int_or_none(try_call(lambda: get_element_by_class('py-xsmall', td)))
+            if not track_id or not release_id:
+                self.report_warning(f'Skipping track {track_number}, ID(s) not found')
+                self.write_debug(f'release_id={repr(release_id)} track_id={repr(track_id)}')
+                continue
+            yield {
+                **album_meta,
+                'title': title,
+                'track': title,
+                'track_number': track_number,
+                'artist': clean_html(try_call(lambda: get_element_by_class('d-block fs-xxsmall', td))),
+                'url': f'https://www.monstercat.com/api/release/{release_id}/track-stream/{track_id}',
+                'id': track_id,
+                'ext': 'mp3'
+            }
+
+    def _real_extract(self, url):
+        url_id = self._match_id(url)
+        html = self._download_webpage(url, url_id)
+        # wrap all `get_elements` in `try_call`, HTMLParser has problems with site's html
+        tracklist_table = try_call(lambda: get_element_by_class('table table-small', html)) or ''
+
+        title = try_call(lambda: get_element_text_and_html_by_tag('h1', html)[0])
+        date = traverse_obj(html, ({lambda html: get_element_by_class('font-italic mb-medium d-tablet-none d-phone-block',
+                            html).partition('Released ')}, 2, {strip_or_none}, {unified_strdate}))
+
+        album_meta = {
+            'title': title,
+            'album': title,
+            'thumbnail': f'https://www.monstercat.com/release/{url_id}/cover',
+            'album_artist': try_call(
+                lambda: get_element_by_class('h-normal text-uppercase mb-desktop-medium mb-smallish', html)),
+            'release_year': int_or_none(date[:4]) if date else None,
+            'release_date': date,
+        }
+
+        return self.playlist_result(
+            self._extract_tracks(tracklist_table, album_meta), playlist_id=url_id, **album_meta)

From 6636021206dad17c7745ae6bce6cb73d6f2ef319 Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Sun, 24 Sep 2023 06:15:01 +0800
Subject: [PATCH 590/871] [ie/PIAULIZAPortal] Add extractor (#7903)

Authored by: pzhlkj6612
---
 yt_dlp/extractor/_extractors.py    |  1 +
 yt_dlp/extractor/piaulizaportal.py | 70 ++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 yt_dlp/extractor/piaulizaportal.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 691cac339..49c35cf71 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1452,6 +1452,7 @@
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .piapro import PiaproIE
+from .piaulizaportal import PIAULIZAPortalIE
 from .picarto import (
     PicartoIE,
     PicartoVodIE,
diff --git a/yt_dlp/extractor/piaulizaportal.py b/yt_dlp/extractor/piaulizaportal.py
new file mode 100644
index 000000000..1eb6d92b7
--- /dev/null
+++ b/yt_dlp/extractor/piaulizaportal.py
@@ -0,0 +1,70 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_qs,
+    time_seconds,
+    traverse_obj,
+)
+
+
+class PIAULIZAPortalIE(InfoExtractor):
+    IE_DESC = 'ulizaportal.jp - PIA LIVE STREAM'
+    _VALID_URL = r'https?://(?:www\.)?ulizaportal\.jp/pages/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})'
+    _TESTS = [{
+        'url': 'https://ulizaportal.jp/pages/005f18b7-e810-5618-cb82-0987c5755d44',
+        'info_dict': {
+            'id': '005f18b7-e810-5618-cb82-0987c5755d44',
+            'title': 'プレゼンテーションプレイヤーのサンプル',
+            'live_status': 'not_live',
+        },
+        'params': {
+            'skip_download': True,
+            'ignore_no_formats_error': True,
+        },
+    }, {
+        'url': 'https://ulizaportal.jp/pages/005e1b23-fe93-5780-19a0-98e917cc4b7d?expires=4102412400&signature=f422a993b683e1068f946caf406d211c17d1ef17da8bef3df4a519502155aa91&version=1',
+        'info_dict': {
+            'id': '005e1b23-fe93-5780-19a0-98e917cc4b7d',
+            'title': '【確認用】視聴サンプルページ（ULIZA）',
+            'live_status': 'not_live',
+        },
+        'params': {
+            'skip_download': True,
+            'ignore_no_formats_error': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        expires = int_or_none(traverse_obj(parse_qs(url), ('expires', 0)))
+        if expires and expires <= time_seconds():
+            raise ExtractorError('The link is expired.', video_id=video_id, expected=True)
+
+        webpage = self._download_webpage(url, video_id)
+
+        player_data = self._download_webpage(
+            self._search_regex(
+                r'<script [^>]*\bsrc="(https://player-api\.p\.uliza\.jp/v1/players/[^"]+)"',
+                webpage, 'player data url'),
+            video_id, headers={'Referer': 'https://ulizaportal.jp/'},
+            note='Fetching player data', errnote='Unable to fetch player data')
+
+        formats = self._extract_m3u8_formats(
+            self._search_regex(
+                r'["\'](https://vms-api\.p\.uliza\.jp/v1/prog-index\.m3u8[^"\']+)', player_data,
+                'm3u8 url', default=None),
+            video_id, fatal=False)
+        m3u8_type = self._search_regex(
+            r'/hls/(dvr|video)/', traverse_obj(formats, (0, 'url')), 'm3u8 type', default=None)
+
+        return {
+            'id': video_id,
+            'title': self._html_extract_title(webpage),
+            'formats': formats,
+            'live_status': {
+                'video': 'is_live',
+                'dvr': 'was_live',  # short-term archives
+            }.get(m3u8_type, 'not_live'),  # VOD or long-term archives
+        }

From 15591940ff102d1ae337d603a46d8f238c83a61f Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Sat, 23 Sep 2023 23:27:13 +0100
Subject: [PATCH 591/871] [ie/cineverse] Add extractors (#8146)

Also removes AsianCrushIE and AsianCrushPlaylistIE (URLs do not work anymore & old IDs are unavailable).

Closes #8109
Authored by: garret1317
---
 yt_dlp/extractor/_extractors.py |   8 +-
 yt_dlp/extractor/asiancrush.py  | 196 --------------------------------
 yt_dlp/extractor/cineverse.py   | 136 ++++++++++++++++++++++
 3 files changed, 140 insertions(+), 200 deletions(-)
 delete mode 100644 yt_dlp/extractor/asiancrush.py
 create mode 100644 yt_dlp/extractor/cineverse.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 49c35cf71..2535ed929 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -137,10 +137,6 @@
     ArteTVCategoryIE,
 )
 from .arnes import ArnesIE
-from .asiancrush import (
-    AsianCrushIE,
-    AsianCrushPlaylistIE,
-)
 from .atresplayer import AtresPlayerIE
 from .atscaleconf import AtScaleConfEventIE
 from .atttechchannel import ATTTechChannelIE
@@ -358,6 +354,10 @@
 from .cinchcast import CinchcastIE
 from .cinemax import CinemaxIE
 from .cinetecamilano import CinetecaMilanoIE
+from .cineverse import (
+    CineverseIE,
+    CineverseDetailsIE,
+)
 from .ciscolive import (
     CiscoLiveSessionIE,
     CiscoLiveSearchIE,
diff --git a/yt_dlp/extractor/asiancrush.py b/yt_dlp/extractor/asiancrush.py
deleted file mode 100644
index 23f310edb..000000000
--- a/yt_dlp/extractor/asiancrush.py
+++ /dev/null
@@ -1,196 +0,0 @@
-import functools
-import re
-
-from .common import InfoExtractor
-from .kaltura import KalturaIE
-from ..utils import (
-    extract_attributes,
-    int_or_none,
-    OnDemandPagedList,
-    parse_age_limit,
-    strip_or_none,
-    try_get,
-)
-
-
-class AsianCrushBaseIE(InfoExtractor):
-    _VALID_URL_BASE = r'https?://(?:www\.)?(?P<host>(?:(?:asiancrush|yuyutv|midnightpulp)\.com|(?:cocoro|retrocrush)\.tv))'
-    _KALTURA_KEYS = [
-        'video_url', 'progressive_url', 'download_url', 'thumbnail_url',
-        'widescreen_thumbnail_url', 'screencap_widescreen',
-    ]
-    _API_SUFFIX = {'retrocrush.tv': '-ott'}
-
-    def _call_api(self, host, endpoint, video_id, query, resource):
-        return self._download_json(
-            'https://api%s.%s/%s' % (self._API_SUFFIX.get(host, ''), host, endpoint), video_id,
-            'Downloading %s JSON metadata' % resource, query=query,
-            headers=self.geo_verification_headers())['objects']
-
-    def _download_object_data(self, host, object_id, resource):
-        return self._call_api(
-            host, 'search', object_id, {'id': object_id}, resource)[0]
-
-    def _get_object_description(self, obj):
-        return strip_or_none(obj.get('long_description') or obj.get('short_description'))
-
-    def _parse_video_data(self, video):
-        title = video['name']
-
-        entry_id, partner_id = [None] * 2
-        for k in self._KALTURA_KEYS:
-            k_url = video.get(k)
-            if k_url:
-                mobj = re.search(r'/p/(\d+)/.+?/entryId/([^/]+)/', k_url)
-                if mobj:
-                    partner_id, entry_id = mobj.groups()
-                    break
-
-        meta_categories = try_get(video, lambda x: x['meta']['categories'], list) or []
-        categories = list(filter(None, [c.get('name') for c in meta_categories]))
-
-        show_info = video.get('show_info') or {}
-
-        return {
-            '_type': 'url_transparent',
-            'url': 'kaltura:%s:%s' % (partner_id, entry_id),
-            'ie_key': KalturaIE.ie_key(),
-            'id': entry_id,
-            'title': title,
-            'description': self._get_object_description(video),
-            'age_limit': parse_age_limit(video.get('mpaa_rating') or video.get('tv_rating')),
-            'categories': categories,
-            'series': show_info.get('show_name'),
-            'season_number': int_or_none(show_info.get('season_num')),
-            'season_id': show_info.get('season_id'),
-            'episode_number': int_or_none(show_info.get('episode_num')),
-        }
-
-
-class AsianCrushIE(AsianCrushBaseIE):
-    _VALID_URL = r'%s/video/(?:[^/]+/)?0+(?P<id>\d+)v\b' % AsianCrushBaseIE._VALID_URL_BASE
-    _TESTS = [{
-        'url': 'https://www.asiancrush.com/video/004289v/women-who-flirt',
-        'md5': 'c3b740e48d0ba002a42c0b72857beae6',
-        'info_dict': {
-            'id': '1_y4tmjm5r',
-            'ext': 'mp4',
-            'title': 'Women Who Flirt',
-            'description': 'md5:b65c7e0ae03a85585476a62a186f924c',
-            'timestamp': 1496936429,
-            'upload_date': '20170608',
-            'uploader_id': 'craig@crifkin.com',
-            'age_limit': 13,
-            'categories': 'count:5',
-            'duration': 5812,
-        },
-    }, {
-        'url': 'https://www.asiancrush.com/video/she-was-pretty/011886v-pretty-episode-3/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.yuyutv.com/video/013886v/the-act-of-killing/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.yuyutv.com/video/peep-show/013922v-warring-factions/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.midnightpulp.com/video/010400v/drifters/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.midnightpulp.com/video/mononoke/016378v-zashikiwarashi-part-1/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.cocoro.tv/video/the-wonderful-wizard-of-oz/008878v-the-wonderful-wizard-of-oz-ep01/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.retrocrush.tv/video/true-tears/012328v-i...gave-away-my-tears',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        host, video_id = self._match_valid_url(url).groups()
-
-        if host == 'cocoro.tv':
-            webpage = self._download_webpage(url, video_id)
-            embed_vars = self._parse_json(self._search_regex(
-                r'iEmbedVars\s*=\s*({.+?})', webpage, 'embed vars',
-                default='{}'), video_id, fatal=False) or {}
-            video_id = embed_vars.get('entry_id') or video_id
-
-        video = self._download_object_data(host, video_id, 'video')
-        return self._parse_video_data(video)
-
-
-class AsianCrushPlaylistIE(AsianCrushBaseIE):
-    _VALID_URL = r'%s/series/0+(?P<id>\d+)s\b' % AsianCrushBaseIE._VALID_URL_BASE
-    _TESTS = [{
-        'url': 'https://www.asiancrush.com/series/006447s/fruity-samurai',
-        'info_dict': {
-            'id': '6447',
-            'title': 'Fruity Samurai',
-            'description': 'md5:7535174487e4a202d3872a7fc8f2f154',
-        },
-        'playlist_count': 13,
-    }, {
-        'url': 'https://www.yuyutv.com/series/013920s/peep-show/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.midnightpulp.com/series/016375s/mononoke/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.cocoro.tv/series/008549s/the-wonderful-wizard-of-oz/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.retrocrush.tv/series/012355s/true-tears',
-        'only_matching': True,
-    }]
-    _PAGE_SIZE = 1000000000
-
-    def _fetch_page(self, domain, parent_id, page):
-        videos = self._call_api(
-            domain, 'getreferencedobjects', parent_id, {
-                'max': self._PAGE_SIZE,
-                'object_type': 'video',
-                'parent_id': parent_id,
-                'start': page * self._PAGE_SIZE,
-            }, 'page %d' % (page + 1))
-        for video in videos:
-            yield self._parse_video_data(video)
-
-    def _real_extract(self, url):
-        host, playlist_id = self._match_valid_url(url).groups()
-
-        if host == 'cocoro.tv':
-            webpage = self._download_webpage(url, playlist_id)
-
-            entries = []
-
-            for mobj in re.finditer(
-                    r'<a[^>]+href=(["\'])(?P<url>%s.*?)\1[^>]*>' % AsianCrushIE._VALID_URL,
-                    webpage):
-                attrs = extract_attributes(mobj.group(0))
-                if attrs.get('class') == 'clearfix':
-                    entries.append(self.url_result(
-                        mobj.group('url'), ie=AsianCrushIE.ie_key()))
-
-            title = self._html_search_regex(
-                r'(?s)<h1\b[^>]\bid=["\']movieTitle[^>]+>(.+?)</h1>', webpage,
-                'title', default=None) or self._og_search_title(
-                webpage, default=None) or self._html_search_meta(
-                'twitter:title', webpage, 'title',
-                default=None) or self._html_extract_title(webpage)
-            if title:
-                title = re.sub(r'\s*\|\s*.+?$', '', title)
-
-            description = self._og_search_description(
-                webpage, default=None) or self._html_search_meta(
-                'twitter:description', webpage, 'description', fatal=False)
-        else:
-            show = self._download_object_data(host, playlist_id, 'show')
-            title = show.get('name')
-            description = self._get_object_description(show)
-            entries = OnDemandPagedList(
-                functools.partial(self._fetch_page, host, playlist_id),
-                self._PAGE_SIZE)
-
-        return self.playlist_result(entries, playlist_id, title, description)
diff --git a/yt_dlp/extractor/cineverse.py b/yt_dlp/extractor/cineverse.py
new file mode 100644
index 000000000..c9fa789b7
--- /dev/null
+++ b/yt_dlp/extractor/cineverse.py
@@ -0,0 +1,136 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    filter_dict,
+    int_or_none,
+    parse_age_limit,
+    smuggle_url,
+    traverse_obj,
+    unsmuggle_url,
+    url_or_none,
+)
+
+
+class CineverseBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https://www\.(?P<host>%s)' % '|'.join(map(re.escape, (
+        'cineverse.com',
+        'asiancrush.com',
+        'dovechannel.com',
+        'screambox.com',
+        'midnightpulp.com',
+        'fandor.com',
+        'retrocrush.tv',
+    )))
+
+
+class CineverseIE(CineverseBaseIE):
+    _VALID_URL = rf'{CineverseBaseIE._VALID_URL_BASE}/watch/(?P<id>[A-Z0-9]+)'
+    _TESTS = [{
+        'url': 'https://www.asiancrush.com/watch/DMR00018919/Women-Who-Flirt',
+        'skip': 'geo-blocked',
+        'info_dict': {
+            'title': 'Women Who Flirt',
+            'ext': 'mp4',
+            'id': 'DMR00018919',
+            'modified_timestamp': 1678744575289,
+            'cast': ['Xun Zhou', 'Xiaoming Huang', 'Yi-Lin Sie', 'Sonia Sui', 'Quniciren'],
+            'duration': 5811.597,
+            'description': 'md5:892fd62a05611d394141e8394ace0bc6',
+            'age_limit': 13,
+        }
+    }, {
+        'url': 'https://www.retrocrush.tv/watch/1000000023016/Archenemy! Crystal Bowie',
+        'skip': 'geo-blocked',
+        'info_dict': {
+            'title': 'Archenemy! Crystal Bowie',
+            'ext': 'mp4',
+            'id': '1000000023016',
+            'episode_number': 3,
+            'season_number': 1,
+            'cast': ['Nachi Nozawa', 'Yoshiko Sakakibara', 'Toshiko Fujita'],
+            'age_limit': 0,
+            'episode': 'Episode 3',
+            'season': 'Season 1',
+            'duration': 1485.067,
+            'description': 'Cobra meets a beautiful bounty hunter by the name of Jane Royal.',
+            'series': 'Space Adventure COBRA (Original Japanese)',
+        }
+    }]
+
+    def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, default={})
+        self._initialize_geo_bypass({
+            'countries': smuggled_data.get('geo_countries'),
+        })
+        video_id = self._match_id(url)
+        html = self._download_webpage(url, video_id)
+        idetails = self._search_nextjs_data(html, video_id)['props']['pageProps']['idetails']
+
+        if idetails.get('err_code') == 1200:
+            self.raise_geo_restricted(
+                'This video is not available from your location due to geo restriction. '
+                'You may be able to bypass it by using the /details/ page instead of the /watch/ page',
+                countries=smuggled_data.get('geo_countries'))
+
+        return {
+            'subtitles': filter_dict({
+                'en': traverse_obj(idetails, (('cc_url_vtt', 'subtitle_url'), {'url': {url_or_none}})) or None,
+            }),
+            'formats': self._extract_m3u8_formats(idetails['url'], video_id),
+            **traverse_obj(idetails, {
+                'title': 'title',
+                'id': ('details', 'item_id'),
+                'description': ('details', 'description'),
+                'duration': ('duration', {lambda x: x / 1000}),
+                'cast': ('details', 'cast', {lambda x: x.split(', ')}),
+                'modified_timestamp': ('details', 'updated_by', 0, 'update_time', 'time', {int_or_none}),
+                'season_number': ('details', 'season', {int_or_none}),
+                'episode_number': ('details', 'episode', {int_or_none}),
+                'age_limit': ('details', 'rating_code', {parse_age_limit}),
+                'series': ('details', 'series_details', 'title'),
+            }),
+        }
+
+
+class CineverseDetailsIE(CineverseBaseIE):
+    _VALID_URL = rf'{CineverseBaseIE._VALID_URL_BASE}/details/(?P<id>[A-Z0-9]+)'
+    _TESTS = [{
+        'url': 'https://www.retrocrush.tv/details/1000000023012/Space-Adventure-COBRA-(Original-Japanese)',
+        'playlist_mincount': 30,
+        'info_dict': {
+            'title': 'Space Adventure COBRA (Original Japanese)',
+            'id': '1000000023012',
+        }
+    }, {
+        'url': 'https://www.asiancrush.com/details/NNVG4938/Hansel-and-Gretel',
+        'info_dict': {
+            'id': 'NNVG4938',
+            'ext': 'mp4',
+            'title': 'Hansel and Gretel',
+            'description': 'md5:e3e4c35309c2e82aee044f972c2fb05d',
+            'cast': ['Jeong-myeong Cheon', 'Eun Won-jae', 'Shim Eun-gyeong', 'Ji-hee Jin', 'Hee-soon Park', 'Lydia Park', 'Kyeong-ik Kim'],
+            'duration': 7030.732,
+        },
+    }]
+
+    def _real_extract(self, url):
+        host, series_id = self._match_valid_url(url).group('host', 'id')
+        html = self._download_webpage(url, series_id)
+        pageprops = self._search_nextjs_data(html, series_id)['props']['pageProps']
+
+        geo_countries = traverse_obj(pageprops, ('itemDetailsData', 'geo_country', {lambda x: x.split(', ')}))
+        geoblocked = traverse_obj(pageprops, (
+            'itemDetailsData', 'playback_err_msg')) == 'This title is not available in your location.'
+
+        def item_result(item):
+            item_url = f'https://www.{host}/watch/{item["item_id"]}/{item["title"]}'
+            if geoblocked:
+                item_url = smuggle_url(item_url, {'geo_countries': geo_countries})
+            return self.url_result(item_url, CineverseIE)
+
+        season = traverse_obj(pageprops, ('seasonEpisodes', ..., 'episodes', lambda _, v: v['item_id'] and v['title']))
+        if season:
+            return self.playlist_result([item_result(ep) for ep in season], playlist_id=series_id,
+                                        playlist_title=traverse_obj(pageprops, ('itemDetailsData', 'title')))
+        return item_result(pageprops['itemDetailsData'])

From 698beb9a497f51693e64d167e572ff9efa4bc25f Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Sun, 24 Sep 2023 06:36:34 +0800
Subject: [PATCH 592/871] [ie/niconicochannelplus] Add extractors (#5686)

Closes #2537
Authored by: pzhlkj6612
---
 README.md                               |   3 +
 yt_dlp/extractor/_extractors.py         |   5 +
 yt_dlp/extractor/niconicochannelplus.py | 426 ++++++++++++++++++++++++
 3 files changed, 434 insertions(+)
 create mode 100644 yt_dlp/extractor/niconicochannelplus.py

diff --git a/README.md b/README.md
index d9b11952d..40515d48d 100644
--- a/README.md
+++ b/README.md
@@ -1845,6 +1845,9 @@ #### hotstar
 * `vcodec`: vcodec to ignore - one or more of `h264`, `h265`, `dvh265`
 * `dr`: dynamic range to ignore - one or more of `sdr`, `hdr10`, `dv`
 
+#### niconicochannelplus
+* `max_comments`: Maximum number of comments to extract - default is `120`
+
 #### tiktok
 * `api_hostname`: Hostname to use for mobile API requests, e.g. `api-h2.tiktokv.com`
 * `app_version`: App version to call mobile APIs with - should be set along with `manifest_app_version`, e.g. `20.2.1`
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 2535ed929..f325864d1 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1299,6 +1299,11 @@
     NineCNineMediaIE,
     CPTwentyFourIE,
 )
+from .niconicochannelplus import (
+    NiconicoChannelPlusIE,
+    NiconicoChannelPlusChannelVideosIE,
+    NiconicoChannelPlusChannelLivesIE,
+)
 from .ninegag import NineGagIE
 from .ninenow import NineNowIE
 from .nintendo import NintendoIE
diff --git a/yt_dlp/extractor/niconicochannelplus.py b/yt_dlp/extractor/niconicochannelplus.py
new file mode 100644
index 000000000..89af3f7b5
--- /dev/null
+++ b/yt_dlp/extractor/niconicochannelplus.py
@@ -0,0 +1,426 @@
+import functools
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    OnDemandPagedList,
+    filter_dict,
+    int_or_none,
+    parse_qs,
+    str_or_none,
+    traverse_obj,
+    unified_timestamp,
+    url_or_none,
+)
+
+
+class NiconicoChannelPlusBaseIE(InfoExtractor):
+    _WEBPAGE_BASE_URL = 'https://nicochannel.jp'
+
+    def _call_api(self, path, item_id, *args, **kwargs):
+        return self._download_json(
+            f'https://nfc-api.nicochannel.jp/fc/{path}', video_id=item_id, *args, **kwargs)
+
+    def _find_fanclub_site_id(self, channel_name):
+        fanclub_list_json = self._call_api(
+            'content_providers/channels', item_id=f'channels/{channel_name}',
+            note='Fetching channel list', errnote='Unable to fetch channel list',
+        )['data']['content_providers']
+        fanclub_id = traverse_obj(fanclub_list_json, (
+            lambda _, v: v['domain'] == f'{self._WEBPAGE_BASE_URL}/{channel_name}', 'id'),
+            get_all=False)
+        if not fanclub_id:
+            raise ExtractorError(f'Channel {channel_name} does not exist', expected=True)
+        return fanclub_id
+
+    def _get_channel_base_info(self, fanclub_site_id):
+        return traverse_obj(self._call_api(
+            f'fanclub_sites/{fanclub_site_id}/page_base_info', item_id=f'fanclub_sites/{fanclub_site_id}',
+            note='Fetching channel base info', errnote='Unable to fetch channel base info', fatal=False,
+        ), ('data', 'fanclub_site', {dict})) or {}
+
+    def _get_channel_user_info(self, fanclub_site_id):
+        return traverse_obj(self._call_api(
+            f'fanclub_sites/{fanclub_site_id}/user_info', item_id=f'fanclub_sites/{fanclub_site_id}',
+            note='Fetching channel user info', errnote='Unable to fetch channel user info', fatal=False,
+            data=json.dumps('null').encode('ascii'),
+        ), ('data', 'fanclub_site', {dict})) or {}
+
+
+class NiconicoChannelPlusIE(NiconicoChannelPlusBaseIE):
+    IE_NAME = 'NiconicoChannelPlus'
+    IE_DESC = 'ニコニコチャンネルプラス'
+    _VALID_URL = r'https?://nicochannel\.jp/(?P<channel>[\w.-]+)/(?:video|live)/(?P<code>sm\w+)'
+    _TESTS = [{
+        'url': 'https://nicochannel.jp/kaorin/video/smsDd8EdFLcVZk9yyAhD6H7H',
+        'info_dict': {
+            'id': 'smsDd8EdFLcVZk9yyAhD6H7H',
+            'title': '前田佳織里はニコ生がしたい！',
+            'ext': 'mp4',
+            'channel': '前田佳織里の世界攻略計画',
+            'channel_id': 'kaorin',
+            'channel_url': 'https://nicochannel.jp/kaorin',
+            'live_status': 'not_live',
+            'thumbnail': 'https://nicochannel.jp/public_html/contents/video_pages/74/thumbnail_path',
+            'description': '２０２１年１１月に放送された\n「前田佳織里はニコ生がしたい！」アーカイブになります。',
+            'timestamp': 1641360276,
+            'duration': 4097,
+            'comment_count': int,
+            'view_count': int,
+            'tags': [],
+            'upload_date': '20220105',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # age limited video; test purpose channel.
+        'url': 'https://nicochannel.jp/testman/video/smDXbcrtyPNxLx9jc4BW69Ve',
+        'info_dict': {
+            'id': 'smDXbcrtyPNxLx9jc4BW69Ve',
+            'title': 'test oshiro',
+            'ext': 'mp4',
+            'channel': '本番チャンネルプラステストマン',
+            'channel_id': 'testman',
+            'channel_url': 'https://nicochannel.jp/testman',
+            'age_limit': 18,
+            'live_status': 'was_live',
+            'timestamp': 1666344616,
+            'duration': 86465,
+            'comment_count': int,
+            'view_count': int,
+            'tags': [],
+            'upload_date': '20221021',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        content_code, channel_id = self._match_valid_url(url).group('code', 'channel')
+        fanclub_site_id = self._find_fanclub_site_id(channel_id)
+
+        data_json = self._call_api(
+            f'video_pages/{content_code}', item_id=content_code, headers={'fc_use_device': 'null'},
+            note='Fetching video page info', errnote='Unable to fetch video page info',
+        )['data']['video_page']
+
+        live_status, session_id = self._get_live_status_and_session_id(content_code, data_json)
+
+        release_timestamp_str = data_json.get('live_scheduled_start_at')
+
+        formats = []
+
+        if live_status == 'is_upcoming':
+            if release_timestamp_str:
+                msg = f'This live event will begin at {release_timestamp_str} UTC'
+            else:
+                msg = 'This event has not started yet'
+            self.raise_no_formats(msg, expected=True, video_id=content_code)
+        else:
+            formats = self._extract_m3u8_formats(
+                # "authenticated_url" is a format string that contains "{session_id}".
+                m3u8_url=data_json['video_stream']['authenticated_url'].format(session_id=session_id),
+                video_id=content_code)
+
+        return {
+            'id': content_code,
+            'formats': formats,
+            '_format_sort_fields': ('tbr', 'vcodec', 'acodec'),
+            'channel': self._get_channel_base_info(fanclub_site_id).get('fanclub_site_name'),
+            'channel_id': channel_id,
+            'channel_url': f'{self._WEBPAGE_BASE_URL}/{channel_id}',
+            'age_limit': traverse_obj(self._get_channel_user_info(fanclub_site_id), ('content_provider', 'age_limit')),
+            'live_status': live_status,
+            'release_timestamp': unified_timestamp(release_timestamp_str),
+            **traverse_obj(data_json, {
+                'title': ('title', {str}),
+                'thumbnail': ('thumbnail_url', {url_or_none}),
+                'description': ('description', {str}),
+                'timestamp': ('released_at', {unified_timestamp}),
+                'duration': ('active_video_filename', 'length', {int_or_none}),
+                'comment_count': ('video_aggregate_info', 'number_of_comments', {int_or_none}),
+                'view_count': ('video_aggregate_info', 'total_views', {int_or_none}),
+                'tags': ('video_tags', ..., 'tag', {str}),
+            }),
+            '__post_extractor': self.extract_comments(
+                content_code=content_code,
+                comment_group_id=traverse_obj(data_json, ('video_comment_setting', 'comment_group_id'))),
+        }
+
+    def _get_comments(self, content_code, comment_group_id):
+        item_id = f'{content_code}/comments'
+
+        if not comment_group_id:
+            return None
+
+        comment_access_token = self._call_api(
+            f'video_pages/{content_code}/comments_user_token', item_id,
+            note='Getting comment token', errnote='Unable to get comment token',
+        )['data']['access_token']
+
+        comment_list = self._download_json(
+            'https://comm-api.sheeta.com/messages.history', video_id=item_id,
+            note='Fetching comments', errnote='Unable to fetch comments',
+            headers={'Content-Type': 'application/json'},
+            query={
+                'sort_direction': 'asc',
+                'limit': int_or_none(self._configuration_arg('max_comments', [''])[0]) or 120,
+            },
+            data=json.dumps({
+                'token': comment_access_token,
+                'group_id': comment_group_id,
+            }).encode('ascii'))
+
+        for comment in traverse_obj(comment_list, ...):
+            yield traverse_obj(comment, {
+                'author': ('nickname', {str}),
+                'author_id': ('sender_id', {str_or_none}),
+                'id': ('id', {str_or_none}),
+                'text': ('message', {str}),
+                'timestamp': (('updated_at', 'sent_at', 'created_at'), {unified_timestamp}),
+                'author_is_uploader': ('sender_id', {lambda x: x == '-1'}),
+            }, get_all=False)
+
+    def _get_live_status_and_session_id(self, content_code, data_json):
+        video_type = data_json.get('type')
+        live_finished_at = data_json.get('live_finished_at')
+
+        payload = {}
+        if video_type == 'vod':
+            if live_finished_at:
+                live_status = 'was_live'
+            else:
+                live_status = 'not_live'
+        elif video_type == 'live':
+            if not data_json.get('live_started_at'):
+                return 'is_upcoming', ''
+
+            if not live_finished_at:
+                live_status = 'is_live'
+            else:
+                live_status = 'was_live'
+                payload = {'broadcast_type': 'dvr'}
+
+                video_allow_dvr_flg = traverse_obj(data_json, ('video', 'allow_dvr_flg'))
+                video_convert_to_vod_flg = traverse_obj(data_json, ('video', 'convert_to_vod_flg'))
+
+                self.write_debug(f'allow_dvr_flg = {video_allow_dvr_flg}, convert_to_vod_flg = {video_convert_to_vod_flg}.')
+
+                if not (video_allow_dvr_flg and video_convert_to_vod_flg):
+                    raise ExtractorError(
+                        'Live was ended, there is no video for download.', video_id=content_code, expected=True)
+        else:
+            raise ExtractorError(f'Unknown type: {video_type}', video_id=content_code, expected=False)
+
+        self.write_debug(f'{content_code}: video_type={video_type}, live_status={live_status}')
+
+        session_id = self._call_api(
+            f'video_pages/{content_code}/session_ids', item_id=f'{content_code}/session',
+            data=json.dumps(payload).encode('ascii'), headers={
+                'Content-Type': 'application/json',
+                'fc_use_device': 'null',
+                'origin': 'https://nicochannel.jp',
+            },
+            note='Getting session id', errnote='Unable to get session id',
+        )['data']['session_id']
+
+        return live_status, session_id
+
+
+class NiconicoChannelPlusChannelBaseIE(NiconicoChannelPlusBaseIE):
+    _PAGE_SIZE = 12
+
+    def _fetch_paged_channel_video_list(self, path, query, channel_name, item_id, page):
+        response = self._call_api(
+            path, item_id, query={
+                **query,
+                'page': (page + 1),
+                'per_page': self._PAGE_SIZE,
+            },
+            headers={'fc_use_device': 'null'},
+            note=f'Getting channel info (page {page + 1})',
+            errnote=f'Unable to get channel info (page {page + 1})')
+
+        for content_code in traverse_obj(response, ('data', 'video_pages', 'list', ..., 'content_code')):
+            # "video/{content_code}" works for both VOD and live, but "live/{content_code}" doesn't work for VOD
+            yield self.url_result(
+                f'{self._WEBPAGE_BASE_URL}/{channel_name}/video/{content_code}', NiconicoChannelPlusIE)
+
+
+class NiconicoChannelPlusChannelVideosIE(NiconicoChannelPlusChannelBaseIE):
+    IE_NAME = 'NiconicoChannelPlus:channel:videos'
+    IE_DESC = 'ニコニコチャンネルプラス - チャンネル - 動画リスト. nicochannel.jp/channel/videos'
+    _VALID_URL = r'https?://nicochannel\.jp/(?P<id>[a-z\d\._-]+)/videos(?:\?.*)?'
+    _TESTS = [{
+        # query: None
+        'url': 'https://nicochannel.jp/testman/videos',
+        'info_dict': {
+            'id': 'testman-videos',
+            'title': '本番チャンネルプラステストマン-videos',
+        },
+        'playlist_mincount': 18,
+    }, {
+        # query: None
+        'url': 'https://nicochannel.jp/testtarou/videos',
+        'info_dict': {
+            'id': 'testtarou-videos',
+            'title': 'チャンネルプラステスト太郎-videos',
+        },
+        'playlist_mincount': 2,
+    }, {
+        # query: None
+        'url': 'https://nicochannel.jp/testjirou/videos',
+        'info_dict': {
+            'id': 'testjirou-videos',
+            'title': 'チャンネルプラステスト二郎-videos',
+        },
+        'playlist_mincount': 12,
+    }, {
+        # query: tag
+        'url': 'https://nicochannel.jp/testman/videos?tag=%E6%A4%9C%E8%A8%BC%E7%94%A8',
+        'info_dict': {
+            'id': 'testman-videos',
+            'title': '本番チャンネルプラステストマン-videos',
+        },
+        'playlist_mincount': 6,
+    }, {
+        # query: vodType
+        'url': 'https://nicochannel.jp/testman/videos?vodType=1',
+        'info_dict': {
+            'id': 'testman-videos',
+            'title': '本番チャンネルプラステストマン-videos',
+        },
+        'playlist_mincount': 18,
+    }, {
+        # query: sort
+        'url': 'https://nicochannel.jp/testman/videos?sort=-released_at',
+        'info_dict': {
+            'id': 'testman-videos',
+            'title': '本番チャンネルプラステストマン-videos',
+        },
+        'playlist_mincount': 18,
+    }, {
+        # query: tag, vodType
+        'url': 'https://nicochannel.jp/testman/videos?tag=%E6%A4%9C%E8%A8%BC%E7%94%A8&vodType=1',
+        'info_dict': {
+            'id': 'testman-videos',
+            'title': '本番チャンネルプラステストマン-videos',
+        },
+        'playlist_mincount': 6,
+    }, {
+        # query: tag, sort
+        'url': 'https://nicochannel.jp/testman/videos?tag=%E6%A4%9C%E8%A8%BC%E7%94%A8&sort=-released_at',
+        'info_dict': {
+            'id': 'testman-videos',
+            'title': '本番チャンネルプラステストマン-videos',
+        },
+        'playlist_mincount': 6,
+    }, {
+        # query: vodType, sort
+        'url': 'https://nicochannel.jp/testman/videos?vodType=1&sort=-released_at',
+        'info_dict': {
+            'id': 'testman-videos',
+            'title': '本番チャンネルプラステストマン-videos',
+        },
+        'playlist_mincount': 18,
+    }, {
+        # query: tag, vodType, sort
+        'url': 'https://nicochannel.jp/testman/videos?tag=%E6%A4%9C%E8%A8%BC%E7%94%A8&vodType=1&sort=-released_at',
+        'info_dict': {
+            'id': 'testman-videos',
+            'title': '本番チャンネルプラステストマン-videos',
+        },
+        'playlist_mincount': 6,
+    }]
+
+    def _real_extract(self, url):
+        """
+        API parameters:
+            sort:
+                -released_at         公開日が新しい順 (newest to oldest)
+                 released_at         公開日が古い順 (oldest to newest)
+                -number_of_vod_views 再生数が多い順 (most play count)
+                 number_of_vod_views コメントが多い順 (most comments)
+            vod_type (is "vodType" in "url"):
+                0 すべて (all)
+                1 会員限定 (members only)
+                2 一部無料 (partially free)
+                3 レンタル (rental)
+                4 生放送アーカイブ (live archives)
+                5 アップロード動画 (uploaded videos)
+        """
+
+        channel_id = self._match_id(url)
+        fanclub_site_id = self._find_fanclub_site_id(channel_id)
+        channel_name = self._get_channel_base_info(fanclub_site_id).get('fanclub_site_name')
+        qs = parse_qs(url)
+
+        return self.playlist_result(
+            OnDemandPagedList(
+                functools.partial(
+                    self._fetch_paged_channel_video_list, f'fanclub_sites/{fanclub_site_id}/video_pages',
+                    filter_dict({
+                        'tag': traverse_obj(qs, ('tag', 0)),
+                        'sort': traverse_obj(qs, ('sort', 0), default='-released_at'),
+                        'vod_type': traverse_obj(qs, ('vodType', 0), default='0'),
+                    }),
+                    channel_id, f'{channel_id}/videos'),
+                self._PAGE_SIZE),
+            playlist_id=f'{channel_id}-videos', playlist_title=f'{channel_name}-videos')
+
+
+class NiconicoChannelPlusChannelLivesIE(NiconicoChannelPlusChannelBaseIE):
+    IE_NAME = 'NiconicoChannelPlus:channel:lives'
+    IE_DESC = 'ニコニコチャンネルプラス - チャンネル - ライブリスト. nicochannel.jp/channel/lives'
+    _VALID_URL = r'https?://nicochannel\.jp/(?P<id>[a-z\d\._-]+)/lives'
+    _TESTS = [{
+        'url': 'https://nicochannel.jp/testman/lives',
+        'info_dict': {
+            'id': 'testman-lives',
+            'title': '本番チャンネルプラステストマン-lives',
+        },
+        'playlist_mincount': 18,
+    }, {
+        'url': 'https://nicochannel.jp/testtarou/lives',
+        'info_dict': {
+            'id': 'testtarou-lives',
+            'title': 'チャンネルプラステスト太郎-lives',
+        },
+        'playlist_mincount': 2,
+    }, {
+        'url': 'https://nicochannel.jp/testjirou/lives',
+        'info_dict': {
+            'id': 'testjirou-lives',
+            'title': 'チャンネルプラステスト二郎-lives',
+        },
+        'playlist_mincount': 6,
+    }]
+
+    def _real_extract(self, url):
+        """
+        API parameters:
+            live_type:
+                1 放送中 (on air)
+                2 放送予定 (scheduled live streams, oldest to newest)
+                3 過去の放送 - すべて (all ended live streams, newest to oldest)
+                4 過去の放送 - 生放送アーカイブ (all archives for live streams, oldest to newest)
+            We use "4" instead of "3" because some recently ended live streams could not be downloaded.
+        """
+
+        channel_id = self._match_id(url)
+        fanclub_site_id = self._find_fanclub_site_id(channel_id)
+        channel_name = self._get_channel_base_info(fanclub_site_id).get('fanclub_site_name')
+
+        return self.playlist_result(
+            OnDemandPagedList(
+                functools.partial(
+                    self._fetch_paged_channel_video_list, f'fanclub_sites/{fanclub_site_id}/live_pages',
+                    {
+                        'live_type': 4,
+                    },
+                    channel_id, f'{channel_id}/lives'),
+                self._PAGE_SIZE),
+            playlist_id=f'{channel_id}-lives', playlist_title=f'{channel_name}-lives')

From 92feb5654c5a4c81ba872904a618700fcbb3e546 Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Sun, 24 Sep 2023 07:42:29 +0800
Subject: [PATCH 593/871] [ie/brilliantpala] Add extractors (#6680)

Authored by: pzhlkj6612
---
 yt_dlp/extractor/_extractors.py   |   4 +
 yt_dlp/extractor/brilliantpala.py | 127 ++++++++++++++++++++++++++++++
 2 files changed, 131 insertions(+)
 create mode 100644 yt_dlp/extractor/brilliantpala.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index f325864d1..908abb8ac 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -271,6 +271,10 @@
     BrightcoveLegacyIE,
     BrightcoveNewIE,
 )
+from .brilliantpala import (
+    BrilliantpalaElearnIE,
+    BrilliantpalaClassesIE,
+)
 from .businessinsider import BusinessInsiderIE
 from .bundesliga import BundesligaIE
 from .buzzfeed import BuzzFeedIE
diff --git a/yt_dlp/extractor/brilliantpala.py b/yt_dlp/extractor/brilliantpala.py
new file mode 100644
index 000000000..6fd5b8148
--- /dev/null
+++ b/yt_dlp/extractor/brilliantpala.py
@@ -0,0 +1,127 @@
+import hashlib
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    traverse_obj,
+    urlencode_postdata,
+)
+
+
+class BrilliantpalaBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'brilliantpala'
+    _DOMAIN = '{subdomain}.brilliantpala.org'
+
+    def _initialize_pre_login(self):
+        self._HOMEPAGE = f'https://{self._DOMAIN}'
+        self._LOGIN_API = f'{self._HOMEPAGE}/login/'
+        self._LOGOUT_DEVICES_API = f'{self._HOMEPAGE}/logout_devices/?next=/'
+        self._CONTENT_API = f'{self._HOMEPAGE}/api/v2.4/contents/{{content_id}}/'
+        self._HLS_AES_URI = f'{self._HOMEPAGE}/api/v2.5/video_contents/{{content_id}}/key/'
+
+    def _get_logged_in_username(self, url, video_id):
+        webpage, urlh = self._download_webpage_handle(url, video_id)
+        if self._LOGIN_API == urlh.url:
+            self.raise_login_required()
+        return self._html_search_regex(
+            r'"username"\s*:\s*"(?P<username>[^"]+)"', webpage, 'stream page info', 'username')
+
+    def _perform_login(self, username, password):
+        login_form = self._hidden_inputs(self._download_webpage(
+            self._LOGIN_API, None, 'Downloading login page'))
+        login_form.update({
+            'username': username,
+            'password': password,
+        })
+        self._set_cookie(self._DOMAIN, 'csrftoken', login_form['csrfmiddlewaretoken'])
+
+        logged_page = self._download_webpage(
+            self._LOGIN_API, None, note='Logging in', headers={'Referer': self._LOGIN_API},
+            data=urlencode_postdata(login_form))
+
+        if self._html_search_regex(
+                r'(Your username / email and password)', logged_page, 'auth fail', default=None):
+            raise ExtractorError('wrong username or password', expected=True)
+
+        # the maximum number of logins is one
+        if self._html_search_regex(
+                r'(Logout Other Devices)', logged_page, 'logout devices button', default=None):
+            logout_device_form = self._hidden_inputs(logged_page)
+            self._download_webpage(
+                self._LOGOUT_DEVICES_API, None, headers={'Referer': self._LOGIN_API},
+                note='Logging out other devices', data=urlencode_postdata(logout_device_form))
+
+    def _real_extract(self, url):
+        course_id, content_id = self._match_valid_url(url).group('course_id', 'content_id')
+        video_id = f'{course_id}-{content_id}'
+
+        username = self._get_logged_in_username(url, video_id)
+
+        content_json = self._download_json(
+            self._CONTENT_API.format(content_id=content_id), video_id,
+            note='Fetching content info', errnote='Unable to fetch content info')
+
+        entries = []
+        for stream in traverse_obj(content_json, ('video', 'streams', lambda _, v: v['id'] and v['url'])):
+            formats = self._extract_m3u8_formats(stream['url'], video_id, fatal=False)
+            if not formats:
+                continue
+            entries.append({
+                'id': str(stream['id']),
+                'title': content_json.get('title'),
+                'formats': formats,
+                'hls_aes': {'uri': self._HLS_AES_URI.format(content_id=content_id)},
+                'http_headers': {'X-Key': hashlib.sha256(username.encode('ascii')).hexdigest()},
+                'thumbnail': content_json.get('cover_image'),
+            })
+
+        return self.playlist_result(
+            entries, playlist_id=video_id, playlist_title=content_json.get('title'))
+
+
+class BrilliantpalaElearnIE(BrilliantpalaBaseIE):
+    IE_NAME = 'Brilliantpala:Elearn'
+    IE_DESC = 'VoD on elearn.brilliantpala.org'
+    _VALID_URL = r'https?://elearn\.brilliantpala\.org/courses/(?P<course_id>\d+)/contents/(?P<content_id>\d+)/?'
+    _TESTS = [{
+        'url': 'https://elearn.brilliantpala.org/courses/42/contents/12345/',
+        'only_matching': True,
+    }, {
+        'url': 'https://elearn.brilliantpala.org/courses/98/contents/36683/',
+        'info_dict': {
+            'id': '23577',
+            'ext': 'mp4',
+            'title': 'Physical World, Units and Measurements  - 1',
+            'thumbnail': 'https://d1j3vi2u94ebt0.cloudfront.net/institute/brilliantpalalms/chapter_contents/26237/e657f81b90874be19795c7ea081f8d5c.png',
+            'live_status': 'not_live',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    _DOMAIN = BrilliantpalaBaseIE._DOMAIN.format(subdomain='elearn')
+
+
+class BrilliantpalaClassesIE(BrilliantpalaBaseIE):
+    IE_NAME = 'Brilliantpala:Classes'
+    IE_DESC = 'VoD on classes.brilliantpala.org'
+    _VALID_URL = r'https?://classes\.brilliantpala\.org/courses/(?P<course_id>\d+)/contents/(?P<content_id>\d+)/?'
+    _TESTS = [{
+        'url': 'https://classes.brilliantpala.org/courses/42/contents/12345/',
+        'only_matching': True,
+    }, {
+        'url': 'https://classes.brilliantpala.org/courses/416/contents/25445/',
+        'info_dict': {
+            'id': '9128',
+            'ext': 'mp4',
+            'title': 'Motion in a Straight Line - Class 1',
+            'thumbnail': 'https://d3e4y8hquds3ek.cloudfront.net/institute/brilliantpalaelearn/chapter_contents/ff5ba838d0ec43419f67387fe1a01fa8.png',
+            'live_status': 'not_live',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    _DOMAIN = BrilliantpalaBaseIE._DOMAIN.format(subdomain='classes')

From 1eaca74bc2ca0f5b1ec532f24c61de44f2e8cb2d Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 23 Sep 2023 18:47:14 -0500
Subject: [PATCH 594/871] [ie/nfl.com:plus:replay] Fix extractor (#7838)

Closes #7836
Authored by: bashonly
---
 README.md               |   3 +
 yt_dlp/extractor/nfl.py | 236 ++++++++++++++++++++++++++--------------
 2 files changed, 157 insertions(+), 82 deletions(-)

diff --git a/README.md b/README.md
index 40515d48d..7bf446572 100644
--- a/README.md
+++ b/README.md
@@ -1868,6 +1868,9 @@ #### twitch
 #### nhkradirulive (NHK らじる★らじる LIVE)
 * `area`: Which regional variation to extract. Valid areas are: `sapporo`, `sendai`, `tokyo`, `nagoya`, `osaka`, `hiroshima`, `matsuyama`, `fukuoka`. Defaults to `tokyo`
 
+#### nflplusreplay
+* `type`: Type(s) of game replays to extract. Valid types are: `full_game`, `full_game_spanish`, `condensed_game` and `all_22`. You can use `all` to extract all available replay types, which is the default
+
 **Note**: These options may be changed/removed in the future without concern for backward compatibility
 
 <!-- MANPAGE: MOVE "INSTALLATION" SECTION HERE -->
diff --git a/yt_dlp/extractor/nfl.py b/yt_dlp/extractor/nfl.py
index cc3f4495c..bd060dba9 100644
--- a/yt_dlp/extractor/nfl.py
+++ b/yt_dlp/extractor/nfl.py
@@ -64,6 +64,85 @@ class NFLBaseIE(InfoExtractor):
     _VIDEO_CONFIG_REGEX = r'<script[^>]+id="[^"]*video-config-[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}[^"]*"[^>]*>\s*({.+});?\s*</script>'
     _ANVATO_PREFIX = 'anvato:GXvEgwyJeWem8KCYXfeoHWknwP48Mboj:'
 
+    _CLIENT_DATA = {
+        'clientKey': '4cFUW6DmwJpzT9L7LrG3qRAcABG5s04g',
+        'clientSecret': 'CZuvCL49d9OwfGsR',
+        'deviceId': str(uuid.uuid4()),
+        'deviceInfo': base64.b64encode(json.dumps({
+            'model': 'desktop',
+            'version': 'Chrome',
+            'osName': 'Windows',
+            'osVersion': '10.0',
+        }, separators=(',', ':')).encode()).decode(),
+        'networkType': 'other',
+        'nflClaimGroupsToAdd': [],
+        'nflClaimGroupsToRemove': [],
+    }
+    _ACCOUNT_INFO = {}
+    _API_KEY = None
+
+    _TOKEN = None
+    _TOKEN_EXPIRY = 0
+
+    def _get_account_info(self, url, slug):
+        if not self._API_KEY:
+            webpage = self._download_webpage(url, slug, fatal=False) or ''
+            self._API_KEY = self._search_regex(
+                r'window\.gigyaApiKey\s*=\s*["\'](\w+)["\'];', webpage, 'API key',
+                fatal=False) or '3_Qa8TkWpIB8ESCBT8tY2TukbVKgO5F6BJVc7N1oComdwFzI7H2L9NOWdm11i_BY9f'
+
+        cookies = self._get_cookies('https://auth-id.nfl.com/')
+        login_token = traverse_obj(cookies, (
+            (f'glt_{self._API_KEY}', lambda k, _: k.startswith('glt_')), {lambda x: x.value}), get_all=False)
+        if not login_token:
+            self.raise_login_required()
+        if 'ucid' not in cookies:
+            raise ExtractorError(
+                'Required cookies for the auth-id.nfl.com domain were not found among passed cookies. '
+                'If using --cookies, these cookies must be exported along with .nfl.com cookies, '
+                'or else try using --cookies-from-browser instead', expected=True)
+
+        account = self._download_json(
+            'https://auth-id.nfl.com/accounts.getAccountInfo', slug,
+            note='Downloading account info', data=urlencode_postdata({
+                'include': 'profile,data',
+                'lang': 'en',
+                'APIKey': self._API_KEY,
+                'sdk': 'js_latest',
+                'login_token': login_token,
+                'authMode': 'cookie',
+                'pageURL': url,
+                'sdkBuild': traverse_obj(cookies, (
+                    'gig_canary_ver', {lambda x: x.value.partition('-')[0]}), default='15170'),
+                'format': 'json',
+            }), headers={'Content-Type': 'application/x-www-form-urlencoded'})
+
+        self._ACCOUNT_INFO = traverse_obj(account, {
+            'signatureTimestamp': 'signatureTimestamp',
+            'uid': 'UID',
+            'uidSignature': 'UIDSignature',
+        })
+
+        if len(self._ACCOUNT_INFO) != 3:
+            raise ExtractorError('Failed to retrieve account info with provided cookies', expected=True)
+
+    def _get_auth_token(self, url, slug):
+        if self._TOKEN and self._TOKEN_EXPIRY > int(time.time() + 30):
+            return
+
+        if not self._ACCOUNT_INFO:
+            self._get_account_info(url, slug)
+
+        token = self._download_json(
+            'https://api.nfl.com/identity/v3/token%s' % (
+                '/refresh' if self._ACCOUNT_INFO.get('refreshToken') else ''),
+            slug, headers={'Content-Type': 'application/json'}, note='Downloading access token',
+            data=json.dumps({**self._CLIENT_DATA, **self._ACCOUNT_INFO}, separators=(',', ':')).encode())
+
+        self._TOKEN = token['accessToken']
+        self._TOKEN_EXPIRY = token['expiresIn']
+        self._ACCOUNT_INFO['refreshToken'] = token['refreshToken']
+
     def _parse_video_config(self, video_config, display_id):
         video_config = self._parse_json(video_config, display_id)
         item = video_config['playlist'][0]
@@ -168,7 +247,7 @@ def _real_extract(self, url):
 
 class NFLPlusReplayIE(NFLBaseIE):
     IE_NAME = 'nfl.com:plus:replay'
-    _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/games/[\w-]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/games/(?P<slug>[\w-]+)(?:/(?P<id>\d+))?'
     _TESTS = [{
         'url': 'https://www.nfl.com/plus/games/giants-at-vikings-2022-post-1/1572108',
         'info_dict': {
@@ -185,23 +264,92 @@ class NFLPlusReplayIE(NFLBaseIE):
             'thumbnail': r're:^https?://.*\.jpg',
         },
         'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'Subscription required',
+        'url': 'https://www.nfl.com/plus/games/giants-at-vikings-2022-post-1',
+        'playlist_count': 4,
+        'info_dict': {
+            'id': 'giants-at-vikings-2022-post-1',
+        },
+    }, {
+        'note': 'Subscription required',
+        'url': 'https://www.nfl.com/plus/games/giants-at-patriots-2011-pre-4',
+        'playlist_count': 2,
+        'info_dict': {
+            'id': 'giants-at-patriots-2011-pre-4',
+        },
+    }, {
+        'note': 'Subscription required',
+        'url': 'https://www.nfl.com/plus/games/giants-at-patriots-2011-pre-4',
+        'info_dict': {
+            'id': '950701',
+            'ext': 'mp4',
+            'title': 'Giants @ Patriots',
+            'description': 'Giants at Patriots on September 01, 2011',
+            'uploader': 'NFL',
+            'upload_date': '20210724',
+            'timestamp': 1627085874,
+            'duration': 1532,
+            'categories': ['Game Highlights'],
+            'tags': ['play-by-play'],
+            'thumbnail': r're:^https?://.*\.jpg',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+            'extractor_args': {'nflplusreplay': {'type': ['condensed_game']}},
+        },
     }]
 
+    _REPLAY_TYPES = {
+        'full_game': 'Full Game',
+        'full_game_spanish': 'Full Game - Spanish',
+        'condensed_game': 'Condensed Game',
+        'all_22': 'All-22',
+    }
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        return self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id)
+        slug, video_id = self._match_valid_url(url).group('slug', 'id')
+        requested_types = self._configuration_arg('type', ['all'])
+        if 'all' in requested_types:
+            requested_types = list(self._REPLAY_TYPES.keys())
+        requested_types = traverse_obj(self._REPLAY_TYPES, (None, requested_types))
+
+        if not video_id:
+            self._get_auth_token(url, slug)
+            headers = {'Authorization': f'Bearer {self._TOKEN}'}
+            game_id = self._download_json(
+                f'https://api.nfl.com/football/v2/games/externalId/slug/{slug}', slug,
+                'Downloading game ID', query={'withExternalIds': 'true'}, headers=headers)['id']
+            replays = self._download_json(
+                'https://api.nfl.com/content/v1/videos/replays', slug, 'Downloading replays JSON',
+                query={'gameId': game_id}, headers=headers)
+            if len(requested_types) == 1:
+                video_id = traverse_obj(replays, (
+                    'items', lambda _, v: v['subType'] == requested_types[0], 'mcpPlaybackId'), get_all=False)
+
+        if video_id:
+            return self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id)
+
+        def entries():
+            for replay in traverse_obj(
+                replays, ('items', lambda _, v: v['mcpPlaybackId'] and v['subType'] in requested_types)
+            ):
+                video_id = replay['mcpPlaybackId']
+                yield self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id)
+
+        return self.playlist_result(entries(), slug)
 
 
 class NFLPlusEpisodeIE(NFLBaseIE):
     IE_NAME = 'nfl.com:plus:episode'
     _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/episodes/(?P<id>[\w-]+)'
     _TESTS = [{
-        'note': 'premium content',
+        'note': 'Subscription required',
         'url': 'https://www.nfl.com/plus/episodes/kurt-s-qb-insider-conference-championships',
         'info_dict': {
             'id': '1576832',
             'ext': 'mp4',
-            'title': 'Kurt\'s QB Insider: Conference Championships',
+            'title': 'Conference Championships',
             'description': 'md5:944f7fab56f7a37430bf8473f5473857',
             'uploader': 'NFL',
             'upload_date': '20230127',
@@ -214,85 +362,9 @@ class NFLPlusEpisodeIE(NFLBaseIE):
         'params': {'skip_download': 'm3u8'},
     }]
 
-    _CLIENT_DATA = {
-        'clientKey': '4cFUW6DmwJpzT9L7LrG3qRAcABG5s04g',
-        'clientSecret': 'CZuvCL49d9OwfGsR',
-        'deviceId': str(uuid.uuid4()),
-        'deviceInfo': base64.b64encode(json.dumps({
-            'model': 'desktop',
-            'version': 'Chrome',
-            'osName': 'Windows',
-            'osVersion': '10.0',
-        }, separators=(',', ':')).encode()).decode(),
-        'networkType': 'other',
-        'nflClaimGroupsToAdd': [],
-        'nflClaimGroupsToRemove': [],
-    }
-    _ACCOUNT_INFO = {}
-    _API_KEY = None
-
-    _TOKEN = None
-    _TOKEN_EXPIRY = 0
-
-    def _get_account_info(self, url, video_id):
-        cookies = self._get_cookies('https://www.nfl.com/')
-        login_token = traverse_obj(cookies, (
-            (f'glt_{self._API_KEY}', f'gig_loginToken_{self._API_KEY}',
-             lambda k, _: k.startswith('glt_') or k.startswith('gig_loginToken_')),
-            {lambda x: x.value}), get_all=False)
-        if not login_token:
-            self.raise_login_required()
-
-        account = self._download_json(
-            'https://auth-id.nfl.com/accounts.getAccountInfo', video_id,
-            note='Downloading account info', data=urlencode_postdata({
-                'include': 'profile,data',
-                'lang': 'en',
-                'APIKey': self._API_KEY,
-                'sdk': 'js_latest',
-                'login_token': login_token,
-                'authMode': 'cookie',
-                'pageURL': url,
-                'sdkBuild': traverse_obj(cookies, (
-                    'gig_canary_ver', {lambda x: x.value.partition('-')[0]}), default='13642'),
-                'format': 'json',
-            }), headers={'Content-Type': 'application/x-www-form-urlencoded'})
-
-        self._ACCOUNT_INFO = traverse_obj(account, {
-            'signatureTimestamp': 'signatureTimestamp',
-            'uid': 'UID',
-            'uidSignature': 'UIDSignature',
-        })
-
-        if len(self._ACCOUNT_INFO) != 3:
-            raise ExtractorError('Failed to retrieve account info with provided cookies', expected=True)
-
-    def _get_auth_token(self, url, video_id):
-        if not self._ACCOUNT_INFO:
-            self._get_account_info(url, video_id)
-
-        token = self._download_json(
-            'https://api.nfl.com/identity/v3/token%s' % (
-                '/refresh' if self._ACCOUNT_INFO.get('refreshToken') else ''),
-            video_id, headers={'Content-Type': 'application/json'}, note='Downloading access token',
-            data=json.dumps({**self._CLIENT_DATA, **self._ACCOUNT_INFO}, separators=(',', ':')).encode())
-
-        self._TOKEN = token['accessToken']
-        self._TOKEN_EXPIRY = token['expiresIn']
-        self._ACCOUNT_INFO['refreshToken'] = token['refreshToken']
-
     def _real_extract(self, url):
         slug = self._match_id(url)
-
-        if not self._API_KEY:
-            webpage = self._download_webpage(url, slug, fatal=False) or ''
-            self._API_KEY = self._search_regex(
-                r'window\.gigyaApiKey=["\'](\w+)["\'];', webpage, 'API key',
-                default='3_Qa8TkWpIB8ESCBT8tY2TukbVKgO5F6BJVc7N1oComdwFzI7H2L9NOWdm11i_BY9f')
-
-        if not self._TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
-            self._get_auth_token(url, slug)
-
+        self._get_auth_token(url, slug)
         video_id = self._download_json(
             f'https://api.nfl.com/content/v1/videos/episodes/{slug}', slug, headers={
                 'Authorization': f'Bearer {self._TOKEN}',

From 61bdf15fc7400601c3da1aa7a43917310a5bf391 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 24 Sep 2023 02:24:47 +0200
Subject: [PATCH 595/871] [core] Raise minimum recommended Python version to
 3.8 (#8183)

Authored by: Grub4K
---
 devscripts/changelog_override.json |  5 +++++
 test/test_execution.py             |  3 +++
 yt_dlp/YoutubeDL.py                | 16 ++++------------
 yt_dlp/update.py                   | 25 +++++++++++++++++++++++++
 4 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index e7f453acf..9dfbf510f 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -88,5 +88,10 @@
         "when": "59e92b1f1833440bb2190f847eb735cf0f90bc85",
         "short": "[rh:urllib] Simplify gzip decoding (#7611)",
         "authors": ["Grub4K"]
+    },
+    {
+        "action": "add",
+        "when": "c1d71d0d9f41db5e4306c86af232f5f6220a130b",
+        "short": "[priority] **The minimum *recommended* Python version has been raised to 3.8**\nSince Python 3.7 has reached end-of-life, support for it will be dropped soon. [Read more](https://github.com/yt-dlp/yt-dlp/issues/7803)"
     }
 ]
diff --git a/test/test_execution.py b/test/test_execution.py
index 7a9e800b6..fb2f6e2e9 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -45,6 +45,9 @@ def test_lazy_extractors(self):
             self.assertTrue(os.path.exists(LAZY_EXTRACTORS))
 
             _, stderr = self.run_yt_dlp(opts=('-s', 'test:'))
+            # `MIN_RECOMMENDED` emits a deprecated feature warning for deprecated python versions
+            if stderr and stderr.startswith('Deprecated Feature: Support for Python'):
+                stderr = ''
             self.assertFalse(stderr)
 
             subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=subprocess.DEVNULL)
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 39aaf2c2e..f322b12a2 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -60,7 +60,7 @@
     get_postprocessor,
 )
 from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
-from .update import REPOSITORY, current_git_head, detect_variant
+from .update import REPOSITORY, _get_system_deprecation, current_git_head, detect_variant
 from .utils import (
     DEFAULT_OUTTMPL,
     IDENTITY,
@@ -640,17 +640,9 @@ def process_color_policy(stream):
             for name, stream in self._out_files.items_ if name != 'console'
         })
 
-        # The code is left like this to be reused for future deprecations
-        MIN_SUPPORTED, MIN_RECOMMENDED = (3, 7), (3, 7)
-        current_version = sys.version_info[:2]
-        if current_version < MIN_RECOMMENDED:
-            msg = ('Support for Python version %d.%d has been deprecated. '
-                   'See  https://github.com/yt-dlp/yt-dlp/issues/3764  for more details.'
-                   '\n                    You will no longer receive updates on this version')
-            if current_version < MIN_SUPPORTED:
-                msg = 'Python version %d.%d is no longer supported'
-            self.deprecated_feature(
-                f'{msg}! Please update to Python %d.%d or above' % (*current_version, *MIN_RECOMMENDED))
+        system_deprecation = _get_system_deprecation()
+        if system_deprecation:
+            self.deprecated_feature(system_deprecation.replace('\n', '\n                    '))
 
         if self.params.get('allow_unplayable_formats'):
             self.report_warning(
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index d708b09e3..db79df127 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -112,6 +112,31 @@ def is_non_updateable():
         detect_variant(), _NON_UPDATEABLE_REASONS['unknown' if VARIANT else 'other'])
 
 
+def _get_system_deprecation():
+    MIN_SUPPORTED, MIN_RECOMMENDED = (3, 7), (3, 8)
+
+    if sys.version_info > MIN_RECOMMENDED:
+        return None
+
+    major, minor = sys.version_info[:2]
+    if sys.version_info < MIN_SUPPORTED:
+        msg = f'Python version {major}.{minor} is no longer supported'
+    else:
+        msg = f'Support for Python version {major}.{minor} has been deprecated. '
+        # Temporary until `win_x86_exe` uses 3.8, which will deprecate Vista and Server 2008
+        if detect_variant() == 'win_x86_exe':
+            platform_name = platform.platform()
+            if any(platform_name.startswith(f'Windows-{name}') for name in ('Vista', '2008Server')):
+                msg = 'Support for Windows Vista/Server 2008 has been deprecated. '
+            else:
+                return None
+        msg += ('See  https://github.com/yt-dlp/yt-dlp/issues/7803  for details.'
+                '\nYou may stop receiving updates on this version at any time')
+
+    major, minor = MIN_RECOMMENDED
+    return f'{msg}! Please update to Python {major}.{minor} or above'
+
+
 def _sha256_file(path):
     h = hashlib.sha256()
     mv = memoryview(bytearray(128 * 1024))

From de015e930747165dbb8fcd360f8775fd973b7d6e Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 24 Sep 2023 02:29:01 +0200
Subject: [PATCH 596/871] [core] Prevent RCE when using `--exec` with `%q`
 (CVE-2023-40581)

The shell escape function is now using `""` instead of `\"`. `utils.Popen` has been patched to properly quote commands.

Prior to this fix using `--exec` together with `%q` when on Windows could cause remote code to execute. See https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-42h4-v29r-42qg for reference.

Authored by: Grub4K
---
 devscripts/changelog_override.json |  5 +++++
 test/test_YoutubeDL.py             |  6 +++---
 test/test_utils.py                 | 16 ++++++++++++++++
 yt_dlp/compat/__init__.py          |  2 +-
 yt_dlp/postprocessor/exec.py       | 12 +++++-------
 yt_dlp/utils/_utils.py             | 18 ++++++++++++++++--
 6 files changed, 46 insertions(+), 13 deletions(-)

diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index 9dfbf510f..fe0c82c66 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -93,5 +93,10 @@
         "action": "add",
         "when": "c1d71d0d9f41db5e4306c86af232f5f6220a130b",
         "short": "[priority] **The minimum *recommended* Python version has been raised to 3.8**\nSince Python 3.7 has reached end-of-life, support for it will be dropped soon. [Read more](https://github.com/yt-dlp/yt-dlp/issues/7803)"
+    },
+    {
+        "action": "add",
+        "when": "61bdf15fc7400601c3da1aa7a43917310a5bf391",
+        "short": "[priority] Security: [[CVE-2023-40581](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-40581)] [Prevent RCE when using `--exec` with `%q` on Windows](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-42h4-v29r-42qg)\n    - The shell escape function is now using `\"\"` instead of `\\\"`.\n    - `utils.Popen` has been patched to properly quote commands."
     }
 ]
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 916ee48b9..0cf130db0 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -784,9 +784,9 @@ def expect_same_infodict(out):
         test('%(title4)#S', 'foo_bar_test')
         test('%(title4).10S', ('foo ＂bar＂ ', 'foo ＂bar＂' + ('#' if compat_os_name == 'nt' else ' ')))
         if compat_os_name == 'nt':
-            test('%(title4)q', ('"foo \\"bar\\" test"', "＂foo ⧹＂bar⧹＂ test＂"))
-            test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', '＂id 1＂ ＂id 2＂ ＂id 3＂'))
-            test('%(formats.0.id)#q', ('"id 1"', '＂id 1＂'))
+            test('%(title4)q', ('"foo ""bar"" test"', None))
+            test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', None))
+            test('%(formats.0.id)#q', ('"id 1"', None))
         else:
             test('%(title4)q', ('\'foo "bar" test\'', '\'foo ＂bar＂ test\''))
             test('%(formats.:.id)#q', "'id 1' 'id 2' 'id 3'")
diff --git a/test/test_utils.py b/test/test_utils.py
index 47d1f71bf..dc2d8ce12 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -14,6 +14,7 @@
 import io
 import itertools
 import json
+import subprocess
 import xml.etree.ElementTree
 
 from yt_dlp.compat import (
@@ -28,6 +29,7 @@
     InAdvancePagedList,
     LazyList,
     OnDemandPagedList,
+    Popen,
     age_restricted,
     args_to_str,
     base_url,
@@ -2388,6 +2390,20 @@ def test_extract_basic_auth(self):
         assert extract_basic_auth('http://user:@foo.bar') == ('http://foo.bar', 'Basic dXNlcjo=')
         assert extract_basic_auth('http://user:pass@foo.bar') == ('http://foo.bar', 'Basic dXNlcjpwYXNz')
 
+    @unittest.skipUnless(compat_os_name == 'nt', 'Only relevant on Windows')
+    def test_Popen_windows_escaping(self):
+        def run_shell(args):
+            stdout, stderr, error = Popen.run(
+                args, text=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            assert not stderr
+            assert not error
+            return stdout
+
+        # Test escaping
+        assert run_shell(['echo', 'test"&']) == '"test""&"\n'
+        # Test if delayed expansion is disabled
+        assert run_shell(['echo', '^!']) == '"^!"\n'
+        assert run_shell('echo "^!"') == '"^!"\n'
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/yt_dlp/compat/__init__.py b/yt_dlp/compat/__init__.py
index 832a9138d..5ad5c70ec 100644
--- a/yt_dlp/compat/__init__.py
+++ b/yt_dlp/compat/__init__.py
@@ -30,7 +30,7 @@ def compat_etree_fromstring(text):
 if compat_os_name == 'nt':
     def compat_shlex_quote(s):
         import re
-        return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
+        return s if re.match(r'^[-_\w./]+$', s) else s.replace('"', '""').join('""')
 else:
     from shlex import quote as compat_shlex_quote  # noqa: F401
 
diff --git a/yt_dlp/postprocessor/exec.py b/yt_dlp/postprocessor/exec.py
index cfc83167c..c2e73fbab 100644
--- a/yt_dlp/postprocessor/exec.py
+++ b/yt_dlp/postprocessor/exec.py
@@ -1,8 +1,6 @@
-import subprocess
-
 from .common import PostProcessor
 from ..compat import compat_shlex_quote
-from ..utils import PostProcessingError, encodeArgument, variadic
+from ..utils import Popen, PostProcessingError, variadic
 
 
 class ExecPP(PostProcessor):
@@ -27,10 +25,10 @@ def parse_cmd(self, cmd, info):
     def run(self, info):
         for tmpl in self.exec_cmd:
             cmd = self.parse_cmd(tmpl, info)
-            self.to_screen('Executing command: %s' % cmd)
-            retCode = subprocess.call(encodeArgument(cmd), shell=True)
-            if retCode != 0:
-                raise PostProcessingError('Command returned error code %d' % retCode)
+            self.to_screen(f'Executing command: {cmd}')
+            _, _, return_code = Popen.run(cmd, shell=True)
+            if return_code != 0:
+                raise PostProcessingError(f'Command returned error code {return_code}')
         return [], info
 
 
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 213ccc636..ba6242380 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -825,7 +825,7 @@ def _fix(key):
         _fix('LD_LIBRARY_PATH')  # Linux
         _fix('DYLD_LIBRARY_PATH')  # macOS
 
-    def __init__(self, *args, env=None, text=False, **kwargs):
+    def __init__(self, args, *remaining, env=None, text=False, shell=False, **kwargs):
         if env is None:
             env = os.environ.copy()
         self._fix_pyinstaller_ld_path(env)
@@ -835,7 +835,21 @@ def __init__(self, *args, env=None, text=False, **kwargs):
             kwargs['universal_newlines'] = True  # For 3.6 compatibility
             kwargs.setdefault('encoding', 'utf-8')
             kwargs.setdefault('errors', 'replace')
-        super().__init__(*args, env=env, **kwargs, startupinfo=self._startupinfo)
+
+        if shell and compat_os_name == 'nt' and kwargs.get('executable') is None:
+            if not isinstance(args, str):
+                args = ' '.join(compat_shlex_quote(a) for a in args)
+            shell = False
+            args = f'{self.__comspec()} /Q /S /D /V:OFF /C "{args}"'
+
+        super().__init__(args, *remaining, env=env, shell=shell, **kwargs, startupinfo=self._startupinfo)
+
+    def __comspec(self):
+        comspec = os.environ.get('ComSpec') or os.path.join(
+            os.environ.get('SystemRoot', ''), 'System32', 'cmd.exe')
+        if os.path.isabs(comspec):
+            return comspec
+        raise FileNotFoundError('shell not found: neither %ComSpec% nor %SystemRoot% is set')
 
     def communicate_or_kill(self, *args, **kwargs):
         try:

From 088add9567d39b758737e4299a0e619fd89d2e8f Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 24 Sep 2023 02:35:23 +0200
Subject: [PATCH 597/871] [cleanup] Misc

Authored by: Grub4K
---
 test/test_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index dc2d8ce12..fd612ff86 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2405,5 +2405,6 @@ def run_shell(args):
         assert run_shell(['echo', '^!']) == '"^!"\n'
         assert run_shell('echo "^!"') == '"^!"\n'
 
+
 if __name__ == '__main__':
     unittest.main()

From c54ddfba0f7d68034339426223d75373c5fc86df Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@example.com>
Date: Sun, 24 Sep 2023 00:38:42 +0000
Subject: [PATCH 598/871] Release 2023.09.24

Created by: Grub4K

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |   8 +-
 .../ISSUE_TEMPLATE/2_site_support_request.yml |   8 +-
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |   8 +-
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |   8 +-
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |   8 +-
 .github/ISSUE_TEMPLATE/6_question.yml         |   8 +-
 CONTRIBUTORS                                  |  36 ++++
 Changelog.md                                  | 196 ++++++++++++++++++
 supportedsites.md                             |  49 ++++-
 yt_dlp/version.py                             |   4 +-
 10 files changed, 298 insertions(+), 35 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index dd1b33dde..f0fc71d57 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -64,7 +64,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -72,8 +72,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.07.06, Current version: 2023.07.06
-        yt-dlp is up to date (2023.07.06)
+        Latest version: 2023.09.24, Current version: 2023.09.24
+        yt-dlp is up to date (2023.09.24)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index 4f4378924..ac9a72a1c 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -76,7 +76,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -84,8 +84,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.07.06, Current version: 2023.07.06
-        yt-dlp is up to date (2023.07.06)
+        Latest version: 2023.09.24, Current version: 2023.09.24
+        yt-dlp is up to date (2023.09.24)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index 05b4dd23b..577e4d491 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -72,7 +72,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -80,8 +80,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.07.06, Current version: 2023.07.06
-        yt-dlp is up to date (2023.07.06)
+        Latest version: 2023.09.24, Current version: 2023.09.24
+        yt-dlp is up to date (2023.09.24)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index 880f1014c..9529c1bd6 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -57,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -65,8 +65,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.07.06, Current version: 2023.07.06
-        yt-dlp is up to date (2023.07.06)
+        Latest version: 2023.09.24, Current version: 2023.09.24
+        yt-dlp is up to date (2023.09.24)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index acb11795f..b17a6e046 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -53,7 +53,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -61,7 +61,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.07.06, Current version: 2023.07.06
-        yt-dlp is up to date (2023.07.06)
+        Latest version: 2023.09.24, Current version: 2023.09.24
+        yt-dlp is up to date (2023.09.24)
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index a2563e975..5345e8917 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.07.06** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -59,7 +59,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.07.06 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -67,7 +67,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.07.06, Current version: 2023.07.06
-        yt-dlp is up to date (2023.07.06)
+        Latest version: 2023.09.24, Current version: 2023.09.24
+        yt-dlp is up to date (2023.09.24)
         <more lines>
       render: shell
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 6b9b9f470..72b9584ec 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -467,3 +467,39 @@ rdamas
 RfadnjdExt
 urectanc
 nao20010128nao/Lesmiscore
+04-pasha-04
+aaruni96
+aky-01
+AmirAflak
+ApoorvShah111
+at-wat
+davinkevin
+demon071
+denhotte
+FinnRG
+fireattack
+Frankgoji
+GD-Slime
+hatsomatt
+ifan-t
+kshitiz305
+kylegustavo
+mabdelfattah
+nathantouze
+niemands
+Rajeshwaran2001
+RedDeffender
+Rohxn16
+sb0stn
+SevenLives
+simon300000
+snixon
+soundchaser128
+szabyg
+trainman261
+trislee
+wader
+Yalab7
+zhallgato
+zhong-yiyu
+Zprokkel
diff --git a/Changelog.md b/Changelog.md
index 32cdaca2a..04511927f 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,202 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.09.24
+
+#### Important changes
+- **The minimum *recommended* Python version has been raised to 3.8**
+Since Python 3.7 has reached end-of-life, support for it will be dropped soon. [Read more](https://github.com/yt-dlp/yt-dlp/issues/7803)
+- Security: [[CVE-2023-40581](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-40581)] [Prevent RCE when using `--exec` with `%q` on Windows](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-42h4-v29r-42qg)
+    - The shell escape function is now using `""` instead of `\"`.
+    - `utils.Popen` has been patched to properly quote commands.
+
+#### Core changes
+- [Fix HTTP headers and cookie handling](https://github.com/yt-dlp/yt-dlp/commit/6c5211cebeacfc53ad5d5ddf4a659be76039656f) by [bashonly](https://github.com/bashonly), [pukkandan](https://github.com/pukkandan)
+- [Fix `--check-formats`](https://github.com/yt-dlp/yt-dlp/commit/8cb7fc44db010e965d808ee679ef0725cb6e147c) by [pukkandan](https://github.com/pukkandan)
+- [Fix support for upcoming Python 3.12](https://github.com/yt-dlp/yt-dlp/commit/836e06d246512f286f30c1371b2c54b72c9ecd93) ([#8130](https://github.com/yt-dlp/yt-dlp/issues/8130)) by [Grub4K](https://github.com/Grub4K)
+- [Merged with youtube-dl 66ab08](https://github.com/yt-dlp/yt-dlp/commit/9d6254069c75877bc88bc3584f4326fb1853a543) by [coletdjnz](https://github.com/coletdjnz)
+- [Prevent RCE when using `--exec` with `%q` (CVE-2023-40581)](https://github.com/yt-dlp/yt-dlp/commit/de015e930747165dbb8fcd360f8775fd973b7d6e) by [Grub4K](https://github.com/Grub4K)
+- [Raise minimum recommended Python version to 3.8](https://github.com/yt-dlp/yt-dlp/commit/61bdf15fc7400601c3da1aa7a43917310a5bf391) ([#8183](https://github.com/yt-dlp/yt-dlp/issues/8183)) by [Grub4K](https://github.com/Grub4K)
+- [`FFmpegFixupM3u8PP` may need to run with ffmpeg](https://github.com/yt-dlp/yt-dlp/commit/f73c11803579889dc8e1c99e25dba9a22fef39d8) by [pukkandan](https://github.com/pukkandan)
+- **compat**
+    - [Add `types.NoneType`](https://github.com/yt-dlp/yt-dlp/commit/e0c4db04dc82a699bdabd9821ddc239ebe17d30a) by [pukkandan](https://github.com/pukkandan) (With fixes in [25b6e8f](https://github.com/yt-dlp/yt-dlp/commit/25b6e8f94679b4458550702b46e61249b875a4fd))
+    - [Deprecate old functions](https://github.com/yt-dlp/yt-dlp/commit/3d2623a898196640f7cc0fc8b70118ff19e6925d) ([#2861](https://github.com/yt-dlp/yt-dlp/issues/2861)) by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+    - [Ensure submodules are imported correctly](https://github.com/yt-dlp/yt-dlp/commit/a250b247334ce9f641e709cbb64974da6034a2b3) by [pukkandan](https://github.com/pukkandan)
+- **cookies**: [Containers JSON should be opened as utf-8](https://github.com/yt-dlp/yt-dlp/commit/dab87ca23650fd87184ff5286b53e6985b59f71d) ([#7800](https://github.com/yt-dlp/yt-dlp/issues/7800)) by [bashonly](https://github.com/bashonly)
+- **dependencies**: [Handle deprecation of `sqlite3.version`](https://github.com/yt-dlp/yt-dlp/commit/35f9a306e6934793cff100200cd03f288ec33f11) ([#8167](https://github.com/yt-dlp/yt-dlp/issues/8167)) by [bashonly](https://github.com/bashonly)
+- **outtmpl**: [Fix replacement for `playlist_index`](https://github.com/yt-dlp/yt-dlp/commit/a264433c9fba147ecae2420091614186cfeeb895) by [pukkandan](https://github.com/pukkandan)
+- **utils**
+    - [Add temporary shim for logging](https://github.com/yt-dlp/yt-dlp/commit/1b392f905d20ef1f1b300b180f867d43c9ce49b8) by [pukkandan](https://github.com/pukkandan)
+    - [Improve `parse_duration`](https://github.com/yt-dlp/yt-dlp/commit/af86873218c24c3859ccf575a87f2b00a73b49d0) by [bashonly](https://github.com/bashonly)
+    - HTTPHeaderDict: [Handle byte values](https://github.com/yt-dlp/yt-dlp/commit/3f7965105d8d2048359e67c1e8b8ebd51588143b) by [pukkandan](https://github.com/pukkandan)
+    - `clean_podcast_url`: [Handle more trackers](https://github.com/yt-dlp/yt-dlp/commit/2af4eeb77246b8183aae75a0a8d19f18c08115b2) ([#7556](https://github.com/yt-dlp/yt-dlp/issues/7556)) by [bashonly](https://github.com/bashonly), [mabdelfattah](https://github.com/mabdelfattah)
+    - `js_to_json`: [Handle `Array` objects](https://github.com/yt-dlp/yt-dlp/commit/52414d64ca7b92d3f83964cdd68247989b0c4625) by [Grub4K](https://github.com/Grub4K), [std-move](https://github.com/std-move)
+
+#### Extractor changes
+- [Extract subtitles from SMIL manifests](https://github.com/yt-dlp/yt-dlp/commit/550e65410a7a1b105923494ac44460a4dc1a15d9) ([#7667](https://github.com/yt-dlp/yt-dlp/issues/7667)) by [bashonly](https://github.com/bashonly), [pukkandan](https://github.com/pukkandan)
+- [Fix `--load-pages`](https://github.com/yt-dlp/yt-dlp/commit/81b4712bca608b9015aa68a4d96661d56e9cb894) by [pukkandan](https://github.com/pukkandan)
+- [Make `_search_nuxt_data` more lenient](https://github.com/yt-dlp/yt-dlp/commit/904a19ee93195ce0bd4b08bd22b186120afb5b17) by [std-move](https://github.com/std-move)
+- **abematv**
+    - [Fix proxy handling](https://github.com/yt-dlp/yt-dlp/commit/497bbbbd7328cb705f70eced94dbd90993819a46) ([#8046](https://github.com/yt-dlp/yt-dlp/issues/8046)) by [SevenLives](https://github.com/SevenLives)
+    - [Temporary fix for protocol handler](https://github.com/yt-dlp/yt-dlp/commit/9f66247289b9f8ecf931833b3f5f127274dd2161) by [pukkandan](https://github.com/pukkandan)
+- **amazonminitv**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/538d37671a17e0782d17f08df17800e2e3bd57c8) by [bashonly](https://github.com/bashonly), [GautamMKGarg](https://github.com/GautamMKGarg)
+- **antenna**: [Support antenna.gr](https://github.com/yt-dlp/yt-dlp/commit/665876034c8d3c031443f6b4958bed02ccdf4164) ([#7584](https://github.com/yt-dlp/yt-dlp/issues/7584)) by [stdedos](https://github.com/stdedos)
+- **artetv**: [Fix HLS formats extraction](https://github.com/yt-dlp/yt-dlp/commit/c2da0b5ea215298135f76e3dc14b972a3c4afacb) by [bashonly](https://github.com/bashonly)
+- **axs**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/aee6b9b88c0bcccf27fd23b7e00fc0b7b168928f) ([#8094](https://github.com/yt-dlp/yt-dlp/issues/8094)) by [barsnick](https://github.com/barsnick)
+- **banbye**: [Support video ids containing a hyphen](https://github.com/yt-dlp/yt-dlp/commit/578a82e497502b951036ce9da6fe0dac6937ac27) ([#8059](https://github.com/yt-dlp/yt-dlp/issues/8059)) by [kshitiz305](https://github.com/kshitiz305)
+- **bbc**: [Extract tracklist as chapters](https://github.com/yt-dlp/yt-dlp/commit/eda0e415d26eb084e570cf5372d38ee1f616b70f) ([#7788](https://github.com/yt-dlp/yt-dlp/issues/7788)) by [garret1317](https://github.com/garret1317)
+- **bild.de**: [Extract HLS formats](https://github.com/yt-dlp/yt-dlp/commit/b4c1c408c63724339eb12b16c91b253a7ee62cfa) ([#8032](https://github.com/yt-dlp/yt-dlp/issues/8032)) by [barsnick](https://github.com/barsnick)
+- **bilibili**
+    - [Add support for series, favorites and watch later](https://github.com/yt-dlp/yt-dlp/commit/9e68747f9607f05e92bb7d9b6e79d678b50070e1) ([#7518](https://github.com/yt-dlp/yt-dlp/issues/7518)) by [c-basalt](https://github.com/c-basalt)
+    - [Extract Dolby audio formats](https://github.com/yt-dlp/yt-dlp/commit/b84fda7388dd20d38921e23b469147f3957c1812) ([#8142](https://github.com/yt-dlp/yt-dlp/issues/8142)) by [ClosedPort22](https://github.com/ClosedPort22)
+    - [Extract `format_id`](https://github.com/yt-dlp/yt-dlp/commit/5336bf57a7061e0955a37f0542fc8ebf50d55b17) ([#7555](https://github.com/yt-dlp/yt-dlp/issues/7555)) by [c-basalt](https://github.com/c-basalt)
+- **bilibilibangumi**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/bdd0b75e3f41ff35440eda6d395008beef19ef2f) ([#7337](https://github.com/yt-dlp/yt-dlp/issues/7337)) by [GD-Slime](https://github.com/GD-Slime)
+- **bpb**: [Overhaul extractor](https://github.com/yt-dlp/yt-dlp/commit/f659e6439444ac64305b5c80688cd82f59d2279c) ([#8119](https://github.com/yt-dlp/yt-dlp/issues/8119)) by [Grub4K](https://github.com/Grub4K)
+- **brilliantpala**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/92feb5654c5a4c81ba872904a618700fcbb3e546) ([#6680](https://github.com/yt-dlp/yt-dlp/issues/6680)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **canal1, caracoltvplay**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/b3febedbeb662dfdf9b5c1d5799039ad4fc969de) ([#7151](https://github.com/yt-dlp/yt-dlp/issues/7151)) by [elyse0](https://github.com/elyse0)
+- **cbc**: [Ignore any 426 from API](https://github.com/yt-dlp/yt-dlp/commit/9bf14be775289bd88cc1f5c89fd761ae51879484) ([#7689](https://github.com/yt-dlp/yt-dlp/issues/7689)) by [makew0rld](https://github.com/makew0rld)
+- **cbcplayer**: [Extract HLS formats and subtitles](https://github.com/yt-dlp/yt-dlp/commit/339c339fec095ff4141b20e6aa83629117fb26df) ([#7484](https://github.com/yt-dlp/yt-dlp/issues/7484)) by [trainman261](https://github.com/trainman261)
+- **cbcplayerplaylist**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/ed711897814f3ee0b1822e4205e74133467e8f1c) ([#7870](https://github.com/yt-dlp/yt-dlp/issues/7870)) by [trainman261](https://github.com/trainman261)
+- **cineverse**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/15591940ff102d1ae337d603a46d8f238c83a61f) ([#8146](https://github.com/yt-dlp/yt-dlp/issues/8146)) by [garret1317](https://github.com/garret1317)
+- **crunchyroll**: [Remove initial state extraction](https://github.com/yt-dlp/yt-dlp/commit/9b16762f48914de9ac914601769c76668e433325) ([#7632](https://github.com/yt-dlp/yt-dlp/issues/7632)) by [Grub4K](https://github.com/Grub4K)
+- **douyutv**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/21f40e75dfc0055ea9cdbd7fe2c46c6f9b561afd) ([#7652](https://github.com/yt-dlp/yt-dlp/issues/7652)) by [c-basalt](https://github.com/c-basalt)
+- **dropbox**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/b9f2bc2dbed2323734a0d18e65e1e2e23dc833d8) ([#7926](https://github.com/yt-dlp/yt-dlp/issues/7926)) by [bashonly](https://github.com/bashonly), [denhotte](https://github.com/denhotte), [nathantouze](https://github.com/nathantouze) (With fixes in [099fb1b](https://github.com/yt-dlp/yt-dlp/commit/099fb1b35cf835303306549f5113d1802d79c9c7) by [bashonly](https://github.com/bashonly))
+- **eplus**: inbound: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/295fbb3ae3a7d0dd50e286be5c487cf145ed5778) ([#5782](https://github.com/yt-dlp/yt-dlp/issues/5782)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **expressen**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/a5e264d74b4bd60c6e7ec4e38f1a23af4e420531) ([#8153](https://github.com/yt-dlp/yt-dlp/issues/8153)) by [kylegustavo](https://github.com/kylegustavo)
+- **facebook**
+    - [Add dash manifest URL](https://github.com/yt-dlp/yt-dlp/commit/a854fbec56d5004f5147116a41d1dd050632a579) ([#7743](https://github.com/yt-dlp/yt-dlp/issues/7743)) by [ringus1](https://github.com/ringus1)
+    - [Fix webpage extraction](https://github.com/yt-dlp/yt-dlp/commit/d3d81cc98f554d0adb87d24bfd6fabaaa803944d) ([#7890](https://github.com/yt-dlp/yt-dlp/issues/7890)) by [ringus1](https://github.com/ringus1)
+    - [Improve format sorting](https://github.com/yt-dlp/yt-dlp/commit/308936619c8a4f3a52d73c829c2006ff6c55fea2) ([#8074](https://github.com/yt-dlp/yt-dlp/issues/8074)) by [fireattack](https://github.com/fireattack)
+    - reel: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/bb5d84c9d2f1e978c3eddfb5ccbe138036682a36) ([#7564](https://github.com/yt-dlp/yt-dlp/issues/7564)) by [bashonly](https://github.com/bashonly), [demon071](https://github.com/demon071)
+- **fox**: [Support foxsports.com](https://github.com/yt-dlp/yt-dlp/commit/30b29f37159e9226e2f2d5434c9a4096ac4efa2e) ([#7724](https://github.com/yt-dlp/yt-dlp/issues/7724)) by [ischmidt20](https://github.com/ischmidt20)
+- **funker530**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/0ce1f48bf1cb78d40d734ce73ee1c90eccf92274) ([#8040](https://github.com/yt-dlp/yt-dlp/issues/8040)) by [04-pasha-04](https://github.com/04-pasha-04)
+- **generic**
+    - [Fix KVS thumbnail extraction](https://github.com/yt-dlp/yt-dlp/commit/53675852195d8dd859555d4789944a6887171ff8) by [bashonly](https://github.com/bashonly)
+    - [Fix generic title for embeds](https://github.com/yt-dlp/yt-dlp/commit/994f7ef8e6003f4b7b258528755d0b6adcc31714) by [pukkandan](https://github.com/pukkandan)
+- **gofile**: [Update token](https://github.com/yt-dlp/yt-dlp/commit/99c99c7185f5d8e9b3699a6fc7f86ec663d7b97e) by [bashonly](https://github.com/bashonly)
+- **hotstar**
+    - [Extract `release_year`](https://github.com/yt-dlp/yt-dlp/commit/7237c8dca0590aa7438ade93f927df88c9381ec7) ([#7869](https://github.com/yt-dlp/yt-dlp/issues/7869)) by [Rajeshwaran2001](https://github.com/Rajeshwaran2001)
+    - [Make metadata extraction non-fatal](https://github.com/yt-dlp/yt-dlp/commit/30ea88591b728cca0896018dbf67c2298070c669) by [bashonly](https://github.com/bashonly)
+    - [Support `/clips/` URLs](https://github.com/yt-dlp/yt-dlp/commit/86eeb044c2342d68c6ef177577f87852e6badd85) ([#7710](https://github.com/yt-dlp/yt-dlp/issues/7710)) by [bashonly](https://github.com/bashonly)
+- **hungama**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/4b3a6ef1b3e235ba9a45142830b6edb357c71696) ([#7757](https://github.com/yt-dlp/yt-dlp/issues/7757)) by [bashonly](https://github.com/bashonly), [Yalab7](https://github.com/Yalab7)
+- **indavideoembed**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/63e0c5748c0eb461a2ccca4181616eb930b4b750) ([#8129](https://github.com/yt-dlp/yt-dlp/issues/8129)) by [aky-01](https://github.com/aky-01)
+- **iprima**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/568f08051841aedea968258889539741e26009e9) ([#7216](https://github.com/yt-dlp/yt-dlp/issues/7216)) by [std-move](https://github.com/std-move)
+- **lbry**: [Fix original format extraction](https://github.com/yt-dlp/yt-dlp/commit/127a22460658ac39cbe5c4b3fb88d578363e0dfa) ([#7711](https://github.com/yt-dlp/yt-dlp/issues/7711)) by [bashonly](https://github.com/bashonly)
+- **lecturio**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/efa2339502a37cf13ae7f143bd8b2c28f452d1cd) ([#7649](https://github.com/yt-dlp/yt-dlp/issues/7649)) by [simon300000](https://github.com/simon300000)
+- **magellantv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/f4ea501551526ebcb54d19b84cf0ebe798583a85) ([#7616](https://github.com/yt-dlp/yt-dlp/issues/7616)) by [bashonly](https://github.com/bashonly)
+- **massengeschmack.tv**: [Fix title extraction](https://github.com/yt-dlp/yt-dlp/commit/81f46ac573dc443ad48560f308582a26784d3015) ([#7813](https://github.com/yt-dlp/yt-dlp/issues/7813)) by [sb0stn](https://github.com/sb0stn)
+- **media.ccc.de**: lists: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/cf11b40ac40e3d23a6352753296f3a732886efb9) ([#8144](https://github.com/yt-dlp/yt-dlp/issues/8144)) by [Rohxn16](https://github.com/Rohxn16)
+- **mediaite**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/630a55df8de7747e79aa680959d785dfff2c4b76) ([#7923](https://github.com/yt-dlp/yt-dlp/issues/7923)) by [Grabien](https://github.com/Grabien)
+- **mediaklikk**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/6e07e4bc7e59f5bdb60e93c011e57b18b009f2b5) ([#8086](https://github.com/yt-dlp/yt-dlp/issues/8086)) by [bashonly](https://github.com/bashonly), [zhallgato](https://github.com/zhallgato)
+- **mediastream**: [Make embed extraction non-fatal](https://github.com/yt-dlp/yt-dlp/commit/635ae31f68a3ac7f6393d59657ed711e34ee3552) by [bashonly](https://github.com/bashonly)
+- **mixcloud**: [Update API URL](https://github.com/yt-dlp/yt-dlp/commit/7b71643cc986de9a3768dac4ac9b64f4d05e7f5e) ([#8114](https://github.com/yt-dlp/yt-dlp/issues/8114)) by [garret1317](https://github.com/garret1317)
+- **monstercat**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/eaee21bf71889d495076037cbe590c8c0b21ef3a) ([#8133](https://github.com/yt-dlp/yt-dlp/issues/8133)) by [garret1317](https://github.com/garret1317)
+- **motortrendondemand**: [Update `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/c03a58ec9933e4a42c2d8fa80b8a0ddb2cde64e6) ([#7683](https://github.com/yt-dlp/yt-dlp/issues/7683)) by [AmirAflak](https://github.com/AmirAflak)
+- **museai**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/65cfa2b057d7946fbe322155a778fe206556d0c6) ([#7614](https://github.com/yt-dlp/yt-dlp/issues/7614)) by [bashonly](https://github.com/bashonly)
+- **mzaalo**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/d7aee8e310b2c4f21d50aac0b420e1b3abde21a4) by [bashonly](https://github.com/bashonly)
+- **n1info**: article: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/8ac5b6d96ae5c60cd5ae2495949e0068a6754c45) ([#7373](https://github.com/yt-dlp/yt-dlp/issues/7373)) by [u-spec-png](https://github.com/u-spec-png)
+- **nfl.com**: plus, replay: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/1eaca74bc2ca0f5b1ec532f24c61de44f2e8cb2d) ([#7838](https://github.com/yt-dlp/yt-dlp/issues/7838)) by [bashonly](https://github.com/bashonly)
+- **niconicochannelplus**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/698beb9a497f51693e64d167e572ff9efa4bc25f) ([#5686](https://github.com/yt-dlp/yt-dlp/issues/5686)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **nitter**: [Fix title extraction fallback](https://github.com/yt-dlp/yt-dlp/commit/a83da3717d30697102e76f63a6f29d77f9373c2a) ([#8102](https://github.com/yt-dlp/yt-dlp/issues/8102)) by [ApoorvShah111](https://github.com/ApoorvShah111)
+- **noodlemagazine**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/bae4834245a708fff97219849ec880c319c88bc6) ([#7830](https://github.com/yt-dlp/yt-dlp/issues/7830)) by [RedDeffender](https://github.com/RedDeffender) (With fixes in [69dbfe0](https://github.com/yt-dlp/yt-dlp/commit/69dbfe01c47cd078682a87f179f5846e2679e927) by [bashonly](https://github.com/bashonly))
+- **novaembed**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/2269065ad60cb0ab62408ae6a7b20283e5252232) ([#7910](https://github.com/yt-dlp/yt-dlp/issues/7910)) by [std-move](https://github.com/std-move)
+- **patreoncampaign**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/11de6fec9c9b8d34d1f90c8e6218ec58a3471b58) ([#7664](https://github.com/yt-dlp/yt-dlp/issues/7664)) by [bashonly](https://github.com/bashonly)
+- **pbs**: [Add extractor `PBSKidsIE`](https://github.com/yt-dlp/yt-dlp/commit/6d6081dda1290a85bdab6717f239289e3aa74c8e) ([#7602](https://github.com/yt-dlp/yt-dlp/issues/7602)) by [snixon](https://github.com/snixon)
+- **piapro**: [Support `/content` URL](https://github.com/yt-dlp/yt-dlp/commit/1bcb9fe8715b1f288efc322be3de409ee0597080) ([#7592](https://github.com/yt-dlp/yt-dlp/issues/7592)) by [FinnRG](https://github.com/FinnRG)
+- **piaulizaportal**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/6636021206dad17c7745ae6bce6cb73d6f2ef319) ([#7903](https://github.com/yt-dlp/yt-dlp/issues/7903)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **picartovod**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/db9743894071760f994f640a4c24358f749a78c0) ([#7727](https://github.com/yt-dlp/yt-dlp/issues/7727)) by [Frankgoji](https://github.com/Frankgoji)
+- **pornbox**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/40999467f72db074a3f13057da9bf82a857530fe) ([#7386](https://github.com/yt-dlp/yt-dlp/issues/7386)) by [niemands](https://github.com/niemands)
+- **pornhub**: [Update access cookies for UK](https://github.com/yt-dlp/yt-dlp/commit/1d3d579c2142f69831b6ae140e1d8e824e07fa0e) ([#7591](https://github.com/yt-dlp/yt-dlp/issues/7591)) by [zhong-yiyu](https://github.com/zhong-yiyu)
+- **pr0gramm**: [Rewrite extractor](https://github.com/yt-dlp/yt-dlp/commit/b532556d0a85e7d76f8f0880861232fb706ddbc5) ([#8151](https://github.com/yt-dlp/yt-dlp/issues/8151)) by [Grub4K](https://github.com/Grub4K)
+- **radiofrance**: [Add support for livestreams, podcasts, playlists](https://github.com/yt-dlp/yt-dlp/commit/ba8e9eb2c8bbb699f314169fab8e544437ad731e) ([#7006](https://github.com/yt-dlp/yt-dlp/issues/7006)) by [elyse0](https://github.com/elyse0)
+- **rbgtum**: [Fix extraction and support new URL format](https://github.com/yt-dlp/yt-dlp/commit/5fccabac27ca3c1165ade1b0df6fbadc24258dc2) ([#7690](https://github.com/yt-dlp/yt-dlp/issues/7690)) by [simon300000](https://github.com/simon300000)
+- **reddit**
+    - [Extract subtitles](https://github.com/yt-dlp/yt-dlp/commit/20c3c9b433dd47faf0dbde6b46e4e34eb76109a5) by [bashonly](https://github.com/bashonly)
+    - [Fix thumbnail extraction](https://github.com/yt-dlp/yt-dlp/commit/9a04113dfbb69b904e4e2bea736da293505786b8) by [bashonly](https://github.com/bashonly)
+- **rtvslo**: [Fix format extraction](https://github.com/yt-dlp/yt-dlp/commit/94389b225d9bcf29aa7ba8afaf1bbd7c62204eae) ([#8131](https://github.com/yt-dlp/yt-dlp/issues/8131)) by [bashonly](https://github.com/bashonly)
+- **rule34video**: [Extract tags](https://github.com/yt-dlp/yt-dlp/commit/58493923e9b6f774947a2131e5258e9f3cf816be) ([#7117](https://github.com/yt-dlp/yt-dlp/issues/7117)) by [soundchaser128](https://github.com/soundchaser128)
+- **rumble**: [Fix embed extraction](https://github.com/yt-dlp/yt-dlp/commit/23d829a3420450bcfb0788e6fb2cf4f6acdbe596) ([#8035](https://github.com/yt-dlp/yt-dlp/issues/8035)) by [trislee](https://github.com/trislee)
+- **s4c**
+    - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/b9de629d78ce31699f2de886071dc257830f9676) ([#7730](https://github.com/yt-dlp/yt-dlp/issues/7730)) by [ifan-t](https://github.com/ifan-t)
+    - [Add series support and extract subs/thumbs](https://github.com/yt-dlp/yt-dlp/commit/fe371dcf0ba5ce8d42480eade54eeeac99ab3cb0) ([#7776](https://github.com/yt-dlp/yt-dlp/issues/7776)) by [ifan-t](https://github.com/ifan-t)
+- **sohu**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/5be7e978867b5f66ad6786c674d79d40e950ae16) ([#7628](https://github.com/yt-dlp/yt-dlp/issues/7628)) by [bashonly](https://github.com/bashonly), [c-basalt](https://github.com/c-basalt)
+- **stageplus**: [Fix m3u8 extraction](https://github.com/yt-dlp/yt-dlp/commit/56b3dc03354b75be995759d8441d2754c0442b9a) ([#7929](https://github.com/yt-dlp/yt-dlp/issues/7929)) by [bashonly](https://github.com/bashonly)
+- **streamanity**: [Remove](https://github.com/yt-dlp/yt-dlp/commit/2cfe221fbbe46faa3f46552c08d947a51f424903) ([#7571](https://github.com/yt-dlp/yt-dlp/issues/7571)) by [alerikaisattera](https://github.com/alerikaisattera)
+- **svtplay**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/2301b5c1b77a65abbb46b72f91e1e4666fd5d985) ([#7789](https://github.com/yt-dlp/yt-dlp/issues/7789)) by [dirkf](https://github.com/dirkf), [wader](https://github.com/wader)
+- **tbsjp**: [Add episode, program, playlist extractors](https://github.com/yt-dlp/yt-dlp/commit/876b70c8edf4c0147f180bd981fbc4d625cbfb9c) ([#7765](https://github.com/yt-dlp/yt-dlp/issues/7765)) by [garret1317](https://github.com/garret1317)
+- **tiktok**
+    - [Fix audio-only format extraction](https://github.com/yt-dlp/yt-dlp/commit/b09bd0c19648f60c59fb980cd454cb0069959fb9) ([#7712](https://github.com/yt-dlp/yt-dlp/issues/7712)) by [bashonly](https://github.com/bashonly)
+    - [Fix webpage extraction](https://github.com/yt-dlp/yt-dlp/commit/069cbece9dba6384f1cc5fcfc7ce562a31af42fc) by [bashonly](https://github.com/bashonly)
+- **triller**: [Fix unlisted video extraction](https://github.com/yt-dlp/yt-dlp/commit/39837ae3199aa934299badbd0d63243ed639e6c8) ([#7670](https://github.com/yt-dlp/yt-dlp/issues/7670)) by [bashonly](https://github.com/bashonly)
+- **tv5mondeplus**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/7d3d658f4c558ee7d72b1c01b46f2126948681cd) ([#7952](https://github.com/yt-dlp/yt-dlp/issues/7952)) by [dirkf](https://github.com/dirkf), [korli](https://github.com/korli)
+- **twitcasting**
+    - [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/cebbd33b1c678149fc8f0e254db6fc0da317ea80) ([#8120](https://github.com/yt-dlp/yt-dlp/issues/8120)) by [c-basalt](https://github.com/c-basalt)
+    - [Support `--wait-for-video`](https://github.com/yt-dlp/yt-dlp/commit/c1d71d0d9f41db5e4306c86af232f5f6220a130b) ([#7975](https://github.com/yt-dlp/yt-dlp/issues/7975)) by [at-wat](https://github.com/at-wat)
+- **twitter**
+    - [Add fallback, improve error handling](https://github.com/yt-dlp/yt-dlp/commit/6014355c6142f68e20c8374e3787e5b5820f19e2) ([#7621](https://github.com/yt-dlp/yt-dlp/issues/7621)) by [bashonly](https://github.com/bashonly)
+    - [Fix GraphQL and legacy API](https://github.com/yt-dlp/yt-dlp/commit/92315c03774cfabb3a921884326beb4b981f786b) ([#7516](https://github.com/yt-dlp/yt-dlp/issues/7516)) by [bashonly](https://github.com/bashonly)
+    - [Fix retweet extraction and syndication API](https://github.com/yt-dlp/yt-dlp/commit/a006ce2b27357c15792eb5c18f06765e640b801c) ([#8016](https://github.com/yt-dlp/yt-dlp/issues/8016)) by [bashonly](https://github.com/bashonly)
+    - [Revert 92315c03774cfabb3a921884326beb4b981f786b](https://github.com/yt-dlp/yt-dlp/commit/b03fa7834579a01cc5fba48c0e73488a16683d48) by [pukkandan](https://github.com/pukkandan)
+    - spaces
+        - [Fix format protocol](https://github.com/yt-dlp/yt-dlp/commit/613dbce177d34ffc31053e8e01acf4bb107bcd1e) ([#7550](https://github.com/yt-dlp/yt-dlp/issues/7550)) by [bashonly](https://github.com/bashonly)
+        - [Pass referer header to downloader](https://github.com/yt-dlp/yt-dlp/commit/c6ef553792ed48462f9fd0e78143bef6b1a71c2e) by [bashonly](https://github.com/bashonly)
+- **unsupported**: [List more sites with DRM](https://github.com/yt-dlp/yt-dlp/commit/e7057383380d7d53815f8feaf90ca3dcbde88983) by [pukkandan](https://github.com/pukkandan)
+- **videa**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/98eac0e6ba0e510ae7dfdfd249d42ee71fb272b1) ([#8003](https://github.com/yt-dlp/yt-dlp/issues/8003)) by [aky-01](https://github.com/aky-01), [hatsomatt](https://github.com/hatsomatt)
+- **vrt**: [Update token signing key](https://github.com/yt-dlp/yt-dlp/commit/325191d0c9bf3fe257b8a7c2eb95080f44f6ddfc) ([#7519](https://github.com/yt-dlp/yt-dlp/issues/7519)) by [Zprokkel](https://github.com/Zprokkel)
+- **wat.tv**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/7cccab79e7d00ed965b48b8cefce1da8a0513409) ([#7898](https://github.com/yt-dlp/yt-dlp/issues/7898)) by [davinkevin](https://github.com/davinkevin)
+- **wdr**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/5d0395498d7065aa5e55bac85fa9354b4b0d48eb) ([#7979](https://github.com/yt-dlp/yt-dlp/issues/7979)) by [szabyg](https://github.com/szabyg)
+- **web.archive**: vlive: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/9652bca1bd02f6bc1b8cb1e186f2ccbf32225561) ([#8132](https://github.com/yt-dlp/yt-dlp/issues/8132)) by [bashonly](https://github.com/bashonly)
+- **weibo**: [Fix extractor and support user extraction](https://github.com/yt-dlp/yt-dlp/commit/69b03f84f8378b0b5a2fbae56f9b7d860b2f529e) ([#7657](https://github.com/yt-dlp/yt-dlp/issues/7657)) by [c-basalt](https://github.com/c-basalt)
+- **weverse**: [Support extraction without auth](https://github.com/yt-dlp/yt-dlp/commit/c2d8ee0000302aba63476b7d5bd8793e57b6c8c6) ([#7924](https://github.com/yt-dlp/yt-dlp/issues/7924)) by [seproDev](https://github.com/seproDev)
+- **wimbledon**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/a15fcd299e767a510debd8dc1646fe863b96ce0e) ([#7551](https://github.com/yt-dlp/yt-dlp/issues/7551)) by [nnoboa](https://github.com/nnoboa)
+- **wrestleuniverseppv**: [Fix HLS AES key extraction](https://github.com/yt-dlp/yt-dlp/commit/dae349da97cafe7357106a8f3187fd48a2ad1210) by [bashonly](https://github.com/bashonly)
+- **youtube**
+    - [Add `player_params` extractor arg](https://github.com/yt-dlp/yt-dlp/commit/ba06d77a316650ff057347d224b5afa8b203ad65) ([#7719](https://github.com/yt-dlp/yt-dlp/issues/7719)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Fix `player_params` arg being converted to lowercase](https://github.com/yt-dlp/yt-dlp/commit/546b2c28a106cf8101d481b215b676d1b091d276) by [coletdjnz](https://github.com/coletdjnz)
+    - [Fix consent cookie](https://github.com/yt-dlp/yt-dlp/commit/378ae9f9fb8e8c86e6ac89c4c5b815b48ce93620) ([#7774](https://github.com/yt-dlp/yt-dlp/issues/7774)) by [coletdjnz](https://github.com/coletdjnz)
+    - tab: [Detect looping feeds](https://github.com/yt-dlp/yt-dlp/commit/1ba6fe9db5f660d5538588315c23ad6cf0371c5f) ([#6621](https://github.com/yt-dlp/yt-dlp/issues/6621)) by [coletdjnz](https://github.com/coletdjnz)
+- **zaiko**: [Improve thumbnail extraction](https://github.com/yt-dlp/yt-dlp/commit/ecef42c3adbcb6a84405139047923c4967316f28) ([#8054](https://github.com/yt-dlp/yt-dlp/issues/8054)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **zee5**: [Update access token endpoint](https://github.com/yt-dlp/yt-dlp/commit/a0de8bb8601146b8f87bf7cd562eef8bfb4690be) ([#7914](https://github.com/yt-dlp/yt-dlp/issues/7914)) by [bashonly](https://github.com/bashonly)
+- **zoom**: [Extract duration](https://github.com/yt-dlp/yt-dlp/commit/66cc64ff6696f9921ff112a278542f8d999ffea4) by [bashonly](https://github.com/bashonly)
+
+#### Downloader changes
+- **external**
+    - [Fix ffmpeg input from stdin](https://github.com/yt-dlp/yt-dlp/commit/e57eb98222d29cc4c09ee975d3c492274a6e5be3) ([#7655](https://github.com/yt-dlp/yt-dlp/issues/7655)) by [bashonly](https://github.com/bashonly)
+    - [Fixes to cookie handling](https://github.com/yt-dlp/yt-dlp/commit/42ded0a429c20ec13dc006825e1508d9a02f0ad4) by [bashonly](https://github.com/bashonly)
+
+#### Postprocessor changes
+- **embedthumbnail**: [Support `m4v`](https://github.com/yt-dlp/yt-dlp/commit/8a4cd12c8f8e93292e3e95200b9d17a3af39624c) ([#7583](https://github.com/yt-dlp/yt-dlp/issues/7583)) by [Neurognostic](https://github.com/Neurognostic)
+
+#### Networking changes
+- [Add module](https://github.com/yt-dlp/yt-dlp/commit/c365dba8430ee33abda85d31f95128605bf240eb) ([#2861](https://github.com/yt-dlp/yt-dlp/issues/2861)) by [pukkandan](https://github.com/pukkandan)
+- [Add request handler preference framework](https://github.com/yt-dlp/yt-dlp/commit/db7b054a6111ca387220d0eb87bf342f9c130eb8) ([#7603](https://github.com/yt-dlp/yt-dlp/issues/7603)) by [coletdjnz](https://github.com/coletdjnz)
+- [Add strict Request extension checking](https://github.com/yt-dlp/yt-dlp/commit/86aea0d3a213da3be1da638b9b828e6f0ee1d59f) ([#7604](https://github.com/yt-dlp/yt-dlp/issues/7604)) by [coletdjnz](https://github.com/coletdjnz)
+- [Fix POST requests with zero-length payloads](https://github.com/yt-dlp/yt-dlp/commit/71baa490ebd3655746430f208a9b605d120cd315) ([#7648](https://github.com/yt-dlp/yt-dlp/issues/7648)) by [bashonly](https://github.com/bashonly)
+- [Fix `--legacy-server-connect`](https://github.com/yt-dlp/yt-dlp/commit/75dc8e673b481a82d0688aeec30f6c65d82bb359) ([#7645](https://github.com/yt-dlp/yt-dlp/issues/7645)) by [bashonly](https://github.com/bashonly)
+- [Fix various socks proxy bugs](https://github.com/yt-dlp/yt-dlp/commit/20fbbd9249a2f26c7ae579bde5ba5d69aa8fac69) ([#8065](https://github.com/yt-dlp/yt-dlp/issues/8065)) by [coletdjnz](https://github.com/coletdjnz)
+- [Ignore invalid proxies in env](https://github.com/yt-dlp/yt-dlp/commit/bbeacff7fcaa3b521066088a5ccbf34ef5070d1d) ([#7704](https://github.com/yt-dlp/yt-dlp/issues/7704)) by [coletdjnz](https://github.com/coletdjnz)
+- [Rewrite architecture](https://github.com/yt-dlp/yt-dlp/commit/227bf1a33be7b89cd7d44ad046844c4ccba104f4) ([#2861](https://github.com/yt-dlp/yt-dlp/issues/2861)) by [coletdjnz](https://github.com/coletdjnz)
+- **Request Handler**
+    - urllib
+        - [Remove dot segments during URL normalization](https://github.com/yt-dlp/yt-dlp/commit/4bf912282a34b58b6b35d8f7e6be535770c89c76) ([#7662](https://github.com/yt-dlp/yt-dlp/issues/7662)) by [coletdjnz](https://github.com/coletdjnz)
+        - [Simplify gzip decoding](https://github.com/yt-dlp/yt-dlp/commit/59e92b1f1833440bb2190f847eb735cf0f90bc85) ([#7611](https://github.com/yt-dlp/yt-dlp/issues/7611)) by [Grub4K](https://github.com/Grub4K) (With fixes in [77bff23](https://github.com/yt-dlp/yt-dlp/commit/77bff23ee97565bab2e0d75b893a21bf7983219a))
+
+#### Misc. changes
+- **build**: [Make sure deprecated modules are added](https://github.com/yt-dlp/yt-dlp/commit/131d132da5c98c6c78bd7eed4b37f4458561b3d9) by [pukkandan](https://github.com/pukkandan)
+- **cleanup**
+    - [Add color to `download-archive` message](https://github.com/yt-dlp/yt-dlp/commit/2b029ca0a9f9105c4f7626993fa60e54c9782749) ([#5138](https://github.com/yt-dlp/yt-dlp/issues/5138)) by [aaruni96](https://github.com/aaruni96), [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan)
+    - Miscellaneous
+        - [6148833](https://github.com/yt-dlp/yt-dlp/commit/6148833f5ceb7674142ddb8d761ffe03cee7df69), [62b5c94](https://github.com/yt-dlp/yt-dlp/commit/62b5c94cadaa5f596dc1a7083db9db12efe357be) by [pukkandan](https://github.com/pukkandan)
+        - [5ca095c](https://github.com/yt-dlp/yt-dlp/commit/5ca095cbcde3e32642a4fe5b2d69e8e3c785a021) by [barsnick](https://github.com/barsnick), [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz), [gamer191](https://github.com/gamer191), [Grub4K](https://github.com/Grub4K), [sqrtNOT](https://github.com/sqrtNOT)
+        - [088add9](https://github.com/yt-dlp/yt-dlp/commit/088add9567d39b758737e4299a0e619fd89d2e8f) by [Grub4K](https://github.com/Grub4K)
+- **devscripts**: `make_changelog`: [Fix changelog grouping and add networking group](https://github.com/yt-dlp/yt-dlp/commit/30ba233d4cee945756ed7344e7ddb3a90d2ae608) ([#8124](https://github.com/yt-dlp/yt-dlp/issues/8124)) by [Grub4K](https://github.com/Grub4K)
+- **docs**: [Update collaborators](https://github.com/yt-dlp/yt-dlp/commit/1be0a96a4d14f629097509fcc89d15f69a8243c7) by [Grub4K](https://github.com/Grub4K)
+- **test**
+    - [Add tests for socks proxies](https://github.com/yt-dlp/yt-dlp/commit/fcd6a76adc49d5cd8783985c7ce35384b72e545f) ([#7908](https://github.com/yt-dlp/yt-dlp/issues/7908)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Fix `httplib_validation_errors` test for old Python versions](https://github.com/yt-dlp/yt-dlp/commit/95abea9a03289da1384e5bda3d590223ccc0a238) ([#7677](https://github.com/yt-dlp/yt-dlp/issues/7677)) by [coletdjnz](https://github.com/coletdjnz)
+    - [Fix `test_load_certifi`](https://github.com/yt-dlp/yt-dlp/commit/de20687ee6b742646128a7629b57096631a20619) by [pukkandan](https://github.com/pukkandan)
+    - download: [Test for `expected_exception`](https://github.com/yt-dlp/yt-dlp/commit/661c9a1d029296b28e0b2f8be8a72a43abaf6536) by [at-wat](https://github.com/at-wat)
+
 ### 2023.07.06
 
 #### Important changes
diff --git a/supportedsites.md b/supportedsites.md
index 379d28ef3..620e0f305 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -77,7 +77,7 @@ # Supported sites
  - **AnimalPlanet**
  - **ant1newsgr:article**: ant1news.gr articles
  - **ant1newsgr:embed**: ant1news.gr embedded videos
- - **ant1newsgr:watch**: ant1news.gr videos
+ - **antenna:watch**: antenna.gr and ant1news.gr videos
  - **Anvato**
  - **aol.com**: Yahoo screen and movies
  - **APA**
@@ -98,8 +98,6 @@ # Supported sites
  - **ArteTVCategory**
  - **ArteTVEmbed**
  - **ArteTVPlaylist**
- - **AsianCrush**
- - **AsianCrushPlaylist**
  - **AtresPlayer**: [*atresplayer*](## "netrc machine")
  - **AtScaleConfEvent**
  - **ATTTechChannel**
@@ -118,6 +116,7 @@ # Supported sites
  - **awaan:live**
  - **awaan:season**
  - **awaan:video**
+ - **axs.tv**
  - **AZMedien**: AZ Medien videos
  - **BaiduVideo**: 百度视频
  - **BanBye**
@@ -162,11 +161,16 @@ # Supported sites
  - **BilibiliAudioAlbum**
  - **BiliBiliBangumi**
  - **BiliBiliBangumiMedia**
+ - **BiliBiliBangumiSeason**
+ - **BilibiliCollectionList**
+ - **BilibiliFavoritesList**
  - **BiliBiliPlayer**
+ - **BilibiliPlaylist**
  - **BiliBiliSearch**: Bilibili video search; "bilisearch:" prefix
+ - **BilibiliSeriesList**
  - **BilibiliSpaceAudio**
- - **BilibiliSpacePlaylist**
  - **BilibiliSpaceVideo**
+ - **BilibiliWatchlater**
  - **BiliIntl**: [*biliintl*](## "netrc machine")
  - **biliIntl:series**: [*biliintl*](## "netrc machine")
  - **BiliLive**
@@ -201,6 +205,8 @@ # Supported sites
  - **BreitBart**
  - **brightcove:legacy**
  - **brightcove:new**
+ - **Brilliantpala:Classes**: [*brilliantpala*](## "netrc machine") VoD on classes.brilliantpala.org
+ - **Brilliantpala:Elearn**: [*brilliantpala*](## "netrc machine") VoD on elearn.brilliantpala.org
  - **BRMediathek**: Bayerischer Rundfunk Mediathek
  - **bt:article**: Bergens Tidende Articles
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
@@ -220,14 +226,17 @@ # Supported sites
  - **Camsoda**
  - **CamtasiaEmbed**
  - **CamWithHer**
+ - **Canal1**
  - **CanalAlpha**
  - **canalc2.tv**
  - **Canalplus**: mycanal.fr and piwiplus.fr
+ - **CaracolTvPlay**: [*caracoltv-play*](## "netrc machine")
  - **CarambaTV**
  - **CarambaTVPage**
  - **CartoonNetwork**
  - **cbc.ca**
  - **cbc.ca:player**
+ - **cbc.ca:​player:playlist**
  - **CBS**
  - **CBSInteractive**
  - **CBSLocal**
@@ -257,6 +266,8 @@ # Supported sites
  - **Cinchcast**
  - **Cinemax**
  - **CinetecaMilano**
+ - **Cineverse**
+ - **CineverseDetails**
  - **CiscoLiveSearch**
  - **CiscoLiveSession**
  - **ciscowebex**: Cisco Webex
@@ -365,7 +376,7 @@ # Supported sites
  - **Dotsub**
  - **Douyin**
  - **DouyuShow**
- - **DouyuTV**: 斗鱼
+ - **DouyuTV**: 斗鱼直播
  - **DPlay**
  - **DRBonanza**
  - **Drooble**
@@ -408,6 +419,7 @@ # Supported sites
  - **Engadget**
  - **Epicon**
  - **EpiconSeries**
+ - **eplus:inbound**: e+ (イープラス) overseas
  - **Epoch**
  - **Eporner**
  - **EroProfile**: [*eroprofile*](## "netrc machine")
@@ -732,6 +744,7 @@ # Supported sites
  - **lynda**: [*lynda*](## "netrc machine") lynda.com videos
  - **lynda:course**: [*lynda*](## "netrc machine") lynda.com online courses
  - **m6**
+ - **MagellanTV**
  - **MagentaMusik360**
  - **mailru**: Видео@Mail.Ru
  - **mailru:music**: Музыка@Mail.Ru
@@ -812,6 +825,7 @@ # Supported sites
  - **Mofosex**
  - **MofosexEmbed**
  - **Mojvideo**
+ - **Monstercat**
  - **MonsterSirenHypergryphMusic**
  - **Morningstar**: morningstar.com
  - **Motherless**
@@ -840,6 +854,7 @@ # Supported sites
  - **MujRozhlas**
  - **Murrtube**
  - **MurrtubeUser**: Murrtube user profile
+ - **MuseAI**
  - **MuseScore**
  - **MusicdexAlbum**
  - **MusicdexArtist**
@@ -944,6 +959,9 @@ # Supported sites
  - **niconico:playlist**
  - **niconico:series**
  - **niconico:tag**: NicoNico video tag URLs
+ - **NiconicoChannelPlus**: ニコニコチャンネルプラス
+ - **NiconicoChannelPlus:​channel:lives**: ニコニコチャンネルプラス - チャンネル - ライブリスト. nicochannel.jp/channel/lives
+ - **NiconicoChannelPlus:​channel:videos**: ニコニコチャンネルプラス - チャンネル - 動画リスト. nicochannel.jp/channel/videos
  - **NiconicoUser**
  - **nicovideo:search**: Nico video search; "nicosearch:" prefix
  - **nicovideo:​search:date**: Nico video search, newest first; "nicosearchdate:" prefix
@@ -1046,6 +1064,7 @@ # Supported sites
  - **Patreon**
  - **PatreonCampaign**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
+ - **PBSKids**
  - **PearVideo**
  - **PeekVids**
  - **peer.tv**
@@ -1062,6 +1081,7 @@ # Supported sites
  - **phoenix.de**
  - **Photobucket**
  - **Piapro**: [*piapro*](## "netrc machine")
+ - **PIAULIZAPortal**: ulizaportal.jp - PIA LIVE STREAM
  - **Picarto**
  - **PicartoVod**
  - **Piksel**
@@ -1105,6 +1125,7 @@ # Supported sites
  - **polskieradio:​podcast:list**
  - **Popcorntimes**
  - **PopcornTV**
+ - **Pornbox**
  - **PornCom**
  - **PornerBros**
  - **Pornez**
@@ -1121,7 +1142,6 @@ # Supported sites
  - **PornTop**
  - **PornTube**
  - **Pr0gramm**
- - **Pr0grammStatic**
  - **PrankCast**
  - **PremiershipRugby**
  - **PressTV**
@@ -1156,6 +1176,10 @@ # Supported sites
  - **radiocanada**
  - **radiocanada:audiovideo**
  - **radiofrance**
+ - **RadioFranceLive**
+ - **RadioFrancePodcast**
+ - **RadioFranceProfile**
+ - **RadioFranceProgramSchedule**
  - **RadioJavan**
  - **radiokapital**
  - **radiokapital:show**
@@ -1177,6 +1201,7 @@ # Supported sites
  - **RayWenderlichCourse**
  - **RbgTum**
  - **RbgTumCourse**
+ - **RbgTumNewCourse**
  - **RBMARadio**
  - **RCS**
  - **RCSEmbeds**
@@ -1259,6 +1284,8 @@ # Supported sites
  - **Ruutu**
  - **Ruv**
  - **ruv.is:spila**
+ - **S4C**
+ - **S4CSeries**
  - **safari**: [*safari*](## "netrc machine") safaribooksonline.com online video
  - **safari:api**: [*safari*](## "netrc machine")
  - **safari:course**: [*safari*](## "netrc machine") safaribooksonline.com online courses
@@ -1325,6 +1352,7 @@ # Supported sites
  - **Smotrim**
  - **Snotr**
  - **Sohu**
+ - **SohuV**
  - **SonyLIV**: [*sonyliv*](## "netrc machine")
  - **SonyLIVSeries**
  - **soundcloud**: [*soundcloud*](## "netrc machine")
@@ -1378,7 +1406,6 @@ # Supported sites
  - **StoryFireSeries**
  - **StoryFireUser**
  - **Streamable**
- - **Streamanity**
  - **streamcloud.eu**
  - **StreamCZ**
  - **StreamFF**
@@ -1403,6 +1430,9 @@ # Supported sites
  - **Tagesschau**
  - **Tass**
  - **TBS**
+ - **TBSJPEpisode**
+ - **TBSJPPlaylist**
+ - **TBSJPProgram**
  - **TDSLifeway**
  - **Teachable**: [*teachable*](## "netrc machine")
  - **TeachableCourse**: [*teachable*](## "netrc machine")
@@ -1702,7 +1732,6 @@ # Supported sites
  - **wdr:mobile**: (**Currently broken**)
  - **WDRElefant**
  - **WDRPage**
- - **web.archive:vlive**: web.archive.org saved vlive videos
  - **web.archive:youtube**: web.archive.org saved youtube videos, "ytarchive:" prefix
  - **Webcamerapl**
  - **Webcaster**
@@ -1710,7 +1739,8 @@ # Supported sites
  - **WebOfStories**
  - **WebOfStoriesPlaylist**
  - **Weibo**
- - **WeiboMobile**
+ - **WeiboUser**
+ - **WeiboVideo**
  - **WeiqiTV**: WQTV
  - **wetv:episode**
  - **WeTvSeries**
@@ -1726,6 +1756,7 @@ # Supported sites
  - **Whyp**
  - **wikimedia.org**
  - **Willow**
+ - **Wimbledon**
  - **WimTV**
  - **WinSportsVideo**
  - **Wistia**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index 67cfe44ef..2a7c84b93 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.07.06'
+__version__ = '2023.09.24'
 
-RELEASE_GIT_HEAD = 'b532a3481046e1eabb6232ee8196fb696c356ff6'
+RELEASE_GIT_HEAD = '088add9567d39b758737e4299a0e619fd89d2e8f'
 
 VARIANT = None
 

From eb5bdbfa70126c7d5355cc0954b63720522e462c Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Tue, 3 Oct 2023 19:42:30 +1300
Subject: [PATCH 599/871] [ie/youtube] Raise a warning for `Incomplete Data`
 instead of an error (#8238)

Closes https://github.com/yt-dlp/yt-dlp/issues/8206

Adds `raise_incomplete_data` extractor arg to revert this behaviour and raise an error.

Authored by: coletdjnz
Co-authored-by: Simon Sawicki <contact@grub4k.xyz>
---
 README.md                   |  1 +
 yt_dlp/extractor/youtube.py | 26 +++++++++++++++++++-------
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 7bf446572..a0b69c9a1 100644
--- a/README.md
+++ b/README.md
@@ -1809,6 +1809,7 @@ #### youtube
 * `formats`: Change the types of formats to return. `dashy` (convert HTTP to DASH), `duplicate` (identical content but different URLs or protocol; includes `dashy`), `incomplete` (cannot be downloaded completely - live dash and post-live m3u8)
 * `innertube_host`: Innertube API host to use for all API requests; e.g. `studio.youtube.com`, `youtubei.googleapis.com`. Note that cookies exported from one subdomain will not work on others
 * `innertube_key`: Innertube API key to use for all API requests
+* `raise_incomplete_data`: `Incomplete Data Received` raises an error instead of reporting a warning
 
 #### youtubetab (YouTube playlists, channels, feeds, etc.)
 * `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details)
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index a39d17cf1..7e13aa779 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -941,7 +941,13 @@ def _parse_time_text(self, text):
     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
                           default_client='web'):
-        for retry in self.RetryManager():
+        raise_for_incomplete = bool(self._configuration_arg('raise_incomplete_data', ie_key=YoutubeIE))
+        # Incomplete Data should be a warning by default when retries are exhausted, while other errors should be fatal.
+        icd_retries = iter(self.RetryManager(fatal=raise_for_incomplete))
+        icd_rm = next(icd_retries)
+        main_retries = iter(self.RetryManager())
+        main_rm = next(main_retries)
+        for _ in range(main_rm.retries + icd_rm.retries + 1):
             try:
                 response = self._call_api(
                     ep=ep, fatal=True, headers=headers,
@@ -953,7 +959,8 @@ def _extract_response(self, item_id, query, note='Downloading API JSON', headers
                 if not isinstance(e.cause, network_exceptions):
                     return self._error_or_warning(e, fatal=fatal)
                 elif not isinstance(e.cause, HTTPError):
-                    retry.error = e
+                    main_rm.error = e
+                    next(main_retries)
                     continue
 
                 first_bytes = e.cause.response.read(512)
@@ -965,27 +972,32 @@ def _extract_response(self, item_id, query, note='Downloading API JSON', headers
                     if yt_error:
                         self._report_alerts([('ERROR', yt_error)], fatal=False)
                 # Downloading page may result in intermittent 5xx HTTP error
-                # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
+                # Sometimes a 404 is also received. See: https://github.com/ytdl-org/youtube-dl/issues/28289
                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
                 if e.cause.status not in (403, 429):
-                    retry.error = e
+                    main_rm.error = e
+                    next(main_retries)
                     continue
                 return self._error_or_warning(e, fatal=fatal)
 
             try:
                 self._extract_and_report_alerts(response, only_once=True)
             except ExtractorError as e:
-                # YouTube servers may return errors we want to retry on in a 200 OK response
+                # YouTube's servers may return errors we want to retry on in a 200 OK response
                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
                 if 'unknown error' in e.msg.lower():
-                    retry.error = e
+                    main_rm.error = e
+                    next(main_retries)
                     continue
                 return self._error_or_warning(e, fatal=fatal)
             # Youtube sometimes sends incomplete data
             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
             if not traverse_obj(response, *variadic(check_get_keys)):
-                retry.error = ExtractorError('Incomplete data received', expected=True)
+                icd_rm.error = ExtractorError('Incomplete data received', expected=True)
+                should_retry = next(icd_retries, None)
+                if not should_retry:
+                    return None
                 continue
 
             return response

From cc8d8441524ec3442d7c0d3f8f33f15b66aa06f3 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 3 Oct 2023 11:33:40 +0200
Subject: [PATCH 600/871] [ie/xhamster:user] Support creator urls (#8232)

Authored by: Grub4K
---
 yt_dlp/extractor/xhamster.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/xhamster.py b/yt_dlp/extractor/xhamster.py
index 37224799b..aec1f20bb 100644
--- a/yt_dlp/extractor/xhamster.py
+++ b/yt_dlp/extractor/xhamster.py
@@ -407,7 +407,7 @@ def _real_extract(self, url):
 
 
 class XHamsterUserIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?%s/users/(?P<id>[^/?#&]+)' % XHamsterIE._DOMAINS
+    _VALID_URL = rf'https?://(?:[^/?#]+\.)?{XHamsterIE._DOMAINS}/(?:(?P<user>users)|creators)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         # Paginated user profile
         'url': 'https://xhamster.com/users/netvideogirls/videos',
@@ -422,6 +422,12 @@ class XHamsterUserIE(InfoExtractor):
             'id': 'firatkaan',
         },
         'playlist_mincount': 1,
+    }, {
+        'url': 'https://xhamster.com/creators/squirt-orgasm-69',
+        'info_dict': {
+            'id': 'squirt-orgasm-69',
+        },
+        'playlist_mincount': 150,
     }, {
         'url': 'https://xhday.com/users/mobhunter',
         'only_matching': True,
@@ -430,8 +436,9 @@ class XHamsterUserIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _entries(self, user_id):
-        next_page_url = 'https://xhamster.com/users/%s/videos/1' % user_id
+    def _entries(self, user_id, is_user):
+        prefix, suffix = ('users', 'videos') if is_user else ('creators', 'exclusive')
+        next_page_url = f'https://xhamster.com/{prefix}/{user_id}/{suffix}/1'
         for pagenum in itertools.count(1):
             page = self._download_webpage(
                 next_page_url, user_id, 'Downloading page %s' % pagenum)
@@ -454,5 +461,5 @@ def _entries(self, user_id):
                 break
 
     def _real_extract(self, url):
-        user_id = self._match_id(url)
-        return self.playlist_result(self._entries(user_id), user_id)
+        user, user_id = self._match_valid_url(url).group('user', 'id')
+        return self.playlist_result(self._entries(user_id, bool(user)), user_id)

From 0730d5a966fa8a937d84bfb7f68be5198acb039b Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Wed, 4 Oct 2023 12:44:13 -0500
Subject: [PATCH 601/871] [ie/gofile] Fix token cookie bug

Authored by: bashonly
---
 yt_dlp/extractor/gofile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/gofile.py b/yt_dlp/extractor/gofile.py
index 898390583..ef14b57d0 100644
--- a/yt_dlp/extractor/gofile.py
+++ b/yt_dlp/extractor/gofile.py
@@ -60,7 +60,7 @@ def _real_initialize(self):
         account_data = self._download_json(
             'https://api.gofile.io/createAccount', None, note='Getting a new guest account')
         self._TOKEN = account_data['data']['token']
-        self._set_cookie('gofile.io', 'accountToken', self._TOKEN)
+        self._set_cookie('.gofile.io', 'accountToken', self._TOKEN)
 
     def _entries(self, file_id):
         query_params = {

From b095fd3fa9d58a65dc9b830bd63b9d909422aa86 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Wed, 4 Oct 2023 13:01:52 -0500
Subject: [PATCH 602/871] [ie/WrestleUniverseVOD] Call API with device ID
 (#8272)

Closes #8271
Authored by: bashonly
---
 yt_dlp/extractor/wrestleuniverse.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/wrestleuniverse.py b/yt_dlp/extractor/wrestleuniverse.py
index dd12804db..145246a14 100644
--- a/yt_dlp/extractor/wrestleuniverse.py
+++ b/yt_dlp/extractor/wrestleuniverse.py
@@ -190,10 +190,7 @@ class WrestleUniverseVODIE(WrestleUniverseBaseIE):
     def _real_extract(self, url):
         lang, video_id = self._match_valid_url(url).group('lang', 'id')
         metadata = self._download_metadata(url, video_id, lang, 'videoEpisodeFallbackData')
-        video_data = self._call_api(video_id, ':watch', 'watch', data={
-            # 'deviceId' is required if ignoreDeviceRestriction is False
-            'ignoreDeviceRestriction': True,
-        })
+        video_data = self._call_api(video_id, ':watch', 'watch', data={'deviceId': self._DEVICE_ID})
 
         return {
             'id': video_id,

From 91a670a4f7babe9c8aa2018f57d8c8952a6f49d8 Mon Sep 17 00:00:00 2001
From: gillux <jiru@users.noreply.github.com>
Date: Sat, 7 Oct 2023 06:27:54 +0800
Subject: [PATCH 603/871] [ie/LiTV] Fix extractor (#7785)

Closes #5456
Authored by: jiru
---
 yt_dlp/extractor/litv.py | 48 ++++++++++++++++++++++------------------
 1 file changed, 27 insertions(+), 21 deletions(-)

diff --git a/yt_dlp/extractor/litv.py b/yt_dlp/extractor/litv.py
index 19b298ec6..2c7c7175e 100644
--- a/yt_dlp/extractor/litv.py
+++ b/yt_dlp/extractor/litv.py
@@ -13,7 +13,7 @@
 class LiTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?litv\.tv/(?:vod|promo)/[^/]+/(?:content\.do)?\?.*?\b(?:content_)?id=(?P<id>[^&]+)'
 
-    _URL_TEMPLATE = 'https://www.litv.tv/vod/%s/content.do?id=%s'
+    _URL_TEMPLATE = 'https://www.litv.tv/vod/%s/content.do?content_id=%s'
 
     _TESTS = [{
         'url': 'https://www.litv.tv/vod/drama/content.do?brc_id=root&id=VOD00041610&isUHEnabled=true&autoPlay=1',
@@ -21,16 +21,18 @@ class LiTVIE(InfoExtractor):
             'id': 'VOD00041606',
             'title': '花千骨',
         },
-        'playlist_count': 50,
+        'playlist_count': 51,  # 50 episodes + 1 trailer
     }, {
         'url': 'https://www.litv.tv/vod/drama/content.do?brc_id=root&id=VOD00041610&isUHEnabled=true&autoPlay=1',
-        'md5': '969e343d9244778cb29acec608e53640',
+        'md5': 'b90ff1e9f1d8f5cfcd0a44c3e2b34c7a',
         'info_dict': {
             'id': 'VOD00041610',
             'ext': 'mp4',
             'title': '花千骨第1集',
             'thumbnail': r're:https?://.*\.jpg$',
-            'description': 'md5:c7017aa144c87467c4fb2909c4b05d6f',
+            'description': '《花千骨》陸劇線上看。十六年前，平靜的村莊內，一名女嬰隨異相出生，途徑此地的蜀山掌門清虛道長算出此女命運非同一般，她體內散發的異香易招惹妖魔。一念慈悲下，他在村莊周邊設下結界阻擋妖魔入侵，讓其年滿十六後去蜀山，並賜名花千骨。',
+            'categories': ['奇幻', '愛情', '中國', '仙俠'],
+            'episode': 'Episode 1',
             'episode_number': 1,
         },
         'params': {
@@ -46,20 +48,17 @@ class LiTVIE(InfoExtractor):
             'title': '芈月傳第1集　霸星芈月降世楚國',
             'description': '楚威王二年，太史令唐昧夜觀星象，發現霸星即將現世。王后得知霸星的預言後，想盡辦法不讓孩子順利出生，幸得莒姬相護化解危機。沒想到眾人期待下出生的霸星卻是位公主，楚威王對此失望至極。楚王后命人將女嬰丟棄河中，居然奇蹟似的被少司命像攔下，楚威王認為此女非同凡響，為她取名芈月。',
         },
-        'skip': 'Georestricted to Taiwan',
+        'skip': 'No longer exists',
     }]
 
-    def _extract_playlist(self, season_list, video_id, program_info, prompt=True):
-        episode_title = program_info['title']
-        content_id = season_list['contentId']
-
+    def _extract_playlist(self, playlist_data, content_type):
         all_episodes = [
             self.url_result(smuggle_url(
-                self._URL_TEMPLATE % (program_info['contentType'], episode['contentId']),
+                self._URL_TEMPLATE % (content_type, episode['contentId']),
                 {'force_noplaylist': True}))  # To prevent infinite recursion
-            for episode in season_list['episode']]
+            for episode in traverse_obj(playlist_data, ('seasons', ..., 'episode', lambda _, v: v['contentId']))]
 
-        return self.playlist_result(all_episodes, content_id, episode_title)
+        return self.playlist_result(all_episodes, playlist_data['contentId'], playlist_data.get('title'))
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
@@ -68,24 +67,31 @@ def _real_extract(self, url):
 
         webpage = self._download_webpage(url, video_id)
 
+        if self._search_regex(
+                r'(?i)<meta\s[^>]*http-equiv="refresh"\s[^>]*content="[0-9]+;\s*url=https://www\.litv\.tv/"',
+                webpage, 'meta refresh redirect', default=False, group=0):
+            raise ExtractorError('No such content found', expected=True)
+
         program_info = self._parse_json(self._search_regex(
             r'var\s+programInfo\s*=\s*([^;]+)', webpage, 'VOD data', default='{}'),
             video_id)
 
-        season_list = list(program_info.get('seasonList', {}).values())
-        playlist_id = traverse_obj(season_list, 0, 'contentId')
-        if self._yes_playlist(playlist_id, video_id, smuggled_data):
-            return self._extract_playlist(season_list[0], video_id, program_info)
-
-        # In browsers `getMainUrl` request is always issued. Usually this
+        # In browsers `getProgramInfo` request is always issued. Usually this
         # endpoint gives the same result as the data embedded in the webpage.
-        # If georestricted, there are no embedded data, so an extra request is
-        # necessary to get the error code
+        # If, for some reason, there are no embedded data, we do an extra request.
         if 'assetId' not in program_info:
             program_info = self._download_json(
                 'https://www.litv.tv/vod/ajax/getProgramInfo', video_id,
                 query={'contentId': video_id},
                 headers={'Accept': 'application/json'})
+
+        series_id = program_info['seriesId']
+        if self._yes_playlist(series_id, video_id, smuggled_data):
+            playlist_data = self._download_json(
+                'https://www.litv.tv/vod/ajax/getSeriesTree', video_id,
+                query={'seriesId': series_id}, headers={'Accept': 'application/json'})
+            return self._extract_playlist(playlist_data, program_info['contentType'])
+
         video_data = self._parse_json(self._search_regex(
             r'uiHlsUrl\s*=\s*testBackendData\(([^;]+)\);',
             webpage, 'video data', default='{}'), video_id)
@@ -96,7 +102,7 @@ def _real_extract(self, url):
                 'contentType': program_info['contentType'],
             }
             video_data = self._download_json(
-                'https://www.litv.tv/vod/getMainUrl', video_id,
+                'https://www.litv.tv/vod/ajax/getMainUrlNoAuth', video_id,
                 data=json.dumps(payload).encode('utf-8'),
                 headers={'Content-Type': 'application/json'})
 

From f980df734cf5c0eaded2f7b38c6c60bccfeebb48 Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Fri, 6 Oct 2023 18:31:33 -0400
Subject: [PATCH 604/871] [ie/neteasemusic] Fix extractors (#8181)

Closes #4388
Authored by: c-basalt
---
 yt_dlp/extractor/neteasemusic.py | 575 +++++++++++++++++--------------
 1 file changed, 312 insertions(+), 263 deletions(-)

diff --git a/yt_dlp/extractor/neteasemusic.py b/yt_dlp/extractor/neteasemusic.py
index 5b7307bc8..68bfcb6ba 100644
--- a/yt_dlp/extractor/neteasemusic.py
+++ b/yt_dlp/extractor/neteasemusic.py
@@ -2,105 +2,74 @@
 import json
 import re
 import time
-from base64 import b64encode
-from binascii import hexlify
-from datetime import datetime
 from hashlib import md5
 from random import randint
 
 from .common import InfoExtractor
 from ..aes import aes_ecb_encrypt, pkcs7_padding
-from ..compat import compat_urllib_parse_urlencode
-from ..networking import Request
 from ..utils import (
     ExtractorError,
-    bytes_to_intlist,
-    error_to_compat_str,
-    float_or_none,
     int_or_none,
-    intlist_to_bytes,
-    try_get,
+    join_nonempty,
+    str_or_none,
+    strftime_or_none,
+    traverse_obj,
+    unified_strdate,
+    url_or_none,
+    urljoin,
+    variadic,
 )
 
 
 class NetEaseMusicBaseIE(InfoExtractor):
     _FORMATS = ['bMusic', 'mMusic', 'hMusic']
-    _NETEASE_SALT = '3go8&$8*3*3h0k(2)2'
     _API_BASE = 'http://music.163.com/api/'
+    _GEO_BYPASS = False
 
-    @classmethod
-    def _encrypt(cls, dfsid):
-        salt_bytes = bytearray(cls._NETEASE_SALT.encode('utf-8'))
-        string_bytes = bytearray(str(dfsid).encode('ascii'))
-        salt_len = len(salt_bytes)
-        for i in range(len(string_bytes)):
-            string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
-        m = md5()
-        m.update(bytes(string_bytes))
-        result = b64encode(m.digest()).decode('ascii')
-        return result.replace('/', '_').replace('+', '-')
+    @staticmethod
+    def kilo_or_none(value):
+        return int_or_none(value, scale=1000)
 
-    def make_player_api_request_data_and_headers(self, song_id, bitrate):
-        KEY = b'e82ckenh8dichen8'
-        URL = '/api/song/enhance/player/url'
-        now = int(time.time() * 1000)
-        rand = randint(0, 1000)
-        cookie = {
-            'osver': None,
-            'deviceId': None,
+    def _create_eapi_cipher(self, api_path, query_body, cookies):
+        request_text = json.dumps({**query_body, 'header': cookies}, separators=(',', ':'))
+
+        message = f'nobody{api_path}use{request_text}md5forencrypt'.encode('latin1')
+        msg_digest = md5(message).hexdigest()
+
+        data = pkcs7_padding(list(str.encode(
+            f'{api_path}-36cd479b6b5-{request_text}-36cd479b6b5-{msg_digest}')))
+        encrypted = bytes(aes_ecb_encrypt(data, list(b'e82ckenh8dichen8')))
+        return f'params={encrypted.hex().upper()}'.encode()
+
+    def _download_eapi_json(self, path, video_id, query_body, headers={}, **kwargs):
+        cookies = {
+            'osver': 'undefined',
+            'deviceId': 'undefined',
             'appver': '8.0.0',
             'versioncode': '140',
-            'mobilename': None,
+            'mobilename': 'undefined',
             'buildver': '1623435496',
             'resolution': '1920x1080',
             '__csrf': '',
             'os': 'pc',
-            'channel': None,
-            'requestId': '{0}_{1:04}'.format(now, rand),
+            'channel': 'undefined',
+            'requestId': f'{int(time.time() * 1000)}_{randint(0, 1000):04}',
+            **traverse_obj(self._get_cookies(self._API_BASE), {
+                'MUSIC_U': ('MUSIC_U', {lambda i: i.value}),
+            })
         }
-        request_text = json.dumps(
-            {'ids': '[{0}]'.format(song_id), 'br': bitrate, 'header': cookie},
-            separators=(',', ':'))
-        message = 'nobody{0}use{1}md5forencrypt'.format(
-            URL, request_text).encode('latin1')
-        msg_digest = md5(message).hexdigest()
-
-        data = '{0}-36cd479b6b5-{1}-36cd479b6b5-{2}'.format(
-            URL, request_text, msg_digest)
-        data = pkcs7_padding(bytes_to_intlist(data))
-        encrypted = intlist_to_bytes(aes_ecb_encrypt(data, bytes_to_intlist(KEY)))
-        encrypted_params = hexlify(encrypted).decode('ascii').upper()
-
-        cookie = '; '.join(
-            ['{0}={1}'.format(k, v if v is not None else 'undefined')
-             for [k, v] in cookie.items()])
-
-        headers = {
-            'User-Agent': self.extractor.get_param('http_headers')['User-Agent'],
-            'Content-Type': 'application/x-www-form-urlencoded',
-            'Referer': 'https://music.163.com',
-            'Cookie': cookie,
-        }
-        return ('params={0}'.format(encrypted_params), headers)
+        return self._download_json(
+            urljoin('https://interface3.music.163.com/', f'/eapi{path}'), video_id,
+            data=self._create_eapi_cipher(f'/api{path}', query_body, cookies), headers={
+                'Referer': 'https://music.163.com',
+                'Cookie': '; '.join([f'{k}={v}' for k, v in cookies.items()]),
+                **headers,
+            }, **kwargs)
 
     def _call_player_api(self, song_id, bitrate):
-        url = 'https://interface3.music.163.com/eapi/song/enhance/player/url'
-        data, headers = self.make_player_api_request_data_and_headers(song_id, bitrate)
-        try:
-            msg = 'empty result'
-            result = self._download_json(
-                url, song_id, data=data.encode('ascii'), headers=headers)
-            if result:
-                return result
-        except ExtractorError as e:
-            if type(e.cause) in (ValueError, TypeError):
-                # JSON load failure
-                raise
-        except Exception as e:
-            msg = error_to_compat_str(e)
-            self.report_warning('%s API call (%s) failed: %s' % (
-                song_id, bitrate, msg))
-        return {}
+        return self._download_eapi_json(
+            '/song/enhance/player/url', song_id, {'ids': f'[{song_id}]', 'br': bitrate},
+            note=f'Downloading song URL info: bitrate {bitrate}')
 
     def extract_formats(self, info):
         err = 0
@@ -110,45 +79,50 @@ def extract_formats(self, info):
             details = info.get(song_format)
             if not details:
                 continue
-
             bitrate = int_or_none(details.get('bitrate')) or 999000
-            data = self._call_player_api(song_id, bitrate)
-            for song in try_get(data, lambda x: x['data'], list) or []:
-                song_url = try_get(song, lambda x: x['url'])
-                if not song_url:
-                    continue
+            for song in traverse_obj(self._call_player_api(song_id, bitrate), ('data', lambda _, v: url_or_none(v['url']))):
+                song_url = song['url']
                 if self._is_valid_url(song_url, info['id'], 'song'):
                     formats.append({
                         'url': song_url,
-                        'ext': details.get('extension'),
-                        'abr': float_or_none(song.get('br'), scale=1000),
                         'format_id': song_format,
-                        'filesize': int_or_none(song.get('size')),
-                        'asr': int_or_none(details.get('sr')),
+                        'asr': traverse_obj(details, ('sr', {int_or_none})),
+                        **traverse_obj(song, {
+                            'ext': ('type', {str}),
+                            'abr': ('br', {self.kilo_or_none}),
+                            'filesize': ('size', {int_or_none}),
+                        }),
                     })
                 elif err == 0:
-                    err = try_get(song, lambda x: x['code'], int)
+                    err = traverse_obj(song, ('code', {int})) or 0
 
         if not formats:
-            msg = 'No media links found'
             if err != 0 and (err < 200 or err >= 400):
-                raise ExtractorError(
-                    '%s (site code %d)' % (msg, err, ), expected=True)
+                raise ExtractorError(f'No media links found (site code {err})', expected=True)
             else:
                 self.raise_geo_restricted(
-                    msg + ': probably this video is not available from your location due to geo restriction.',
-                    countries=['CN'])
-
+                    'No media links found: probably due to geo restriction.', countries=['CN'])
         return formats
 
-    @classmethod
-    def convert_milliseconds(cls, ms):
-        return int(round(ms / 1000.0))
-
     def query_api(self, endpoint, video_id, note):
-        req = Request('%s%s' % (self._API_BASE, endpoint))
-        req.headers['Referer'] = self._API_BASE
-        return self._download_json(req, video_id, note)
+        result = self._download_json(
+            f'{self._API_BASE}{endpoint}', video_id, note, headers={'Referer': self._API_BASE})
+        code = traverse_obj(result, ('code', {int}))
+        message = traverse_obj(result, ('message', {str})) or ''
+        if code == -462:
+            self.raise_login_required(f'Login required to download: {message}')
+        elif code != 200:
+            raise ExtractorError(f'Failed to get meta info: {code} {message}')
+        return result
+
+    def _get_entries(self, songs_data, entry_keys=None, id_key='id', name_key='name'):
+        for song in traverse_obj(songs_data, (
+                *variadic(entry_keys, (str, bytes, dict, set)),
+                lambda _, v: int_or_none(v[id_key]) is not None)):
+            song_id = str(song[id_key])
+            yield self.url_result(
+                f'http://music.163.com/#/song?id={song_id}', NetEaseMusicIE,
+                song_id, traverse_obj(song, (name_key, {str})))
 
 
 class NetEaseMusicIE(NetEaseMusicBaseIE):
@@ -156,16 +130,18 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
     IE_DESC = '网易云音乐'
     _VALID_URL = r'https?://(y\.)?music\.163\.com/(?:[#m]/)?song\?.*?\bid=(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://music.163.com/#/song?id=32102397',
-        'md5': '3e909614ce09b1ccef4a3eb205441190',
+        'url': 'https://music.163.com/#/song?id=548648087',
         'info_dict': {
-            'id': '32102397',
+            'id': '548648087',
             'ext': 'mp3',
-            'title': 'Bad Blood',
-            'creator': 'Taylor Swift / Kendrick Lamar',
-            'upload_date': '20150516',
-            'timestamp': 1431792000,
-            'description': 'md5:25fc5f27e47aad975aa6d36382c7833c',
+            'title': '戒烟 (Live)',
+            'creator': '李荣浩 / 朱正廷 / 陈立农 / 尤长靖 / ONER灵超 / ONER木子洋 / 杨非同 / 陆定昊',
+            'timestamp': 1522944000,
+            'upload_date': '20180405',
+            'description': 'md5:3650af9ee22c87e8637cb2dde22a765c',
+            'subtitles': {'lyrics': [{'ext': 'lrc'}]},
+            "duration": 256,
+            'thumbnail': r're:^http.*\.jpg',
         },
     }, {
         'note': 'No lyrics.',
@@ -176,21 +152,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'title': 'Opus 28',
             'creator': 'Dustin O\'Halloran',
             'upload_date': '20080211',
-            'description': 'md5:f12945b0f6e0365e3b73c5032e1b0ff4',
             'timestamp': 1202745600,
-        },
-    }, {
-        'note': 'Has translated name.',
-        'url': 'http://music.163.com/#/song?id=22735043',
-        'info_dict': {
-            'id': '22735043',
-            'ext': 'mp3',
-            'title': '소원을 말해봐 (Genie)',
-            'creator': '少女时代',
-            'description': 'md5:79d99cc560e4ca97e0c4d86800ee4184',
-            'upload_date': '20100127',
-            'timestamp': 1264608000,
-            'alt_title': '说出愿望吧(Genie)',
+            'duration': 263,
+            'thumbnail': r're:^http.*\.jpg',
         },
     }, {
         'url': 'https://y.music.163.com/m/song?app_version=8.8.45&id=95670&uct2=sKnvS4+0YStsWkqsPhFijw%3D%3D&dlt=0846',
@@ -203,59 +167,99 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'upload_date': '19911130',
             'timestamp': 691516800,
             'description': 'md5:1ba2f911a2b0aa398479f595224f2141',
+            'subtitles': {'lyrics': [{'ext': 'lrc'}]},
+            'duration': 268,
+            'alt_title': '伴唱:现代人乐队 合唱:总政歌舞团',
+            'thumbnail': r're:^http.*\.jpg',
         },
+    }, {
+        'url': 'http://music.163.com/#/song?id=32102397',
+        'md5': '3e909614ce09b1ccef4a3eb205441190',
+        'info_dict': {
+            'id': '32102397',
+            'ext': 'mp3',
+            'title': 'Bad Blood',
+            'creator': 'Taylor Swift / Kendrick Lamar',
+            'upload_date': '20150516',
+            'timestamp': 1431792000,
+            'description': 'md5:21535156efb73d6d1c355f95616e285a',
+            'subtitles': {'lyrics': [{'ext': 'lrc'}]},
+            'duration': 199,
+            'thumbnail': r're:^http.*\.jpg',
+        },
+        'skip': 'Blocked outside Mainland China',
+    }, {
+        'note': 'Has translated name.',
+        'url': 'http://music.163.com/#/song?id=22735043',
+        'info_dict': {
+            'id': '22735043',
+            'ext': 'mp3',
+            'title': '소원을 말해봐 (Genie)',
+            'creator': '少女时代',
+            'upload_date': '20100127',
+            'timestamp': 1264608000,
+            'description': 'md5:03d1ffebec3139aa4bafe302369269c5',
+            'subtitles': {'lyrics': [{'ext': 'lrc'}]},
+            'duration': 229,
+            'alt_title': '说出愿望吧(Genie)',
+            'thumbnail': r're:^http.*\.jpg',
+        },
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _process_lyrics(self, lyrics_info):
-        original = lyrics_info.get('lrc', {}).get('lyric')
-        translated = lyrics_info.get('tlyric', {}).get('lyric')
+        original = traverse_obj(lyrics_info, ('lrc', 'lyric', {str}))
+        translated = traverse_obj(lyrics_info, ('tlyric', 'lyric', {str}))
+
+        if not original or original == '[99:00.00]纯音乐，请欣赏\n':
+            return None
 
         if not translated:
-            return original
+            return {
+                'lyrics': [{'data': original, 'ext': 'lrc'}],
+            }
 
         lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
         original_ts_texts = re.findall(lyrics_expr, original)
-        translation_ts_dict = dict(
-            (time_stamp, text) for time_stamp, text in re.findall(lyrics_expr, translated)
-        )
-        lyrics = '\n'.join([
-            '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
-            for time_stamp, text in original_ts_texts
-        ])
-        return lyrics
+        translation_ts_dict = dict(re.findall(lyrics_expr, translated))
+
+        merged = '\n'.join(
+            join_nonempty(f'{timestamp}{text}', translation_ts_dict.get(timestamp, ''), delim=' / ')
+            for timestamp, text in original_ts_texts)
+
+        return {
+            'lyrics_merged': [{'data': merged, 'ext': 'lrc'}],
+            'lyrics': [{'data': original, 'ext': 'lrc'}],
+            'lyrics_translated': [{'data': translated, 'ext': 'lrc'}],
+        }
 
     def _real_extract(self, url):
         song_id = self._match_id(url)
 
-        params = {
-            'id': song_id,
-            'ids': '[%s]' % song_id
-        }
         info = self.query_api(
-            'song/detail?' + compat_urllib_parse_urlencode(params),
-            song_id, 'Downloading song info')['songs'][0]
+            f'song/detail?id={song_id}&ids=%5B{song_id}%5D', song_id, 'Downloading song info')['songs'][0]
 
         formats = self.extract_formats(info)
 
-        lyrics_info = self.query_api(
-            'song/lyric?id=%s&lv=-1&tv=-1' % song_id,
-            song_id, 'Downloading lyrics data')
-        lyrics = self._process_lyrics(lyrics_info)
-
-        alt_title = None
-        if info.get('transNames'):
-            alt_title = '/'.join(info.get('transNames'))
+        lyrics = self._process_lyrics(self.query_api(
+            f'song/lyric?id={song_id}&lv=-1&tv=-1', song_id, 'Downloading lyrics data'))
+        lyric_data = {
+            'description': traverse_obj(lyrics, (('lyrics_merged', 'lyrics'), 0, 'data'), get_all=False),
+            'subtitles': lyrics,
+        } if lyrics else {}
 
         return {
             'id': song_id,
-            'title': info['name'],
-            'alt_title': alt_title,
-            'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
-            'timestamp': self.convert_milliseconds(info.get('album', {}).get('publishTime')),
-            'thumbnail': info.get('album', {}).get('picUrl'),
-            'duration': self.convert_milliseconds(info.get('duration', 0)),
-            'description': lyrics,
             'formats': formats,
+            'alt_title': '/'.join(traverse_obj(info, (('transNames', 'alias'), ...))) or None,
+            'creator': ' / '.join(traverse_obj(info, ('artists', ..., 'name'))) or None,
+            **lyric_data,
+            **traverse_obj(info, {
+                'title': ('name', {str}),
+                'timestamp': ('album', 'publishTime', {self.kilo_or_none}),
+                'thumbnail': ('album', 'picUrl', {url_or_none}),
+                'duration': ('duration', {self.kilo_or_none}),
+            }),
         }
 
 
@@ -263,31 +267,44 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:album'
     IE_DESC = '网易云音乐 - 专辑'
     _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
+        'url': 'https://music.163.com/#/album?id=133153666',
+        'info_dict': {
+            'id': '133153666',
+            'title': '桃几的翻唱',
+            'upload_date': '20210913',
+            'description': '桃几2021年翻唱合集',
+            'thumbnail': r're:^http.*\.jpg',
+        },
+        'playlist_mincount': 13,
+    }, {
         'url': 'http://music.163.com/#/album?id=220780',
         'info_dict': {
             'id': '220780',
-            'title': 'B\'day',
+            'title': 'B\'Day',
+            'upload_date': '20060904',
+            'description': 'md5:71a74e1d8f392d88cf1bbe48879ad0b0',
+            'thumbnail': r're:^http.*\.jpg',
         },
         'playlist_count': 23,
-        'skip': 'Blocked outside Mainland China',
-    }
+    }]
 
     def _real_extract(self, url):
         album_id = self._match_id(url)
+        webpage = self._download_webpage(f'https://music.163.com/album?id={album_id}', album_id)
 
-        info = self.query_api(
-            'album/%s?id=%s' % (album_id, album_id),
-            album_id, 'Downloading album data')['album']
-
-        name = info['name']
-        desc = info.get('description')
-        entries = [
-            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
-                            'NetEaseMusic', song['id'])
-            for song in info['songs']
-        ]
-        return self.playlist_result(entries, album_id, name, desc)
+        songs = self._search_json(
+            r'<textarea[^>]+\bid="song-list-pre-data"[^>]*>', webpage, 'metainfo', album_id,
+            end_pattern=r'</textarea>', contains_pattern=r'\[(?s:.+)\]')
+        metainfo = {
+            'title': self._og_search_property('title', webpage, 'title', fatal=False),
+            'description': self._html_search_regex(
+                (rf'<div[^>]+\bid="album-desc-{suffix}"[^>]*>(.*?)</div>' for suffix in ('more', 'dot')),
+                webpage, 'description', flags=re.S, fatal=False),
+            'thumbnail': self._og_search_property('image', webpage, 'thumbnail', fatal=False),
+            'upload_date': unified_strdate(self._html_search_meta('music:release_date', webpage, 'date', fatal=False)),
+        }
+        return self.playlist_result(self._get_entries(songs), album_id, **metainfo)
 
 
 class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
@@ -299,10 +316,9 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
         'url': 'http://music.163.com/#/artist?id=10559',
         'info_dict': {
             'id': '10559',
-            'title': '张惠妹 - aMEI;阿密特',
+            'title': '张惠妹 - aMEI;阿妹;阿密特',
         },
         'playlist_count': 50,
-        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Singer has translated name.',
         'url': 'http://music.163.com/#/artist?id=124098',
@@ -311,28 +327,28 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
             'title': '李昇基 - 이승기',
         },
         'playlist_count': 50,
-        'skip': 'Blocked outside Mainland China',
+    }, {
+        'note': 'Singer with both translated and alias',
+        'url': 'https://music.163.com/#/artist?id=159692',
+        'info_dict': {
+            'id': '159692',
+            'title': '初音ミク - 初音未来;Hatsune Miku',
+        },
+        'playlist_count': 50,
     }]
 
     def _real_extract(self, url):
         singer_id = self._match_id(url)
 
         info = self.query_api(
-            'artist/%s?id=%s' % (singer_id, singer_id),
-            singer_id, 'Downloading singer data')
+            f'artist/{singer_id}?id={singer_id}', singer_id, note='Downloading singer data')
 
-        name = info['artist']['name']
-        if info['artist']['trans']:
-            name = '%s - %s' % (name, info['artist']['trans'])
-        if info['artist']['alias']:
-            name = '%s - %s' % (name, ';'.join(info['artist']['alias']))
+        name = join_nonempty(
+            traverse_obj(info, ('artist', 'name', {str})),
+            join_nonempty(*traverse_obj(info, ('artist', ('trans', ('alias', ...)), {str})), delim=';'),
+            delim=' - ')
 
-        entries = [
-            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
-                            'NetEaseMusic', song['id'])
-            for song in info['hotSongs']
-        ]
-        return self.playlist_result(entries, singer_id, name)
+        return self.playlist_result(self._get_entries(info, 'hotSongs'), singer_id, name)
 
 
 class NetEaseMusicListIE(NetEaseMusicBaseIE):
@@ -344,10 +360,28 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
         'info_dict': {
             'id': '79177352',
             'title': 'Billboard 2007 Top 100',
-            'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
+            'description': 'md5:12fd0819cab2965b9583ace0f8b7b022',
+            'tags': ['欧美'],
+            'uploader': '浑然破灭',
+            'uploader_id': '67549805',
+            'timestamp': int,
+            'upload_date': r're:\d{8}',
         },
-        'playlist_count': 99,
-        'skip': 'Blocked outside Mainland China',
+        'playlist_mincount': 95,
+    }, {
+        'note': 'Toplist/Charts sample',
+        'url': 'https://music.163.com/#/discover/toplist?id=60198',
+        'info_dict': {
+            'id': '60198',
+            'title': 're:美国Billboard榜 [0-9]{4}-[0-9]{2}-[0-9]{2}',
+            'description': '美国Billboard排行榜',
+            'tags': ['流行', '欧美', '榜单'],
+            'uploader': 'Billboard公告牌',
+            'uploader_id': '48171',
+            'timestamp': int,
+            'upload_date': r're:\d{8}',
+        },
+        'playlist_count': 100,
     }, {
         'note': 'Toplist/Charts sample',
         'url': 'http://music.163.com/#/discover/toplist?id=3733003',
@@ -363,64 +397,86 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
-        info = self.query_api(
-            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id,
-            list_id, 'Downloading playlist data')['result']
+        info = self._download_eapi_json(
+            '/v3/playlist/detail', list_id,
+            {'id': list_id, 't': '-1', 'n': '500', 's': '0'},
+            note="Downloading playlist info")
 
-        name = info['name']
-        desc = info.get('description')
+        metainfo = traverse_obj(info, ('playlist', {
+            'title': ('name', {str}),
+            'description': ('description', {str}),
+            'tags': ('tags', ..., {str}),
+            'uploader': ('creator', 'nickname', {str}),
+            'uploader_id': ('creator', 'userId', {str_or_none}),
+            'timestamp': ('updateTime', {self.kilo_or_none}),
+        }))
+        if traverse_obj(info, ('playlist', 'specialType')) == 10:
+            metainfo['title'] = f'{metainfo.get("title")} {strftime_or_none(metainfo.get("timestamp"), "%Y-%m-%d")}'
 
-        if info.get('specialType') == 10:  # is a chart/toplist
-            datestamp = datetime.fromtimestamp(
-                self.convert_milliseconds(info['updateTime'])).strftime('%Y-%m-%d')
-            name = '%s %s' % (name, datestamp)
-
-        entries = [
-            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
-                            'NetEaseMusic', song['id'])
-            for song in info['tracks']
-        ]
-        return self.playlist_result(entries, list_id, name, desc)
+        return self.playlist_result(self._get_entries(info, ('playlist', 'tracks')), list_id, **metainfo)
 
 
 class NetEaseMusicMvIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:mv'
     IE_DESC = '网易云音乐 - MV'
     _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
+        'url': 'https://music.163.com/#/mv?id=10958064',
+        'info_dict': {
+            'id': '10958064',
+            'ext': 'mp4',
+            'title': '交换余生',
+            'description': 'md5:e845872cff28820642a2b02eda428fea',
+            'creator': '林俊杰',
+            'upload_date': '20200916',
+            'thumbnail': r're:http.*\.jpg',
+            'duration': 364,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+    }, {
         'url': 'http://music.163.com/#/mv?id=415350',
         'info_dict': {
             'id': '415350',
             'ext': 'mp4',
             'title': '이럴거면 그러지말지',
             'description': '白雅言自作曲唱甜蜜爱情',
-            'creator': '白雅言',
+            'creator': '白娥娟',
             'upload_date': '20150520',
+            'thumbnail': r're:http.*\.jpg',
+            'duration': 216,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
         },
-        'skip': 'Blocked outside Mainland China',
-    }
+    }]
 
     def _real_extract(self, url):
         mv_id = self._match_id(url)
 
         info = self.query_api(
-            'mv/detail?id=%s&type=mp4' % mv_id,
-            mv_id, 'Downloading mv info')['data']
+            f'mv/detail?id={mv_id}&type=mp4', mv_id, 'Downloading mv info')['data']
 
         formats = [
-            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'height': int(brs)}
+            {'url': mv_url, 'ext': 'mp4', 'format_id': f'{brs}p', 'height': int_or_none(brs)}
             for brs, mv_url in info['brs'].items()
         ]
 
         return {
             'id': mv_id,
-            'title': info['name'],
-            'description': info.get('desc') or info.get('briefDesc'),
-            'creator': info['artistName'],
-            'upload_date': info['publishTime'].replace('-', ''),
             'formats': formats,
-            'thumbnail': info.get('cover'),
-            'duration': self.convert_milliseconds(info.get('duration', 0)),
+            **traverse_obj(info, {
+                'title': ('name', {str}),
+                'description': (('desc', 'briefDesc'), {str}, {lambda x: x or None}),
+                'creator': ('artistName', {str}),
+                'upload_date': ('publishTime', {unified_strdate}),
+                'thumbnail': ('cover', {url_or_none}),
+                'duration': ('duration', {self.kilo_or_none}),
+                'view_count': ('playCount', {int_or_none}),
+                'like_count': ('likeCount', {int_or_none}),
+                'comment_count': ('commentCount', {int_or_none}),
+            }, get_all=False),
         }
 
 
@@ -431,75 +487,74 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
     _TESTS = [{
         'url': 'http://music.163.com/#/program?id=10109055',
         'info_dict': {
-            'id': '10109055',
+            'id': '32593346',
             'ext': 'mp3',
             'title': '不丹足球背后的故事',
             'description': '喜马拉雅人的足球梦 ...',
             'creator': '大话西藏',
-            'timestamp': 1434179342,
+            'timestamp': 1434179287,
             'upload_date': '20150613',
+            'thumbnail': r're:http.*\.jpg',
             'duration': 900,
         },
-        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'This program has accompanying songs.',
         'url': 'http://music.163.com/#/program?id=10141022',
         'info_dict': {
             'id': '10141022',
-            'title': '25岁，你是自在如风的少年<27°C>',
+            'title': '滚滚电台的有声节目',
             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
+            'creator': '滚滚电台ORZ',
+            'timestamp': 1434450733,
+            'upload_date': '20150616',
+            'thumbnail': r're:http.*\.jpg',
         },
         'playlist_count': 4,
-        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'This program has accompanying songs.',
         'url': 'http://music.163.com/#/program?id=10141022',
         'info_dict': {
-            'id': '10141022',
+            'id': '32647209',
             'ext': 'mp3',
-            'title': '25岁，你是自在如风的少年<27°C>',
+            'title': '滚滚电台的有声节目',
             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
-            'timestamp': 1434450841,
+            'creator': '滚滚电台ORZ',
+            'timestamp': 1434450733,
             'upload_date': '20150616',
+            'thumbnail': r're:http.*\.jpg',
+            'duration': 1104,
         },
         'params': {
             'noplaylist': True
         },
-        'skip': 'Blocked outside Mainland China',
     }]
 
     def _real_extract(self, url):
         program_id = self._match_id(url)
 
         info = self.query_api(
-            'dj/program/detail?id=%s' % program_id,
-            program_id, 'Downloading program info')['program']
+            f'dj/program/detail?id={program_id}', program_id, note='Downloading program info')['program']
 
-        name = info['name']
-        description = info['description']
+        metainfo = traverse_obj(info, {
+            'title': ('name', {str}),
+            'description': ('description', {str}),
+            'creator': ('dj', 'brand', {str}),
+            'thumbnail': ('coverUrl', {url_or_none}),
+            'timestamp': ('createTime', {self.kilo_or_none}),
+        })
 
         if not self._yes_playlist(info['songs'] and program_id, info['mainSong']['id']):
             formats = self.extract_formats(info['mainSong'])
 
             return {
-                'id': info['mainSong']['id'],
-                'title': name,
-                'description': description,
-                'creator': info['dj']['brand'],
-                'timestamp': self.convert_milliseconds(info['createTime']),
-                'thumbnail': info['coverUrl'],
-                'duration': self.convert_milliseconds(info.get('duration', 0)),
+                'id': str(info['mainSong']['id']),
                 'formats': formats,
+                'duration': traverse_obj(info, ('mainSong', 'duration', {self.kilo_or_none})),
+                **metainfo,
             }
 
-        song_ids = [info['mainSong']['id']]
-        song_ids.extend([song['id'] for song in info['songs']])
-        entries = [
-            self.url_result('http://music.163.com/#/song?id=%s' % song_id,
-                            'NetEaseMusic', song_id)
-            for song_id in song_ids
-        ]
-        return self.playlist_result(entries, program_id, name, description)
+        songs = traverse_obj(info, (('mainSong', ('songs', ...)),))
+        return self.playlist_result(self._get_entries(songs), program_id, **metainfo)
 
 
 class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
@@ -511,38 +566,32 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
         'info_dict': {
             'id': '42',
             'title': '声音蔓延',
-            'description': 'md5:766220985cbd16fdd552f64c578a6b15'
+            'description': 'md5:c7381ebd7989f9f367668a5aee7d5f08'
         },
         'playlist_mincount': 40,
-        'skip': 'Blocked outside Mainland China',
     }
     _PAGE_SIZE = 1000
 
     def _real_extract(self, url):
         dj_id = self._match_id(url)
 
-        name = None
-        desc = None
+        metainfo = {}
         entries = []
         for offset in itertools.count(start=0, step=self._PAGE_SIZE):
             info = self.query_api(
-                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
-                % (self._PAGE_SIZE, dj_id, offset),
-                dj_id, 'Downloading dj programs - %d' % offset)
+                f'dj/program/byradio?asc=false&limit={self._PAGE_SIZE}&radioId={dj_id}&offset={offset}',
+                dj_id, note=f'Downloading dj programs - {offset}')
 
-            entries.extend([
-                self.url_result(
-                    'http://music.163.com/#/program?id=%s' % program['id'],
-                    'NetEaseMusicProgram', program['id'])
-                for program in info['programs']
-            ])
-
-            if name is None:
-                radio = info['programs'][0]['radio']
-                name = radio['name']
-                desc = radio['desc']
+            entries.extend(self.url_result(
+                f'http://music.163.com/#/program?id={program["id"]}', NetEaseMusicProgramIE,
+                program['id'], program.get('name')) for program in info['programs'])
+            if not metainfo:
+                metainfo = traverse_obj(info, ('programs', 0, 'radio', {
+                    'title': ('name', {str}),
+                    'description': ('desc', {str}),
+                }))
 
             if not info['more']:
                 break
 
-        return self.playlist_result(entries, dj_id, name, desc)
+        return self.playlist_result(entries, dj_id, **metainfo)

From a9efb4b8d74f3583450ffda0ee57259a47d39c70 Mon Sep 17 00:00:00 2001
From: xofe <22776566+xofe@users.noreply.github.com>
Date: Fri, 6 Oct 2023 22:35:11 +0000
Subject: [PATCH 605/871] [ie/abc.net.au:iview] Improve `episode` extraction
 (#8201)

Authored by: xofe
---
 yt_dlp/extractor/abc.py | 90 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 88 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/abc.py b/yt_dlp/extractor/abc.py
index d2cf5f7c5..9d527246a 100644
--- a/yt_dlp/extractor/abc.py
+++ b/yt_dlp/extractor/abc.py
@@ -181,18 +181,102 @@ class ABCIViewIE(InfoExtractor):
     _GEO_COUNTRIES = ['AU']
 
     _TESTS = [{
+        'url': 'https://iview.abc.net.au/show/utopia/series/1/video/CO1211V001S00',
+        'md5': '52a942bfd7a0b79a6bfe9b4ce6c9d0ed',
+        'info_dict': {
+            'id': 'CO1211V001S00',
+            'ext': 'mp4',
+            'title': 'Series 1 Ep 1 Wood For The Trees',
+            'series': 'Utopia',
+            'description': 'md5:0cfb2c183c1b952d1548fd65c8a95c00',
+            'upload_date': '20230726',
+            'uploader_id': 'abc1',
+            'series_id': 'CO1211V',
+            'episode_id': 'CO1211V001S00',
+            'season_number': 1,
+            'season': 'Season 1',
+            'episode_number': 1,
+            'episode': 'Wood For The Trees',
+            'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/co/CO1211V001S00_5ad8353f4df09_1280.jpg',
+            'timestamp': 1690403700,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'note': 'No episode name',
         'url': 'https://iview.abc.net.au/show/gruen/series/11/video/LE1927H001S00',
         'md5': '67715ce3c78426b11ba167d875ac6abf',
         'info_dict': {
             'id': 'LE1927H001S00',
             'ext': 'mp4',
-            'title': "Series 11 Ep 1",
-            'series': "Gruen",
+            'title': 'Series 11 Ep 1',
+            'series': 'Gruen',
             'description': 'md5:52cc744ad35045baf6aded2ce7287f67',
             'upload_date': '20190925',
             'uploader_id': 'abc1',
+            'series_id': 'LE1927H',
+            'episode_id': 'LE1927H001S00',
+            'season_number': 11,
+            'season': 'Season 11',
+            'episode_number': 1,
+            'episode': 'Episode 1',
+            'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/le/LE1927H001S00_5d954fbd79e25_1280.jpg',
             'timestamp': 1569445289,
         },
+        'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'note': 'No episode number',
+        'url': 'https://iview.abc.net.au/show/four-corners/series/2022/video/NC2203H039S00',
+        'md5': '77cb7d8434440e3b28fbebe331c2456a',
+        'info_dict': {
+            'id': 'NC2203H039S00',
+            'ext': 'mp4',
+            'title': 'Series 2022 Locking Up Kids',
+            'series': 'Four Corners',
+            'description': 'md5:54829ca108846d1a70e1fcce2853e720',
+            'upload_date': '20221114',
+            'uploader_id': 'abc1',
+            'series_id': 'NC2203H',
+            'episode_id': 'NC2203H039S00',
+            'season_number': 2022,
+            'season': 'Season 2022',
+            'episode_number': None,
+            'episode': 'Locking Up Kids',
+            'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/nc/NC2203H039S00_636d8a0944a22_1920.jpg',
+            'timestamp': 1668460497,
+
+        },
+        'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'note': 'No episode name or number',
+        'url': 'https://iview.abc.net.au/show/landline/series/2021/video/RF2004Q043S00',
+        'md5': '2e17dec06b13cc81dc119d2565289396',
+        'info_dict': {
+            'id': 'RF2004Q043S00',
+            'ext': 'mp4',
+            'title': 'Series 2021',
+            'series': 'Landline',
+            'description': 'md5:c9f30d9c0c914a7fd23842f6240be014',
+            'upload_date': '20211205',
+            'uploader_id': 'abc1',
+            'series_id': 'RF2004Q',
+            'episode_id': 'RF2004Q043S00',
+            'season_number': 2021,
+            'season': 'Season 2021',
+            'episode_number': None,
+            'episode': None,
+            'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/rf/RF2004Q043S00_61a950639dbc0_1920.jpg',
+            'timestamp': 1638710705,
+
+        },
+        'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
         'params': {
             'skip_download': True,
         },
@@ -254,6 +338,8 @@ def tokenize_url(url, token):
             'episode_number': int_or_none(self._search_regex(
                 r'\bEp\s+(\d+)\b', title, 'episode number', default=None)),
             'episode_id': house_number,
+            'episode': self._search_regex(
+                r'^(?:Series\s+\d+)?\s*(?:Ep\s+\d+)?\s*(.*)$', title, 'episode', default='') or None,
             'uploader_id': video_params.get('channel'),
             'formats': formats,
             'subtitles': subtitles,

From 48cceec1ddb8649b5e771df8df79eb9c39c82b90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Droz?= <raphael.droz@gmail.com>
Date: Fri, 6 Oct 2023 19:38:26 -0300
Subject: [PATCH 606/871] [ie/lbry] Add playlist support (#8213)

Closes #5982, Closes #8204
Authored by: drzraf, bashonly, Grub4K
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/lbry.py        | 184 ++++++++++++++++++++------------
 2 files changed, 116 insertions(+), 69 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 908abb8ac..ef6123e8a 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -951,6 +951,7 @@
 from .lbry import (
     LBRYIE,
     LBRYChannelIE,
+    LBRYPlaylistIE,
 )
 from .lci import LCIIE
 from .lcp import (
diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py
index 9a9f9256f..ccce300b5 100644
--- a/yt_dlp/extractor/lbry.py
+++ b/yt_dlp/extractor/lbry.py
@@ -22,10 +22,11 @@
 
 
 class LBRYBaseIE(InfoExtractor):
-    _BASE_URL_REGEX = r'(?:https?://(?:www\.)?(?:lbry\.tv|odysee\.com)/|lbry://)'
+    _BASE_URL_REGEX = r'(?x)(?:https?://(?:www\.)?(?:lbry\.tv|odysee\.com)/|lbry://)'
     _CLAIM_ID_REGEX = r'[0-9a-f]{1,40}'
-    _OPT_CLAIM_ID = '[^:/?#&]+(?:[:#]%s)?' % _CLAIM_ID_REGEX
+    _OPT_CLAIM_ID = '[^$@:/?#&]+(?:[:#]%s)?' % _CLAIM_ID_REGEX
     _SUPPORTED_STREAM_TYPES = ['video', 'audio']
+    _PAGE_SIZE = 50
 
     def _call_api_proxy(self, method, display_id, params, resource):
         headers = {'Content-Type': 'application/json-rpc'}
@@ -77,10 +78,70 @@ def _parse_stream(self, stream, url):
 
         return info
 
+    def _fetch_page(self, display_id, url, params, page):
+        page += 1
+        page_params = {
+            'no_totals': True,
+            'page': page,
+            'page_size': self._PAGE_SIZE,
+            **params,
+        }
+        result = self._call_api_proxy(
+            'claim_search', display_id, page_params, f'page {page}')
+        for item in traverse_obj(result, ('items', lambda _, v: v['name'] and v['claim_id'])):
+            yield {
+                **self._parse_stream(item, url),
+                '_type': 'url',
+                'id': item['claim_id'],
+                'url': self._permanent_url(url, item['name'], item['claim_id']),
+            }
+
+    def _playlist_entries(self, url, display_id, claim_param, metadata):
+        qs = parse_qs(url)
+        content = qs.get('content', [None])[0]
+        params = {
+            'fee_amount': qs.get('fee_amount', ['>=0'])[0],
+            'order_by': {
+                'new': ['release_time'],
+                'top': ['effective_amount'],
+                'trending': ['trending_group', 'trending_mixed'],
+            }[qs.get('order', ['new'])[0]],
+            'claim_type': 'stream',
+            'stream_types': [content] if content in ['audio', 'video'] else self._SUPPORTED_STREAM_TYPES,
+            **claim_param,
+        }
+        duration = qs.get('duration', [None])[0]
+        if duration:
+            params['duration'] = {
+                'long': '>=1200',
+                'short': '<=240',
+            }[duration]
+        language = qs.get('language', ['all'])[0]
+        if language != 'all':
+            languages = [language]
+            if language == 'en':
+                languages.append('none')
+            params['any_languages'] = languages
+
+        entries = OnDemandPagedList(
+            functools.partial(self._fetch_page, display_id, url, params),
+            self._PAGE_SIZE)
+
+        return self.playlist_result(
+            entries, display_id, **traverse_obj(metadata, ('value', {
+                'title': 'title',
+                'description': 'description',
+            })))
+
 
 class LBRYIE(LBRYBaseIE):
     IE_NAME = 'lbry'
-    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + r'(?P<id>\$/[^/]+/[^/]+/{1}|@{0}/{0}|(?!@){0})'.format(LBRYBaseIE._OPT_CLAIM_ID, LBRYBaseIE._CLAIM_ID_REGEX)
+    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + rf'''
+        (?:\$/(?:download|embed)/)?
+        (?P<id>
+            [^$@:/?#]+/{LBRYBaseIE._CLAIM_ID_REGEX}
+            |(?:@{LBRYBaseIE._OPT_CLAIM_ID}/)?{LBRYBaseIE._OPT_CLAIM_ID}
+        )'''
     _TESTS = [{
         # Video
         'url': 'https://lbry.tv/@Mantega:1/First-day-LBRY:1',
@@ -149,7 +210,7 @@ class LBRYIE(LBRYBaseIE):
             'channel': 'Gardening In Canada',
             'channel_id': 'b8be0e93b423dad221abe29545fbe8ec36e806bc',
             'channel_url': 'https://odysee.com/@gardeningincanada:b8be0e93b423dad221abe29545fbe8ec36e806bc',
-            'formats': 'mincount:3',
+            'formats': 'mincount:3',  # FIXME
             'thumbnail': 'https://thumbnails.lbry.com/AgHSc_HzrrE',
             'license': 'Copyrighted (contact publisher)',
         }
@@ -184,12 +245,12 @@ class LBRYIE(LBRYBaseIE):
             'id': '41fbfe805eb73c8d3012c0c49faa0f563274f634',
             'ext': 'mp4',
             'title': 'Biotechnological Invasion of Skin (April 2023)',
-            'description': 'md5:709a2f4c07bd8891cda3a7cc2d6fcf5c',
+            'description': 'md5:fe28689db2cb7ba3436d819ac3ffc378',
             'channel': 'Wicked Truths',
             'channel_id': '23d2bbf856b0ceed5b1d7c5960bcc72da5a20cb0',
             'channel_url': 'https://odysee.com/@wickedtruths:23d2bbf856b0ceed5b1d7c5960bcc72da5a20cb0',
-            'timestamp': 1685790036,
-            'upload_date': '20230603',
+            'timestamp': 1695114347,
+            'upload_date': '20230919',
             'release_timestamp': 1685617473,
             'release_date': '20230601',
             'duration': 1063,
@@ -229,10 +290,10 @@ class LBRYIE(LBRYBaseIE):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        if display_id.startswith('$/'):
-            display_id = display_id.split('/', 2)[-1].replace('/', ':')
-        else:
+        if display_id.startswith('@'):
             display_id = display_id.replace(':', '#')
+        else:
+            display_id = display_id.replace('/', ':')
         display_id = urllib.parse.unquote(display_id)
         uri = 'lbry://' + display_id
         result = self._resolve_url(uri, display_id, 'stream')
@@ -299,7 +360,7 @@ def _real_extract(self, url):
 
 class LBRYChannelIE(LBRYBaseIE):
     IE_NAME = 'lbry:channel'
-    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + r'(?P<id>@%s)/?(?:[?&]|$)' % LBRYBaseIE._OPT_CLAIM_ID
+    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + rf'(?P<id>@{LBRYBaseIE._OPT_CLAIM_ID})/?(?:[?&]|$)'
     _TESTS = [{
         'url': 'https://lbry.tv/@LBRYFoundation:0',
         'info_dict': {
@@ -315,65 +376,50 @@ class LBRYChannelIE(LBRYBaseIE):
         'url': 'lbry://@lbry#3f',
         'only_matching': True,
     }]
-    _PAGE_SIZE = 50
-
-    def _fetch_page(self, claim_id, url, params, page):
-        page += 1
-        page_params = {
-            'channel_ids': [claim_id],
-            'claim_type': 'stream',
-            'no_totals': True,
-            'page': page,
-            'page_size': self._PAGE_SIZE,
-        }
-        page_params.update(params)
-        result = self._call_api_proxy(
-            'claim_search', claim_id, page_params, 'page %d' % page)
-        for item in (result.get('items') or []):
-            stream_claim_name = item.get('name')
-            stream_claim_id = item.get('claim_id')
-            if not (stream_claim_name and stream_claim_id):
-                continue
-
-            yield {
-                **self._parse_stream(item, url),
-                '_type': 'url',
-                'id': stream_claim_id,
-                'url': self._permanent_url(url, stream_claim_name, stream_claim_id),
-            }
 
     def _real_extract(self, url):
         display_id = self._match_id(url).replace(':', '#')
-        result = self._resolve_url(
-            'lbry://' + display_id, display_id, 'channel')
+        result = self._resolve_url(f'lbry://{display_id}', display_id, 'channel')
         claim_id = result['claim_id']
-        qs = parse_qs(url)
-        content = qs.get('content', [None])[0]
-        params = {
-            'fee_amount': qs.get('fee_amount', ['>=0'])[0],
-            'order_by': {
-                'new': ['release_time'],
-                'top': ['effective_amount'],
-                'trending': ['trending_group', 'trending_mixed'],
-            }[qs.get('order', ['new'])[0]],
-            'stream_types': [content] if content in ['audio', 'video'] else self._SUPPORTED_STREAM_TYPES,
-        }
-        duration = qs.get('duration', [None])[0]
-        if duration:
-            params['duration'] = {
-                'long': '>=1200',
-                'short': '<=240',
-            }[duration]
-        language = qs.get('language', ['all'])[0]
-        if language != 'all':
-            languages = [language]
-            if language == 'en':
-                languages.append('none')
-            params['any_languages'] = languages
-        entries = OnDemandPagedList(
-            functools.partial(self._fetch_page, claim_id, url, params),
-            self._PAGE_SIZE)
-        result_value = result.get('value') or {}
-        return self.playlist_result(
-            entries, claim_id, result_value.get('title'),
-            result_value.get('description'))
+
+        return self._playlist_entries(url, claim_id, {'channel_ids': [claim_id]}, result)
+
+
+class LBRYPlaylistIE(LBRYBaseIE):
+    IE_NAME = 'lbry:playlist'
+    _VALID_URL = LBRYBaseIE._BASE_URL_REGEX + r'\$/(?:play)?list/(?P<id>[0-9a-f-]+)'
+    _TESTS = [{
+        'url': 'https://odysee.com/$/playlist/ffef782f27486f0ac138bde8777f72ebdd0548c2',
+        'info_dict': {
+            'id': 'ffef782f27486f0ac138bde8777f72ebdd0548c2',
+            'title': 'Théâtre Classique',
+            'description': 'Théâtre Classique',
+        },
+        'playlist_mincount': 4,
+    }, {
+        'url': 'https://odysee.com/$/list/9c6658b3dd21e4f2a0602d523a13150e2b48b770',
+        'info_dict': {
+            'id': '9c6658b3dd21e4f2a0602d523a13150e2b48b770',
+            'title': 'Social Media Exposed',
+            'description': 'md5:98af97317aacd5b85d595775ea37d80e',
+        },
+        'playlist_mincount': 34,
+    }, {
+        'url': 'https://odysee.com/$/playlist/938fb11d-215f-4d1c-ad64-723954df2184',
+        'info_dict': {
+            'id': '938fb11d-215f-4d1c-ad64-723954df2184',
+        },
+        'playlist_mincount': 1000,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        result = traverse_obj(self._call_api_proxy('claim_search', display_id, {
+            'claim_ids': [display_id],
+            'no_totals': True,
+            'page': 1,
+            'page_size': self._PAGE_SIZE,
+        }, 'playlist'), ('items', 0))
+        claim_param = {'claim_ids': traverse_obj(result, ('value', 'claims', ..., {str}))}
+
+        return self._playlist_entries(url, display_id, claim_param, result)

From fbcc299bd8a19cf8b3c8805d6c268a9110230973 Mon Sep 17 00:00:00 2001
From: Umar Getagazov <umar@handlerug.me>
Date: Sat, 7 Oct 2023 01:45:46 +0300
Subject: [PATCH 607/871] [ie/substack] Fix embed extraction (#8218)

Authored by: handlerug
---
 yt_dlp/extractor/substack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/substack.py b/yt_dlp/extractor/substack.py
index 3782ceed1..5835a5a8d 100644
--- a/yt_dlp/extractor/substack.py
+++ b/yt_dlp/extractor/substack.py
@@ -50,7 +50,7 @@ def _extract_embed_urls(cls, url, webpage):
         if not re.search(r'<script[^>]+src=["\']https://substackcdn.com/[^"\']+\.js', webpage):
             return
 
-        mobj = re.search(r'{[^}]*["\']subdomain["\']\s*:\s*["\'](?P<subdomain>[^"]+)', webpage)
+        mobj = re.search(r'{[^}]*\\?["\']subdomain\\?["\']\s*:\s*\\?["\'](?P<subdomain>[^\\"\']+)', webpage)
         if mobj:
             parsed = urllib.parse.urlparse(url)
             yield parsed._replace(netloc=f'{mobj.group("subdomain")}.substack.com').geturl()

From 2f2dda3a7e85148773da3cdbc03ac9949ec1bc45 Mon Sep 17 00:00:00 2001
From: Umar Getagazov <umar@handlerug.me>
Date: Sat, 7 Oct 2023 01:48:54 +0300
Subject: [PATCH 608/871] [ie/substack] Fix download cookies bug (#8219)

Authored by: handlerug
---
 yt_dlp/extractor/substack.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/substack.py b/yt_dlp/extractor/substack.py
index 5835a5a8d..6ee3f75e1 100644
--- a/yt_dlp/extractor/substack.py
+++ b/yt_dlp/extractor/substack.py
@@ -56,10 +56,10 @@ def _extract_embed_urls(cls, url, webpage):
             yield parsed._replace(netloc=f'{mobj.group("subdomain")}.substack.com').geturl()
             raise cls.StopExtraction()
 
-    def _extract_video_formats(self, video_id, username):
+    def _extract_video_formats(self, video_id, url):
         formats, subtitles = [], {}
         for video_format in ('hls', 'mp4'):
-            video_url = f'https://{username}.substack.com/api/v1/video/upload/{video_id}/src?type={video_format}'
+            video_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}/src?type={video_format}')
 
             if video_format == 'hls':
                 fmts, subs = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', fatal=False)
@@ -81,12 +81,17 @@ def _real_extract(self, url):
             r'window\._preloads\s*=\s*JSON\.parse\(', webpage, 'json string',
             display_id, transform_source=js_to_json, contains_pattern=r'"{(?s:.+)}"'), display_id)
 
+        canonical_url = url
+        domain = traverse_obj(webpage_info, ('domainInfo', 'customDomain', {str}))
+        if domain:
+            canonical_url = urllib.parse.urlparse(url)._replace(netloc=domain).geturl()
+
         post_type = webpage_info['post']['type']
         formats, subtitles = [], {}
         if post_type == 'podcast':
             formats, subtitles = [{'url': webpage_info['post']['podcast_url']}], {}
         elif post_type == 'video':
-            formats, subtitles = self._extract_video_formats(webpage_info['post']['videoUpload']['id'], username)
+            formats, subtitles = self._extract_video_formats(webpage_info['post']['videoUpload']['id'], canonical_url)
         else:
             self.raise_no_formats(f'Page type "{post_type}" is not supported')
 
@@ -99,4 +104,5 @@ def _real_extract(self, url):
             'thumbnail': traverse_obj(webpage_info, ('post', 'cover_image')),
             'uploader': traverse_obj(webpage_info, ('pub', 'name')),
             'uploader_id': str_or_none(traverse_obj(webpage_info, ('post', 'publication_id'))),
+            'webpage_url': canonical_url,
         }

From 2ad3873f0dfa9285c91d2160e36c039e69d597c7 Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Fri, 6 Oct 2023 23:53:11 +0100
Subject: [PATCH 609/871] [ie/radiko] Improve extraction (#8221)

Authored by: garret1317
---
 yt_dlp/extractor/radiko.py | 67 ++++++++++++++++++++++----------------
 1 file changed, 39 insertions(+), 28 deletions(-)

diff --git a/yt_dlp/extractor/radiko.py b/yt_dlp/extractor/radiko.py
index cef68eba0..8c8fb1a8f 100644
--- a/yt_dlp/extractor/radiko.py
+++ b/yt_dlp/extractor/radiko.py
@@ -1,4 +1,5 @@
 import base64
+import random
 import urllib.parse
 
 from .common import InfoExtractor
@@ -13,6 +14,7 @@
 
 
 class RadikoBaseIE(InfoExtractor):
+    _GEO_BYPASS = False
     _FULL_KEY = None
     _HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED = (
         'https://c-rpaa.smartstream.ne.jp',
@@ -32,7 +34,7 @@ class RadikoBaseIE(InfoExtractor):
         'https://c-radiko.smartstream.ne.jp',
     )
 
-    def _auth_client(self):
+    def _negotiate_token(self):
         _, auth1_handle = self._download_webpage_handle(
             'https://radiko.jp/v2/api/auth1', None, 'Downloading authentication page',
             headers={
@@ -58,10 +60,23 @@ def _auth_client(self):
                 'x-radiko-partialkey': partial_key,
             }).split(',')[0]
 
+        if area_id == 'OUT':
+            self.raise_geo_restricted(countries=['JP'])
+
         auth_data = (auth_token, area_id)
         self.cache.store('radiko', 'auth_data', auth_data)
         return auth_data
 
+    def _auth_client(self):
+        cachedata = self.cache.load('radiko', 'auth_data')
+        if cachedata is not None:
+            response = self._download_webpage(
+                'https://radiko.jp/v2/api/auth_check', None, 'Checking cached token', expected_status=401,
+                headers={'X-Radiko-AuthToken': cachedata[0], 'X-Radiko-AreaId': cachedata[1]})
+            if response == 'OK':
+                return cachedata
+        return self._negotiate_token()
+
     def _extract_full_key(self):
         if self._FULL_KEY:
             return self._FULL_KEY
@@ -75,7 +90,7 @@ def _extract_full_key(self):
 
         if full_key:
             full_key = full_key.encode()
-        else:  # use full key ever known
+        else:  # use only full key ever known
             full_key = b'bcd151073c03b352e1ef2fd66c32209da9ca0afa'
 
         self._FULL_KEY = full_key
@@ -103,24 +118,24 @@ def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token,
         m3u8_playlist_data = self._download_xml(
             f'https://radiko.jp/v3/station/stream/pc_html5/{station}.xml', video_id,
             note='Downloading stream information')
-        m3u8_urls = m3u8_playlist_data.findall('.//url')
 
         formats = []
         found = set()
-        for url_tag in m3u8_urls:
-            pcu = url_tag.find('playlist_create_url').text
-            url_attrib = url_tag.attrib
+
+        timefree_int = 0 if is_onair else 1
+
+        for element in m3u8_playlist_data.findall(f'.//url[@timefree="{timefree_int}"]/playlist_create_url'):
+            pcu = element.text
+            if pcu in found:
+                continue
+            found.add(pcu)
             playlist_url = update_url_query(pcu, {
                 'station_id': station,
                 **query,
                 'l': '15',
-                'lsid': '88ecea37e968c1f17d5413312d9f8003',
+                'lsid': ''.join(random.choices('0123456789abcdef', k=32)),
                 'type': 'b',
             })
-            if playlist_url in found:
-                continue
-            else:
-                found.add(playlist_url)
 
             time_to_skip = None if is_onair else cursor - ft
 
@@ -138,7 +153,7 @@ def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token,
                         not is_onair and pcu.startswith(self._HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED)):
                     sf['preference'] = -100
                     sf['format_note'] = 'not preferred'
-                if not is_onair and url_attrib['timefree'] == '1' and time_to_skip:
+                if not is_onair and timefree_int == 1 and time_to_skip:
                     sf['downloader_options'] = {'ffmpeg_args': ['-ss', time_to_skip]}
             formats.extend(subformats)
 
@@ -166,21 +181,7 @@ def _real_extract(self, url):
         vid_int = unified_timestamp(video_id, False)
         prog, station_program, ft, radio_begin, radio_end = self._find_program(video_id, station, vid_int)
 
-        auth_cache = self.cache.load('radiko', 'auth_data')
-        for attempt in range(2):
-            auth_token, area_id = (not attempt and auth_cache) or self._auth_client()
-            formats = self._extract_formats(
-                video_id=video_id, station=station, is_onair=False,
-                ft=ft, cursor=vid_int, auth_token=auth_token, area_id=area_id,
-                query={
-                    'start_at': radio_begin,
-                    'ft': radio_begin,
-                    'end_at': radio_end,
-                    'to': radio_end,
-                    'seek': video_id,
-                })
-            if formats:
-                break
+        auth_token, area_id = self._auth_client()
 
         return {
             'id': video_id,
@@ -189,8 +190,18 @@ def _real_extract(self, url):
             'uploader': try_call(lambda: station_program.find('.//name').text),
             'uploader_id': station,
             'timestamp': vid_int,
-            'formats': formats,
             'is_live': True,
+            'formats': self._extract_formats(
+                video_id=video_id, station=station, is_onair=False,
+                ft=ft, cursor=vid_int, auth_token=auth_token, area_id=area_id,
+                query={
+                    'start_at': radio_begin,
+                    'ft': radio_begin,
+                    'end_at': radio_end,
+                    'to': radio_end,
+                    'seek': video_id
+                }
+            ),
         }
 
 

From 35d9cbaf9638ccc9daf8a863063b2e7c135bc664 Mon Sep 17 00:00:00 2001
From: AS6939 <46506352+AS6939@users.noreply.github.com>
Date: Sat, 7 Oct 2023 06:56:12 +0800
Subject: [PATCH 610/871] [ie/iq.com] Fix extraction and subtitles (#8260)

Closes #7734, Closes #8123
Authored by: AS6939
---
 yt_dlp/extractor/iqiyi.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/iqiyi.py b/yt_dlp/extractor/iqiyi.py
index fa602ba88..3368ab1d9 100644
--- a/yt_dlp/extractor/iqiyi.py
+++ b/yt_dlp/extractor/iqiyi.py
@@ -499,9 +499,10 @@ class IqIE(InfoExtractor):
                     'tm': tm,
                     'qdy': 'a',
                     'qds': 0,
-                    'k_ft1': 141287244169348,
-                    'k_ft4': 34359746564,
-                    'k_ft5': 1,
+                    'k_ft1': '143486267424900',
+                    'k_ft4': '1572868',
+                    'k_ft7': '4',
+                    'k_ft5': '1',
                     'bop': JSON.stringify({
                         'version': '10.0',
                         'dfp': dfp
@@ -529,14 +530,22 @@ def _extract_vms_player_js(self, webpage, video_id):
         webpack_js_url = self._proto_relative_url(self._search_regex(
             r'<script src="((?:https?:)?//stc\.iqiyipic\.com/_next/static/chunks/webpack-\w+\.js)"', webpage, 'webpack URL'))
         webpack_js = self._download_webpage(webpack_js_url, video_id, note='Downloading webpack JS', errnote='Unable to download webpack JS')
+
         webpack_map = self._search_json(
             r'["\']\s*\+\s*', webpack_js, 'JS locations', video_id,
             contains_pattern=r'{\s*(?:\d+\s*:\s*["\'][\da-f]+["\']\s*,?\s*)+}',
             end_pattern=r'\[\w+\]\+["\']\.js', transform_source=js_to_json)
 
+        replacement_map = self._search_json(
+            r'["\']\s*\+\(\s*', webpack_js, 'replacement map', video_id,
+            contains_pattern=r'{\s*(?:\d+\s*:\s*["\'][\w.-]+["\']\s*,?\s*)+}',
+            end_pattern=r'\[\w+\]\|\|\w+\)\+["\']\.', transform_source=js_to_json,
+            fatal=False) or {}
+
         for module_index in reversed(webpack_map):
+            real_module = replacement_map.get(module_index) or module_index
             module_js = self._download_webpage(
-                f'https://stc.iqiyipic.com/_next/static/chunks/{module_index}.{webpack_map[module_index]}.js',
+                f'https://stc.iqiyipic.com/_next/static/chunks/{real_module}.{webpack_map[module_index]}.js',
                 video_id, note=f'Downloading #{module_index} module JS', errnote='Unable to download module JS', fatal=False) or ''
             if 'vms request' in module_js:
                 self.cache.store('iq', 'player_js', module_js)

From 47c598783c98c179e04dd12c2a3fee0f3dc53087 Mon Sep 17 00:00:00 2001
From: Esme <madewokherd@gmail.com>
Date: Fri, 6 Oct 2023 17:58:28 -0500
Subject: [PATCH 611/871] [ie/erocast] Add extractor (#8264)

Closes #4001
Authored by: madewokherd
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/erocast.py     | 63 +++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 yt_dlp/extractor/erocast.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ef6123e8a..b10ef2f33 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -574,6 +574,7 @@
 from .eplus import EplusIbIE
 from .epoch import EpochIE
 from .eporner import EpornerIE
+from .erocast import ErocastIE
 from .eroprofile import (
     EroProfileIE,
     EroProfileAlbumIE,
diff --git a/yt_dlp/extractor/erocast.py b/yt_dlp/extractor/erocast.py
new file mode 100644
index 000000000..92a57536c
--- /dev/null
+++ b/yt_dlp/extractor/erocast.py
@@ -0,0 +1,63 @@
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    str_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class ErocastIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?erocast\.me/track/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://erocast.me/track/9787/f',
+        'md5': 'af63b91f5f231096aba54dd682abea3b',
+        'info_dict': {
+            'id': '9787',
+            'title': '[F4M] Your roommate, who is definitely not possessed by an alien, suddenly wants to fuck you',
+            'url': 'https://erocast.s3.us-east-2.wasabisys.com/1220419/track.m3u8',
+            'ext': 'm4a',
+            'age_limit': 18,
+            'release_timestamp': 1696178652,
+            'release_date': '20231001',
+            'modified_timestamp': int,
+            'modified_date': str,
+            'description': 'ExtraTerrestrial Tuesday!',
+            'uploader': 'clarissaisshy',
+            'uploader_id': '8113',
+            'uploader_url': 'https://erocast.me/clarissaisshy',
+            'thumbnail': 'https://erocast.s3.us-east-2.wasabisys.com/1220418/conversions/1696179247-lg.jpg',
+            'duration': 2307,
+            'view_count': int,
+            'comment_count': int,
+            'webpage_url': 'https://erocast.me/track/9787/f4m-your-roommate-who-is-definitely-not-possessed-by-an-alien-suddenly-wants-to-fuck-you',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        data = self._search_json(
+            rf'<script>\s*var song_data_{video_id}\s*=', webpage, 'data', video_id, end_pattern=r'</script>')
+
+        return {
+            'id': video_id,
+            'formats': self._extract_m3u8_formats(
+                data.get('file_url') or data['stream_url'], video_id, 'm4a', m3u8_id='hls'),
+            'age_limit': 18,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'release_timestamp': ('created_at', {parse_iso8601}),
+                'modified_timestamp': ('updated_at', {parse_iso8601}),
+                'uploader': ('user', 'name', {str}),
+                'uploader_id': ('user', 'id', {str_or_none}),
+                'uploader_url': ('user', 'permalink_url', {url_or_none}),
+                'thumbnail': ('artwork_url', {url_or_none}),
+                'duration': ('duration', {int_or_none}),
+                'view_count': ('plays', {int_or_none}),
+                'comment_count': ('comment_count', {int_or_none}),
+                'webpage_url': ('permalink_url', {url_or_none}),
+            }),
+        }

From 0e722f2f3ca42e634fd7b06ee70b16bf833ce132 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Droz?= <raphael.droz@gmail.com>
Date: Fri, 6 Oct 2023 19:59:42 -0300
Subject: [PATCH 612/871] [ie/lbry] Extract `uploader_id` (#8244)

Closes #123
Authored by: drzraf
---
 yt_dlp/extractor/lbry.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py
index ccce300b5..cc37c41e8 100644
--- a/yt_dlp/extractor/lbry.py
+++ b/yt_dlp/extractor/lbry.py
@@ -70,11 +70,11 @@ def _parse_stream(self, stream, url):
             'duration': ('value', stream_type, 'duration', {int_or_none}),
             'channel': ('signing_channel', 'value', 'title', {str}),
             'channel_id': ('signing_channel', 'claim_id', {str}),
+            'uploader_id': ('signing_channel', 'name', {str}),
         })
 
-        channel_name = traverse_obj(stream, ('signing_channel', 'name', {str}))
-        if channel_name and info.get('channel_id'):
-            info['channel_url'] = self._permanent_url(url, channel_name, info['channel_id'])
+        if info.get('uploader_id') and info.get('channel_id'):
+            info['channel_url'] = self._permanent_url(url, info['uploader_id'], info['channel_id'])
 
         return info
 
@@ -159,6 +159,7 @@ class LBRYIE(LBRYBaseIE):
             'height': 720,
             'thumbnail': 'https://spee.ch/7/67f2d809c263288c.png',
             'license': 'None',
+            'uploader_id': '@Mantega',
             'duration': 346,
             'channel': 'LBRY/Odysee rats united!!!',
             'channel_id': '1c8ad6a2ab4e889a71146ae4deeb23bb92dab627',
@@ -192,6 +193,7 @@ class LBRYIE(LBRYBaseIE):
             'vcodec': 'none',
             'thumbnail': 'https://spee.ch/d/0bc63b0e6bf1492d.png',
             'license': 'None',
+            'uploader_id': '@LBRYFoundation',
         }
     }, {
         'url': 'https://odysee.com/@gardeningincanada:b/plants-i-will-never-grow-again.-the:e',
@@ -210,7 +212,8 @@ class LBRYIE(LBRYBaseIE):
             'channel': 'Gardening In Canada',
             'channel_id': 'b8be0e93b423dad221abe29545fbe8ec36e806bc',
             'channel_url': 'https://odysee.com/@gardeningincanada:b8be0e93b423dad221abe29545fbe8ec36e806bc',
-            'formats': 'mincount:3',  # FIXME
+            'uploader_id': '@gardeningincanada',
+            'formats': 'mincount:3',
             'thumbnail': 'https://thumbnails.lbry.com/AgHSc_HzrrE',
             'license': 'Copyrighted (contact publisher)',
         }
@@ -235,6 +238,7 @@ class LBRYIE(LBRYBaseIE):
             'formats': 'mincount:1',
             'thumbnail': 'startswith:https://thumb',
             'license': 'None',
+            'uploader_id': '@RT',
         },
         'params': {'skip_download': True}
     }, {
@@ -249,6 +253,7 @@ class LBRYIE(LBRYBaseIE):
             'channel': 'Wicked Truths',
             'channel_id': '23d2bbf856b0ceed5b1d7c5960bcc72da5a20cb0',
             'channel_url': 'https://odysee.com/@wickedtruths:23d2bbf856b0ceed5b1d7c5960bcc72da5a20cb0',
+            'uploader_id': '@wickedtruths',
             'timestamp': 1695114347,
             'upload_date': '20230919',
             'release_timestamp': 1685617473,

From e831c80e8b2fc025b3b67d82974cc59e3526fdc8 Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Sat, 7 Oct 2023 00:05:48 +0100
Subject: [PATCH 613/871] [ie/nhk] Fix VOD extraction (#8249)

Closes #8242
Authored by: garret1317
---
 yt_dlp/extractor/nhk.py | 46 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 43 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py
index fbd6a18f6..bcbc2279f 100644
--- a/yt_dlp/extractor/nhk.py
+++ b/yt_dlp/extractor/nhk.py
@@ -28,6 +28,44 @@ def _call_api(self, m_id, lang, is_video, is_episode, is_clip):
                 m_id, lang, '/all' if is_video else ''),
             m_id, query={'apikey': 'EJfK8jdS57GqlupFgAfAAwr573q01y6k'})['data']['episodes'] or []
 
+    def _get_api_info(self, refresh=True):
+        if not refresh:
+            return self.cache.load('nhk', 'api_info')
+
+        self.cache.store('nhk', 'api_info', {})
+        movie_player_js = self._download_webpage(
+            'https://movie-a.nhk.or.jp/world/player/js/movie-player.js', None,
+            note='Downloading stream API information')
+        api_info = {
+            'url': self._search_regex(
+                r'prod:[^;]+\bapiUrl:\s*[\'"]([^\'"]+)[\'"]', movie_player_js, None, 'stream API url'),
+            'token': self._search_regex(
+                r'prod:[^;]+\btoken:\s*[\'"]([^\'"]+)[\'"]', movie_player_js, None, 'stream API token'),
+        }
+        self.cache.store('nhk', 'api_info', api_info)
+        return api_info
+
+    def _extract_formats_and_subtitles(self, vod_id):
+        for refresh in (False, True):
+            api_info = self._get_api_info(refresh)
+            if not api_info:
+                continue
+
+            api_url = api_info.pop('url')
+            stream_url = traverse_obj(
+                self._download_json(
+                    api_url, vod_id, 'Downloading stream url info', fatal=False, query={
+                        **api_info,
+                        'type': 'json',
+                        'optional_id': vod_id,
+                        'active_flg': 1,
+                    }),
+                ('meta', 0, 'movie_url', ('mb_auto', 'auto_sp', 'auto_pc'), {url_or_none}), get_all=False)
+            if stream_url:
+                return self._extract_m3u8_formats_and_subtitles(stream_url, vod_id)
+
+        raise ExtractorError('Unable to extract stream url')
+
     def _extract_episode_info(self, url, episode=None):
         fetch_episode = episode is None
         lang, m_type, episode_id = NhkVodIE._match_valid_url(url).groups()
@@ -67,12 +105,14 @@ def get_clean_field(key):
         }
         if is_video:
             vod_id = episode['vod_id']
+            formats, subs = self._extract_formats_and_subtitles(vod_id)
+
             info.update({
-                '_type': 'url_transparent',
-                'ie_key': 'Piksel',
-                'url': 'https://movie-s.nhk.or.jp/v/refid/nhkworld/prefid/' + vod_id,
                 'id': vod_id,
+                'formats': formats,
+                'subtitles': subs,
             })
+
         else:
             if fetch_episode:
                 audio_path = episode['audio']['audio']

From 19c90e405b4137c06dfe6f9aaa02396df0da93e5 Mon Sep 17 00:00:00 2001
From: trainman261 <trainman261@users.noreply.github.com>
Date: Sat, 7 Oct 2023 01:56:19 +0200
Subject: [PATCH 614/871] [cleanup] Update extractor tests (#7718)

Authored by: trainman261
---
 yt_dlp/extractor/aenetworks.py         |  1 +
 yt_dlp/extractor/amcnetworks.py        |  1 +
 yt_dlp/extractor/cbc.py                |  7 ++++++-
 yt_dlp/extractor/cbs.py                |  2 ++
 yt_dlp/extractor/cnbc.py               |  2 ++
 yt_dlp/extractor/corus.py              |  3 ++-
 yt_dlp/extractor/generic.py            | 13 ++++++++++---
 yt_dlp/extractor/mediaset.py           |  3 ++-
 yt_dlp/extractor/movieclips.py         |  1 +
 yt_dlp/extractor/nationalgeographic.py |  3 +++
 yt_dlp/extractor/nbc.py                | 22 +++++++++++++++++-----
 yt_dlp/extractor/scrippsnetworks.py    |  4 ++++
 yt_dlp/extractor/syfy.py               |  1 +
 yt_dlp/extractor/theplatform.py        |  6 +++---
 yt_dlp/extractor/theweatherchannel.py  | 20 +++++++++++---------
 15 files changed, 66 insertions(+), 23 deletions(-)

diff --git a/yt_dlp/extractor/aenetworks.py b/yt_dlp/extractor/aenetworks.py
index f049a0fb3..cc26653c1 100644
--- a/yt_dlp/extractor/aenetworks.py
+++ b/yt_dlp/extractor/aenetworks.py
@@ -338,6 +338,7 @@ class BiographyIE(AENetworksBaseIE):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+        'skip': '404 Not Found',
     }]
 
     def _real_extract(self, url):
diff --git a/yt_dlp/extractor/amcnetworks.py b/yt_dlp/extractor/amcnetworks.py
index c58bc7bfb..10bd021c5 100644
--- a/yt_dlp/extractor/amcnetworks.py
+++ b/yt_dlp/extractor/amcnetworks.py
@@ -26,6 +26,7 @@ class AMCNetworksIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
             # m3u8 download
             'skip_download': True,
         },
+        'skip': '404 Not Found',
     }, {
         'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge',
         'only_matching': True,
diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index 2920b9027..be2d13e44 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -66,6 +66,7 @@ class CBCIE(InfoExtractor):
             'uploader': 'CBCC-NEW',
             'timestamp': 255977160,
         },
+        'skip': '404 Not Found',
     }, {
         # multiple iframes
         'url': 'http://www.cbc.ca/natureofthings/blog/birds-eye-view-from-vancouvers-burrard-street-bridge-how-we-got-the-shot',
@@ -97,7 +98,7 @@ class CBCIE(InfoExtractor):
         # multiple CBC.APP.Caffeine.initInstance(...)
         'url': 'http://www.cbc.ca/news/canada/calgary/dog-indoor-exercise-winter-1.3928238',
         'info_dict': {
-            'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',
+            'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',  # FIXME
             'id': 'dog-indoor-exercise-winter-1.3928238',
             'description': 'md5:c18552e41726ee95bd75210d1ca9194c',
         },
@@ -476,6 +477,10 @@ class CBCGemPlaylistIE(InfoExtractor):
             'id': 'schitts-creek/s06',
             'title': 'Season 6',
             'description': 'md5:6a92104a56cbeb5818cc47884d4326a2',
+            'series': 'Schitt\'s Creek',
+            'season_number': 6,
+            'season': 'Season 6',
+            'thumbnail': 'https://images.radio-canada.ca/v1/synps-cbc/season/perso/cbc_schitts_creek_season_06_carousel_v03.jpg?impolicy=ott&im=Resize=(_Size_)&quality=75',
         },
     }, {
         'url': 'https://gem.cbc.ca/schitts-creek/s06',
diff --git a/yt_dlp/extractor/cbs.py b/yt_dlp/extractor/cbs.py
index 1c0dbdea9..d97fbd758 100644
--- a/yt_dlp/extractor/cbs.py
+++ b/yt_dlp/extractor/cbs.py
@@ -101,6 +101,7 @@ class CBSIE(CBSBaseIE):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'Subscription required',
     }, {
         'url': 'https://www.cbs.com/shows/video/sZH1MGgomIosZgxGJ1l263MFq16oMtW1/',
         'info_dict': {
@@ -117,6 +118,7 @@ class CBSIE(CBSBaseIE):
         },
         'expected_warnings': [
             'This content expired on', 'No video formats found', 'Requested format is not available'],
+        'skip': '404 Not Found',
     }, {
         'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
         'only_matching': True,
diff --git a/yt_dlp/extractor/cnbc.py b/yt_dlp/extractor/cnbc.py
index 68fd025b7..7d209b6d9 100644
--- a/yt_dlp/extractor/cnbc.py
+++ b/yt_dlp/extractor/cnbc.py
@@ -19,6 +19,7 @@ class CNBCIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'Dead link',
     }
 
     def _real_extract(self, url):
@@ -49,6 +50,7 @@ class CNBCVideoIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'Dead link',
     }
 
     def _real_extract(self, url):
diff --git a/yt_dlp/extractor/corus.py b/yt_dlp/extractor/corus.py
index c03d65310..bcc34ddd8 100644
--- a/yt_dlp/extractor/corus.py
+++ b/yt_dlp/extractor/corus.py
@@ -41,7 +41,7 @@ class CorusIE(ThePlatformFeedIE):  # XXX: Do not subclass from concrete IE
                         )
                     '''
     _TESTS = [{
-        'url': 'http://www.hgtv.ca/shows/bryan-inc/videos/movie-night-popcorn-with-bryan-870923331648/',
+        'url': 'https://www.hgtv.ca/video/bryan-inc/movie-night-popcorn-with-bryan/870923331648/',
         'info_dict': {
             'id': '870923331648',
             'ext': 'mp4',
@@ -54,6 +54,7 @@ class CorusIE(ThePlatformFeedIE):  # XXX: Do not subclass from concrete IE
             'skip_download': True,
         },
         'expected_warnings': ['Failed to parse JSON'],
+        # FIXME: yt-dlp wrongly raises for geo restriction
     }, {
         'url': 'http://www.foodnetwork.ca/shows/chopped/video/episode/chocolate-obsession/video.html?v=872683587753',
         'only_matching': True,
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 33e71d1c5..5e1240c13 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -58,6 +58,8 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'trailer',
                 'upload_date': '20100513',
+                'direct': True,
+                'timestamp': 1273772943.0,
             }
         },
         # Direct link to media delivered compressed (until Accept-Encoding is *)
@@ -101,6 +103,8 @@ class GenericIE(InfoExtractor):
                 'ext': 'webm',
                 'title': '5_Lennart_Poettering_-_Systemd',
                 'upload_date': '20141120',
+                'direct': True,
+                'timestamp': 1416498816.0,
             },
             'expected_warnings': [
                 'URL could be a direct video link, returning it as such.'
@@ -133,6 +137,7 @@ class GenericIE(InfoExtractor):
                     'upload_date': '20201204',
                 },
             }],
+            'skip': 'Dead link',
         },
         # RSS feed with item with description and thumbnails
         {
@@ -145,12 +150,12 @@ class GenericIE(InfoExtractor):
             'playlist': [{
                 'info_dict': {
                     'ext': 'm4a',
-                    'id': 'c1c879525ce2cb640b344507e682c36d',
+                    'id': '818a5d38-01cd-152f-2231-ee479677fa82',
                     'title': 're:Hydrogen!',
                     'description': 're:.*In this episode we are going.*',
                     'timestamp': 1567977776,
                     'upload_date': '20190908',
-                    'duration': 459,
+                    'duration': 423,
                     'thumbnail': r're:^https?://.*\.jpg$',
                     'episode_number': 1,
                     'season_number': 1,
@@ -267,6 +272,7 @@ class GenericIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': '404 Not Found',
         },
         # MPD from http://dash-mse-test.appspot.com/media.html
         {
@@ -278,6 +284,7 @@ class GenericIE(InfoExtractor):
                 'title': 'car-20120827-manifest',
                 'formats': 'mincount:9',
                 'upload_date': '20130904',
+                'timestamp': 1378272859.0,
             },
         },
         # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
@@ -318,7 +325,7 @@ class GenericIE(InfoExtractor):
                 'id': 'cmQHVoWB5FY',
                 'ext': 'mp4',
                 'upload_date': '20130224',
-                'uploader_id': 'TheVerge',
+                'uploader_id': '@TheVerge',
                 'description': r're:^Chris Ziegler takes a look at the\.*',
                 'uploader': 'The Verge',
                 'title': 'First Firefox OS phones side-by-side',
diff --git a/yt_dlp/extractor/mediaset.py b/yt_dlp/extractor/mediaset.py
index e3b728dca..2d6204298 100644
--- a/yt_dlp/extractor/mediaset.py
+++ b/yt_dlp/extractor/mediaset.py
@@ -127,7 +127,8 @@ class MediasetIE(ThePlatformBaseIE):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Dead link',
     }, {
         # WittyTV embed
         'url': 'https://www.wittytv.it/mauriziocostanzoshow/ultima-puntata-venerdi-25-novembre/',
diff --git a/yt_dlp/extractor/movieclips.py b/yt_dlp/extractor/movieclips.py
index 4777f440e..f7f2921fd 100644
--- a/yt_dlp/extractor/movieclips.py
+++ b/yt_dlp/extractor/movieclips.py
@@ -23,6 +23,7 @@ class MovieClipsIE(InfoExtractor):
             'uploader': 'Movieclips',
         },
         'add_ie': ['ThePlatform'],
+        'skip': 'redirects to YouTube',
     }
 
     def _real_extract(self, url):
diff --git a/yt_dlp/extractor/nationalgeographic.py b/yt_dlp/extractor/nationalgeographic.py
index ad525c258..6f046bc29 100644
--- a/yt_dlp/extractor/nationalgeographic.py
+++ b/yt_dlp/extractor/nationalgeographic.py
@@ -24,6 +24,7 @@ class NationalGeographicVideoIE(InfoExtractor):
                 'uploader': 'NAGS',
             },
             'add_ie': ['ThePlatform'],
+            'skip': 'Redirects to main page',
         },
         {
             'url': 'http://video.nationalgeographic.com/wild/when-sharks-attack/the-real-jaws',
@@ -38,6 +39,7 @@ class NationalGeographicVideoIE(InfoExtractor):
                 'uploader': 'NAGS',
             },
             'add_ie': ['ThePlatform'],
+            'skip': 'Redirects to main page',
         },
     ]
 
@@ -75,6 +77,7 @@ class NationalGeographicTVIE(FOXIE):  # XXX: Do not subclass from concrete IE
         'params': {
             'skip_download': True,
         },
+        'skip': 'Content not available',
     }]
     _HOME_PAGE_URL = 'https://www.nationalgeographic.com/tv/'
     _API_KEY = '238bb0a0c2aba67922c48709ce0c06fd'
diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index b3c28ab55..666550a49 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -284,7 +284,7 @@ class NBCSportsIE(InfoExtractor):
 
     _TESTS = [{
         # iframe src
-        'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
+        'url': 'https://www.nbcsports.com/watch/nfl/profootballtalk/pft-pm/unpacking-addisons-reckless-driving-citation',
         'info_dict': {
             'id': 'PHJSaFWbrTY9',
             'ext': 'mp4',
@@ -379,7 +379,7 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
     _TESTS = [
         {
             'url': 'http://www.nbcnews.com/watch/nbcnews-com/how-twitter-reacted-to-the-snowden-interview-269389891880',
-            'md5': 'cf4bc9e6ce0130f00f545d80ecedd4bf',
+            'md5': 'fb3dcd2d7b1dd9804305fa2fc95ab610',  # md5 tends to fluctuate
             'info_dict': {
                 'id': '269389891880',
                 'ext': 'mp4',
@@ -387,6 +387,8 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
                 'timestamp': 1401363060,
                 'upload_date': '20140529',
+                'duration': 46.0,
+                'thumbnail': 'https://media-cldnry.s-nbcnews.com/image/upload/MSNBC/Components/Video/140529/p_tweet_snow_140529.jpg',
             },
         },
         {
@@ -402,7 +404,7 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
         },
         {
             'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
-            'md5': '8eb831eca25bfa7d25ddd83e85946548',
+            'md5': '40d0e48c68896359c80372306ece0fc3',
             'info_dict': {
                 'id': '394064451844',
                 'ext': 'mp4',
@@ -410,11 +412,13 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
                 'timestamp': 1423104900,
                 'upload_date': '20150205',
+                'duration': 1236.0,
+                'thumbnail': 'https://media-cldnry.s-nbcnews.com/image/upload/MSNBC/Components/Video/__NEW/nn_netcast_150204.jpg',
             },
         },
         {
             'url': 'http://www.nbcnews.com/business/autos/volkswagen-11-million-vehicles-could-have-suspect-software-emissions-scandal-n431456',
-            'md5': '4a8c4cec9e1ded51060bdda36ff0a5c0',
+            'md5': 'ffb59bcf0733dc3c7f0ace907f5e3939',
             'info_dict': {
                 'id': 'n431456',
                 'ext': 'mp4',
@@ -422,11 +426,13 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:d22d1281a24f22ea0880741bb4dd6301',
                 'upload_date': '20150922',
                 'timestamp': 1442917800,
+                'duration': 37.0,
+                'thumbnail': 'https://media-cldnry.s-nbcnews.com/image/upload/MSNBC/Components/Video/__NEW/x_lon_vwhorn_150922.jpg',
             },
         },
         {
             'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
-            'md5': '118d7ca3f0bea6534f119c68ef539f71',
+            'md5': '693d1fa21d23afcc9b04c66b227ed9ff',
             'info_dict': {
                 'id': '669831235788',
                 'ext': 'mp4',
@@ -434,6 +440,8 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
                 'upload_date': '20160420',
                 'timestamp': 1461152093,
+                'duration': 69.0,
+                'thumbnail': 'https://media-cldnry.s-nbcnews.com/image/upload/MSNBC/Components/Video/201604/2016-04-20T11-35-09-133Z--1280x720.jpg',
             },
         },
         {
@@ -447,6 +455,7 @@ class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'timestamp': 1406937606,
                 'upload_date': '20140802',
+                'duration': 940.0,
             },
         },
         {
@@ -535,6 +544,7 @@ class NBCOlympicsIE(InfoExtractor):
             'upload_date': '20160815',
             'uploader': 'NBCU-SPORTS',
         },
+        'skip': '404 Not Found',
     }
 
     def _real_extract(self, url):
@@ -578,6 +588,7 @@ class NBCOlympicsStreamIE(AdobePassIE):
             'params': {
                 'skip_download': 'm3u8',
             },
+            'skip': 'Livestream',
         }, {
             'note': 'Plain m3u8 source URL',
             'url': 'https://stream.nbcolympics.com/gymnastics-event-finals-mens-floor-pommel-horse-womens-vault-bars',
@@ -589,6 +600,7 @@ class NBCOlympicsStreamIE(AdobePassIE):
             'params': {
                 'skip_download': 'm3u8',
             },
+            'skip': 'Livestream',
         },
     ]
 
diff --git a/yt_dlp/extractor/scrippsnetworks.py b/yt_dlp/extractor/scrippsnetworks.py
index adfd7e5f2..7f0bc9645 100644
--- a/yt_dlp/extractor/scrippsnetworks.py
+++ b/yt_dlp/extractor/scrippsnetworks.py
@@ -39,6 +39,7 @@ class ScrippsNetworksWatchIE(AWSIE):
             'skip_download': True,
         },
         'add_ie': [AnvatoIE.ie_key()],
+        'skip': '404 Not Found',
     }]
 
     _SNI_TABLE = {
@@ -113,6 +114,9 @@ class ScrippsNetworksIE(InfoExtractor):
             'timestamp': 1475678834,
             'upload_date': '20161005',
             'uploader': 'SCNI-SCND',
+            'duration': 29.995,
+            'chapters': [{'start_time': 0.0, 'end_time': 29.995, 'title': '<Untitled Chapter 1>'}],
+            'thumbnail': 'https://images.dds.discovery.com/up/tp/Scripps_-_Food_Category_Prod/122/987/0260338_630x355.jpg',
         },
         'add_ie': ['ThePlatform'],
         'expected_warnings': ['No HLS formats found'],
diff --git a/yt_dlp/extractor/syfy.py b/yt_dlp/extractor/syfy.py
index c79d27a0d..afcdbf780 100644
--- a/yt_dlp/extractor/syfy.py
+++ b/yt_dlp/extractor/syfy.py
@@ -23,6 +23,7 @@ class SyfyIE(AdobePassIE):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+        'skip': 'Redirects to main page',
     }]
 
     def _real_extract(self, url):
diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py
index 99caeb5f9..433ce8427 100644
--- a/yt_dlp/extractor/theplatform.py
+++ b/yt_dlp/extractor/theplatform.py
@@ -167,7 +167,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
             # rtmp download
             'skip_download': True,
         },
-        'skip': '404 Not Found',
+        'skip': 'CNet no longer uses ThePlatform',
     }, {
         'url': 'https://player.theplatform.com/p/D6x-PC/pulse_preview/embed/select/media/yMBg9E8KFxZD',
         'info_dict': {
@@ -177,7 +177,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
             'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
             'uploader': 'EGSM',
         },
-        'skip': '404 Not Found',
+        'skip': 'Dead link',
     }, {
         'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
         'only_matching': True,
@@ -195,7 +195,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
             'upload_date': '20150701',
             'uploader': 'NBCU-NEWS',
         },
-        'skip': '404 Not Found',
+        'skip': 'Error: Player PID "nbcNewsOffsite" is disabled',
     }, {
         # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
         # geo-restricted (US), HLS encrypted with AES-128
diff --git a/yt_dlp/extractor/theweatherchannel.py b/yt_dlp/extractor/theweatherchannel.py
index 682e4335d..d1921e4f9 100644
--- a/yt_dlp/extractor/theweatherchannel.py
+++ b/yt_dlp/extractor/theweatherchannel.py
@@ -11,17 +11,19 @@
 class TheWeatherChannelIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
     _VALID_URL = r'https?://(?:www\.)?weather\.com(?P<asset_name>(?:/(?P<locale>[a-z]{2}-[A-Z]{2}))?/(?:[^/]+/)*video/(?P<id>[^/?#]+))'
     _TESTS = [{
-        'url': 'https://weather.com/series/great-outdoors/video/ice-climber-is-in-for-a-shock',
-        'md5': 'c4cbe74c9c17c5676b704b950b73dd92',
+        'url': 'https://weather.com/storms/hurricane/video/invest-95l-in-atlantic-has-a-medium-chance-of-development',
+        'md5': '68f0cf616435683f27ce36bd9c927394',
         'info_dict': {
-            'id': 'cc82397e-cc3f-4d11-9390-a785add090e8',
+            'id': '81acef2d-ee8c-4545-ba83-bff3cc80db97',
             'ext': 'mp4',
-            'title': 'Ice Climber Is In For A Shock',
-            'description': 'md5:55606ce1378d4c72e6545e160c9d9695',
-            'uploader': 'TWC - Digital (No Distro)',
-            'uploader_id': '6ccd5455-16bb-46f2-9c57-ff858bb9f62c',
-            'upload_date': '20160720',
-            'timestamp': 1469018835,
+            'title': 'Invest 95L In Atlantic Has A Medium Chance Of Development',
+            'description': 'md5:0de720fd5f0d0e32207bd4c270fff824',
+            'uploader': 'TWC - Digital',
+            'uploader_id': 'b5a999e0-9e04-11e1-9ee2-001d092f5a10',
+            'upload_date': '20230721',
+            'timestamp': 1689967343,
+            'display_id': 'invest-95l-in-atlantic-has-a-medium-chance-of-development',
+            'duration': 34.0,
         }
     }, {
         'url': 'https://weather.com/en-CA/international/videos/video/unidentified-object-falls-from-sky-in-india',

From 792f1e64f6a2beac51e85408d142b3118115c4fd Mon Sep 17 00:00:00 2001
From: Aleri Kaisattera <73682764+alerikaisattera@users.noreply.github.com>
Date: Sat, 7 Oct 2023 05:56:47 +0600
Subject: [PATCH 615/871] [ie/theta] Remove extractors (#8251)

Authored by: alerikaisattera
---
 yt_dlp/extractor/_extractors.py |  4 --
 yt_dlp/extractor/theta.py       | 90 ---------------------------------
 2 files changed, 94 deletions(-)
 delete mode 100644 yt_dlp/extractor/theta.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b10ef2f33..55c3c2f8e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2004,10 +2004,6 @@
 )
 from .thestar import TheStarIE
 from .thesun import TheSunIE
-from .theta import (
-    ThetaVideoIE,
-    ThetaStreamIE,
-)
 from .theweatherchannel import TheWeatherChannelIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
diff --git a/yt_dlp/extractor/theta.py b/yt_dlp/extractor/theta.py
deleted file mode 100644
index ecf0ea091..000000000
--- a/yt_dlp/extractor/theta.py
+++ /dev/null
@@ -1,90 +0,0 @@
-from .common import InfoExtractor
-from ..utils import try_get
-
-
-class ThetaStreamIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?theta\.tv/(?!video/)(?P<id>[a-z0-9-]+)'
-    _TESTS = [{
-        'url': 'https://www.theta.tv/davirus',
-        'skip': 'The live may have ended',
-        'info_dict': {
-            'id': 'DaVirus',
-            'ext': 'mp4',
-            'title': 'I choose you - My Community is King -👀 - YO HABLO ESPANOL - CODE DAVIRUS',
-            'thumbnail': r're:https://live-thumbnails-prod-theta-tv\.imgix\.net/thumbnail/.+\.jpg',
-        }
-    }, {
-        'url': 'https://www.theta.tv/mst3k',
-        'note': 'This channel is live 24/7',
-        'info_dict': {
-            'id': 'MST3K',
-            'ext': 'mp4',
-            'title': 'Mystery Science Theatre 3000 24/7 Powered by the THETA Network.',
-            'thumbnail': r're:https://user-prod-theta-tv\.imgix\.net/.+\.jpg',
-        }
-    }, {
-        'url': 'https://www.theta.tv/contv-anime',
-        'info_dict': {
-            'id': 'ConTVAnime',
-            'ext': 'mp4',
-            'title': 'CONTV ANIME 24/7. Powered by THETA Network.',
-            'thumbnail': r're:https://user-prod-theta-tv\.imgix\.net/.+\.jpg',
-        }
-    }]
-
-    def _real_extract(self, url):
-        channel_id = self._match_id(url)
-        info = self._download_json(f'https://api.theta.tv/v1/channel?alias={channel_id}', channel_id)['body']
-
-        m3u8_playlist = next(
-            data['url'] for data in info['live_stream']['video_urls']
-            if data.get('type') != 'embed' and data.get('resolution') in ('master', 'source'))
-
-        formats = self._extract_m3u8_formats(m3u8_playlist, channel_id, 'mp4', m3u8_id='hls', live=True)
-
-        channel = try_get(info, lambda x: x['user']['username'])  # using this field instead of channel_id due to capitalization
-
-        return {
-            'id': channel,
-            'title': try_get(info, lambda x: x['live_stream']['title']),
-            'channel': channel,
-            'view_count': try_get(info, lambda x: x['live_stream']['view_count']),
-            'is_live': True,
-            'formats': formats,
-            'thumbnail': try_get(info, lambda x: x['live_stream']['thumbnail_url']),
-        }
-
-
-class ThetaVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?theta\.tv/video/(?P<id>vid[a-z0-9]+)'
-    _TEST = {
-        'url': 'https://www.theta.tv/video/vidiq6aaet3kzf799p0',
-        'md5': '633d8c29eb276bb38a111dbd591c677f',
-        'info_dict': {
-            'id': 'vidiq6aaet3kzf799p0',
-            'ext': 'mp4',
-            'title': 'Theta EdgeCast Tutorial',
-            'uploader': 'Pixiekittie',
-            'description': 'md5:e316253f5bdced8b5a46bb50ae60a09f',
-            'thumbnail': r're:https://user-prod-theta-tv\.imgix\.net/.+/vod_thumb/.+.jpg',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        info = self._download_json(f'https://api.theta.tv/v1/video/{video_id}/raw', video_id)['body']
-
-        m3u8_playlist = try_get(info, lambda x: x['video_urls'][0]['url'])
-
-        formats = self._extract_m3u8_formats(m3u8_playlist, video_id, 'mp4', m3u8_id='hls')
-
-        return {
-            'id': video_id,
-            'title': info.get('title'),
-            'uploader': try_get(info, lambda x: x['user']['username']),
-            'description': info.get('description'),
-            'view_count': info.get('view_count'),
-            'like_count': info.get('like_count'),
-            'formats': formats,
-            'thumbnail': info.get('thumbnail_url'),
-        }

From 03e85ea99db76a2fddb65bf46f8819bda780aaf3 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 6 Oct 2023 20:00:15 -0500
Subject: [PATCH 616/871] [ie/youtube] Fix `heatmap` extraction (#8299)

Closes #8189
Authored by: bashonly
---
 yt_dlp/extractor/youtube.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 7e13aa779..b7ac3e9cc 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3292,16 +3292,15 @@ def _extract_chapters_from_engagement_panel(self, data, duration):
                                           chapter_time, chapter_title, duration)
             for contents in content_list)), [])
 
-    def _extract_heatmap_from_player_overlay(self, data):
-        content_list = traverse_obj(data, (
-            'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer', 'decoratedPlayerBarRenderer', 'playerBar',
-            'multiMarkersPlayerBarRenderer', 'markersMap', ..., 'value', 'heatmap', 'heatmapRenderer', 'heatMarkers', {list}))
-        return next(filter(None, (
-            traverse_obj(contents, (..., 'heatMarkerRenderer', {
-                'start_time': ('timeRangeStartMillis', {functools.partial(float_or_none, scale=1000)}),
-                'end_time': {lambda x: (x['timeRangeStartMillis'] + x['markerDurationMillis']) / 1000},
-                'value': ('heatMarkerIntensityScoreNormalized', {float_or_none}),
-            })) for contents in content_list)), None)
+    def _extract_heatmap(self, data):
+        return traverse_obj(data, (
+            'frameworkUpdates', 'entityBatchUpdate', 'mutations',
+            lambda _, v: v['payload']['macroMarkersListEntity']['markersList']['markerType'] == 'MARKER_TYPE_HEATMAP',
+            'payload', 'macroMarkersListEntity', 'markersList', 'markers', ..., {
+                'start_time': ('startMillis', {functools.partial(float_or_none, scale=1000)}),
+                'end_time': {lambda x: (int(x['startMillis']) + int(x['durationMillis'])) / 1000},
+                'value': ('intensityScoreNormalized', {float_or_none}),
+            })) or None
 
     def _extract_comment(self, comment_renderer, parent=None):
         comment_id = comment_renderer.get('commentId')
@@ -4435,7 +4434,7 @@ def process_language(container, base_url, lang_code, sub_name, query):
                 or self._extract_chapters_from_description(video_description, duration)
                 or None)
 
-            info['heatmap'] = self._extract_heatmap_from_player_overlay(initial_data)
+            info['heatmap'] = self._extract_heatmap(initial_data)
 
         contents = traverse_obj(
             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),

From 377e85a1797db9e98b78b38203ed9d4ded229991 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 7 Oct 2023 03:02:45 +0200
Subject: [PATCH 617/871] [cleanup] Misc (#8300)

* Simplify nuxt regex
* Fix tmz quotes and tests
* Update test python versions

Authored by: dirkf, gamer191, Grub4K
---
 .github/workflows/core.yml     |   4 +-
 .github/workflows/download.yml |   2 +-
 yt_dlp/extractor/common.py     |   2 +-
 yt_dlp/extractor/tmz.py        | 266 +++++++++++++++++----------------
 4 files changed, 138 insertions(+), 136 deletions(-)

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 689408c50..7fcf11dfa 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -13,7 +13,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         # CPython 3.11 is in quick-test
-        python-version: ['3.8', '3.9', '3.10', '3.12-dev', pypy-3.7, pypy-3.8, pypy-3.10]
+        python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.7, pypy-3.8, pypy-3.10]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
@@ -21,7 +21,7 @@ jobs:
           python-version: '3.7'
           run-tests-ext: bat
         - os: windows-latest
-          python-version: '3.12-dev'
+          python-version: '3.12'
           run-tests-ext: bat
         - os: windows-latest
           python-version: pypy-3.9
diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml
index 2b2387d4f..c3478721c 100644
--- a/.github/workflows/download.yml
+++ b/.github/workflows/download.yml
@@ -28,7 +28,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.7', '3.10', 3.11-dev, pypy-3.7, pypy-3.8]
+        python-version: ['3.7', '3.10', '3.12', pypy-3.7, pypy-3.8, pypy-3.10]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index c94b4abdc..c3ceb0039 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -1687,7 +1687,7 @@ def _search_nextjs_data(self, webpage, video_id, *, transform_source=None, fatal
     def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', *, fatal=True, traverse=('data', 0)):
         """Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
         rectx = re.escape(context_name)
-        FUNCTION_RE = r'\(function\((?P<arg_keys>.*?)\){(?:.*?)return\s+(?P<js>{.*?})\s*;?\s*}\((?P<arg_vals>.*?)\)'
+        FUNCTION_RE = r'\(function\((?P<arg_keys>.*?)\){.*?\breturn\s+(?P<js>{.*?})\s*;?\s*}\((?P<arg_vals>.*?)\)'
         js, arg_keys, arg_vals = self._search_regex(
             (rf'<script>\s*window\.{rectx}={FUNCTION_RE}\s*\)\s*;?\s*</script>', rf'{rectx}\(.*?{FUNCTION_RE}'),
             webpage, context_name, group=('js', 'arg_keys', 'arg_vals'),
diff --git a/yt_dlp/extractor/tmz.py b/yt_dlp/extractor/tmz.py
index ffb30c6b8..edd16bc5b 100644
--- a/yt_dlp/extractor/tmz.py
+++ b/yt_dlp/extractor/tmz.py
@@ -8,158 +8,160 @@
 
 
 class TMZIE(InfoExtractor):
-    _VALID_URL = r"https?://(?:www\.)?tmz\.com/.*"
+    _VALID_URL = r'https?://(?:www\.)?tmz\.com/.*'
     _TESTS = [
         {
-            "url": "http://www.tmz.com/videos/0-cegprt2p/",
-            "info_dict": {
-                "id": "http://www.tmz.com/videos/0-cegprt2p/",
-                "ext": "mp4",
-                "title": "No Charges Against Hillary Clinton? Harvey Says It Ain't Over Yet",
-                "description": "Harvey talks about Director Comey’s decision not to prosecute Hillary Clinton.",
-                "timestamp": 1467831837,
-                "uploader": "TMZ Staff",
-                "upload_date": "20160706",
-                "thumbnail": "https://imagez.tmz.com/image/5e/4by3/2016/07/06/5eea7dc01baa5c2e83eb06930c170e46_xl.jpg",
-                "duration": 772.0,
+            'url': 'http://www.tmz.com/videos/0-cegprt2p/',
+            'info_dict': {
+                'id': 'http://www.tmz.com/videos/0-cegprt2p/',
+                'ext': 'mp4',
+                'title': 'No Charges Against Hillary Clinton? Harvey Says It Ain\'t Over Yet',
+                'description': 'Harvey talks about Director Comey’s decision not to prosecute Hillary Clinton.',
+                'timestamp': 1467831837,
+                'uploader': 'TMZ Staff',
+                'upload_date': '20160706',
+                'thumbnail': 'https://imagez.tmz.com/image/5e/4by3/2016/07/06/5eea7dc01baa5c2e83eb06930c170e46_xl.jpg',
+                'duration': 772.0,
             },
         },
         {
-            "url": "https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/",
-            "info_dict": {
-                "id": "https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/",
-                "ext": "mp4",
-                "title": "Angry Bagel Shop Guy Says He Doesn't Trust Women",
-                "description": "The enraged man who went viral for ranting about women on dating sites before getting ragdolled in a bagel shop is defending his misogyny ... he says it's women's fault in the first place.",
-                "timestamp": 1562889485,
-                "uploader": "TMZ Staff",
-                "upload_date": "20190711",
-                "thumbnail": "https://imagez.tmz.com/image/a8/4by3/2019/07/12/a85480d27b2f50a7bfea2322151d67a5_xl.jpg",
-                "duration": 123.0,
+            'url': 'https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/',
+            'info_dict': {
+                'id': 'https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/',
+                'ext': 'mp4',
+                'title': 'Angry Bagel Shop Guy Says He Doesn\'t Trust Women',
+                'description': 'The enraged man who went viral for ranting about women on dating sites before getting ragdolled in a bagel shop is defending his misogyny ... he says it\'s women\'s fault in the first place.',
+                'timestamp': 1562889485,
+                'uploader': 'TMZ Staff',
+                'upload_date': '20190711',
+                'thumbnail': 'https://imagez.tmz.com/image/a8/4by3/2019/07/12/a85480d27b2f50a7bfea2322151d67a5_xl.jpg',
+                'duration': 123.0,
             },
         },
         {
-            "url": "http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert",
-            "md5": "5429c85db8bde39a473a56ca8c4c5602",
-            "info_dict": {
-                "id": "http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert",
-                "ext": "mp4",
-                "title": "Bobby Brown Tells Crowd ... Bobbi Kristina is Awake",
-                "description": 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake.  She\'s watching me."',
-                "timestamp": 1429467813,
-                "uploader": "TMZ Staff",
-                "upload_date": "20150419",
-                "duration": 29.0,
-                "thumbnail": "https://imagez.tmz.com/image/15/4by3/2015/04/20/1539c7ae136359fc979236fa6a9449dd_xl.jpg",
+            'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert',
+            'md5': '5429c85db8bde39a473a56ca8c4c5602',
+            'info_dict': {
+                'id': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert',
+                'ext': 'mp4',
+                'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake',
+                'description': 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake.  She\'s watching me."',
+                'timestamp': 1429467813,
+                'uploader': 'TMZ Staff',
+                'upload_date': '20150419',
+                'duration': 29.0,
+                'thumbnail': 'https://imagez.tmz.com/image/15/4by3/2015/04/20/1539c7ae136359fc979236fa6a9449dd_xl.jpg',
             },
         },
         {
-            "url": "http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/",
-            "info_dict": {
-                "id": "http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/",
-                "ext": "mp4",
-                "title": "Patti LaBelle -- Goes Nuclear On Stripping Fan",
-                "description": "Patti LaBelle made it known loud and clear last night ... NO "
-                "ONE gets on her stage and strips down.",
-                "timestamp": 1442683746,
-                "uploader": "TMZ Staff",
-                "upload_date": "20150919",
-                "duration": 104.0,
-                "thumbnail": "https://imagez.tmz.com/image/5e/4by3/2015/09/20/5e57d7575062528082994e18ac3f0f48_xl.jpg",
+            'url': 'http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/',
+            'info_dict': {
+                'id': 'http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/',
+                'ext': 'mp4',
+                'title': 'Patti LaBelle -- Goes Nuclear On Stripping Fan',
+                'description': 'Patti LaBelle made it known loud and clear last night ... NO '
+                'ONE gets on her stage and strips down.',
+                'timestamp': 1442683746,
+                'uploader': 'TMZ Staff',
+                'upload_date': '20150919',
+                'duration': 104.0,
+                'thumbnail': 'https://imagez.tmz.com/image/5e/4by3/2015/09/20/5e57d7575062528082994e18ac3f0f48_xl.jpg',
             },
         },
         {
-            "url": "http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/",
-            "info_dict": {
-                "id": "http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/",
-                "ext": "mp4",
-                "title": "NBA's Adam Silver -- Blake Griffin's a Great Guy ... He'll Learn from This",
-                "description": "Two pretty parts of this video with NBA Commish Adam Silver.",
-                "timestamp": 1454010989,
-                "uploader": "TMZ Staff",
-                "upload_date": "20160128",
-                "duration": 59.0,
-                "thumbnail": "https://imagez.tmz.com/image/38/4by3/2016/01/29/3856e83e0beb57059ec412122b842fb1_xl.jpg",
+            'url': 'http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/',
+            'info_dict': {
+                'id': 'http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/',
+                'ext': 'mp4',
+                'title': 'NBA\'s Adam Silver -- Blake Griffin\'s a Great Guy ... He\'ll Learn from This',
+                'description': 'Two pretty parts of this video with NBA Commish Adam Silver.',
+                'timestamp': 1454010989,
+                'uploader': 'TMZ Staff',
+                'upload_date': '20160128',
+                'duration': 59.0,
+                'thumbnail': 'https://imagez.tmz.com/image/38/4by3/2016/01/29/3856e83e0beb57059ec412122b842fb1_xl.jpg',
             },
         },
         {
-            "url": "http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/",
-            "info_dict": {
-                "id": "http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/",
-                "ext": "mp4",
-                "title": "Trump Star Vandal -- I'm Not Afraid of Donald or the Cops!",
-                "description": "James Otis is the the guy who took a pickaxe to Donald Trump's star on the Walk of Fame, and he tells TMZ .. he's ready and willing to go to jail for the crime.",
-                "timestamp": 1477500095,
-                "uploader": "TMZ Staff",
-                "upload_date": "20161026",
-                "thumbnail": "https://imagez.tmz.com/image/0d/4by3/2016/10/27/0d904814d4a75dcf9cc3b8cfd1edc1a3_xl.jpg",
-                "duration": 128.0,
+            'url': 'http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/',
+            'info_dict': {
+                'id': 'http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/',
+                'ext': 'mp4',
+                'title': 'Trump Star Vandal -- I\'m Not Afraid of Donald or the Cops!',
+                'description': 'James Otis is the the guy who took a pickaxe to Donald Trump\'s star on the Walk of Fame, and he tells TMZ .. he\'s ready and willing to go to jail for the crime.',
+                'timestamp': 1477500095,
+                'uploader': 'TMZ Staff',
+                'upload_date': '20161026',
+                'thumbnail': 'https://imagez.tmz.com/image/0d/4by3/2016/10/27/0d904814d4a75dcf9cc3b8cfd1edc1a3_xl.jpg',
+                'duration': 128.0,
             },
         },
         {
-            "url": "https://www.tmz.com/videos/2020-10-31-103120-beverly-hills-protest-4878209/",
-            "info_dict": {
-                "id": "https://www.tmz.com/videos/2020-10-31-103120-beverly-hills-protest-4878209/",
-                "ext": "mp4",
-                "title": "Cops Use Billy Clubs Against Pro-Trump and Anti-Fascist "
-                "Demonstrators",
-                "description": "Beverly Hills may be an omen of what's coming next week, "
-                "because things got crazy on the streets and cops started "
-                "swinging their billy clubs at both Anti-Fascist and Pro-Trump "
-                "demonstrators.",
-                "timestamp": 1604182772,
-                "uploader": "TMZ Staff",
-                "upload_date": "20201031",
-                "duration": 96.0,
-                "thumbnail": "https://imagez.tmz.com/image/f3/4by3/2020/10/31/f37bd5a8aef84497866f425130c58be3_xl.jpg",
+            'url': 'https://www.tmz.com/videos/2020-10-31-103120-beverly-hills-protest-4878209/',
+            'info_dict': {
+                'id': 'https://www.tmz.com/videos/2020-10-31-103120-beverly-hills-protest-4878209/',
+                'ext': 'mp4',
+                'title': 'Cops Use Billy Clubs Against Pro-Trump and Anti-Fascist '
+                'Demonstrators',
+                'description': 'Beverly Hills may be an omen of what\'s coming next week, '
+                'because things got crazy on the streets and cops started '
+                'swinging their billy clubs at both Anti-Fascist and Pro-Trump '
+                'demonstrators.',
+                'timestamp': 1604182772,
+                'uploader': 'TMZ Staff',
+                'upload_date': '20201031',
+                'duration': 96.0,
+                'thumbnail': 'https://imagez.tmz.com/image/f3/4by3/2020/10/31/f37bd5a8aef84497866f425130c58be3_xl.jpg',
             },
         },
         {
-            "url": "https://www.tmz.com/2020/11/05/gervonta-davis-car-crash-hit-and-run-police/",
-            "info_dict": {
-                "id": "Dddb6IGe-ws",
-                "ext": "mp4",
-                "title": "SICK LAMBO GERVONTA DAVIS IN HIS NEW RIDE RIGHT AFTER KO AFTER LEO  EsNews Boxing",
-                "uploader": "ESNEWS",
-                "description": "md5:49675bc58883ccf80474b8aa701e1064",
-                "upload_date": "20201102",
-                "uploader_id": "ESNEWS",
-                "uploader_url": "http://www.youtube.com/user/ESNEWS",
-                "like_count": int,
-                "channel_id": "UCI-Oq7oFGakzSzHFlTtsUsQ",
-                "channel": "ESNEWS",
-                "view_count": int,
-                "duration": 225,
-                "live_status": "not_live",
-                "thumbnail": "https://i.ytimg.com/vi_webp/Dddb6IGe-ws/maxresdefault.webp",
-                "channel_url": "https://www.youtube.com/channel/UCI-Oq7oFGakzSzHFlTtsUsQ",
-                "channel_follower_count": int,
-                "playable_in_embed": True,
-                "categories": ["Sports"],
-                "age_limit": 0,
-                "tags": "count:10",
-                "availability": "public",
+            'url': 'https://www.tmz.com/2020/11/05/gervonta-davis-car-crash-hit-and-run-police/',
+            'info_dict': {
+                'id': 'Dddb6IGe-ws',
+                'ext': 'mp4',
+                'title': 'SICK LAMBO GERVONTA DAVIS IN HIS NEW RIDE RIGHT AFTER KO AFTER LEO  EsNews Boxing',
+                'uploader': 'ESNEWS',
+                'description': 'md5:49675bc58883ccf80474b8aa701e1064',
+                'upload_date': '20201102',
+                'uploader_id': '@ESNEWS',
+                'uploader_url': 'https://www.youtube.com/@ESNEWS',
+                'like_count': int,
+                'channel_id': 'UCI-Oq7oFGakzSzHFlTtsUsQ',
+                'channel': 'ESNEWS',
+                'view_count': int,
+                'duration': 225,
+                'live_status': 'not_live',
+                'thumbnail': 'https://i.ytimg.com/vi_webp/Dddb6IGe-ws/maxresdefault.webp',
+                'channel_url': 'https://www.youtube.com/channel/UCI-Oq7oFGakzSzHFlTtsUsQ',
+                'channel_follower_count': int,
+                'playable_in_embed': True,
+                'categories': ['Sports'],
+                'age_limit': 0,
+                'tags': 'count:10',
+                'availability': 'public',
+                'comment_count': int,
             },
         },
         {
-            "url": "https://www.tmz.com/2020/11/19/conor-mcgregor-dustin-poirier-contract-fight-ufc-257-fight-island/",
-            "info_dict": {
-                "id": "1329450007125225473",
-                "ext": "mp4",
-                "title": "The Mac Life - BREAKING: Conor McGregor (@thenotoriousmma) has signed his bout agreement for his rematch with Dustin Poirier for January 23.",
-                "uploader": "The Mac Life",
-                "description": "md5:56e6009bbc3d12498e10d08a8e1f1c69",
-                "upload_date": "20201119",
-                "uploader_id": "TheMacLife",
-                "timestamp": 1605800556,
-                "thumbnail": "https://pbs.twimg.com/media/EnMmfT8XYAExgxJ.jpg?name=small",
-                "like_count": int,
-                "duration": 11.812,
-                "uploader_url": "https://twitter.com/TheMacLife",
-                "age_limit": 0,
-                "repost_count": int,
-                "tags": [],
-                "comment_count": int,
+            'url': 'https://www.tmz.com/2020/11/19/conor-mcgregor-dustin-poirier-contract-fight-ufc-257-fight-island/',
+            'info_dict': {
+                'id': '1329448013937471491',
+                'ext': 'mp4',
+                'title': 'The Mac Life - BREAKING: Conor McGregor (@thenotoriousmma) has signed his bout agreement for his rematch with Dustin Poirier for January 23.',
+                'uploader': 'The Mac Life',
+                'description': 'md5:56e6009bbc3d12498e10d08a8e1f1c69',
+                'upload_date': '20201119',
+                'display_id': '1329450007125225473',
+                'uploader_id': 'TheMacLife',
+                'timestamp': 1605800556,
+                'thumbnail': 'https://pbs.twimg.com/media/EnMmfT8XYAExgxJ.jpg?name=small',
+                'like_count': int,
+                'duration': 11.812,
+                'uploader_url': 'https://twitter.com/TheMacLife',
+                'age_limit': 0,
+                'repost_count': int,
+                'tags': [],
+                'comment_count': int,
             },
         },
     ]
@@ -167,25 +169,25 @@ class TMZIE(InfoExtractor):
     def _real_extract(self, url):
         webpage = self._download_webpage(url, url)
         jsonld = self._search_json_ld(webpage, url)
-        if not jsonld or "url" not in jsonld:
+        if not jsonld or 'url' not in jsonld:
             # try to extract from YouTube Player API
             # see https://developers.google.com/youtube/iframe_api_reference#Video_Queueing_Functions
             match_obj = re.search(r'\.cueVideoById\(\s*(?P<quote>[\'"])(?P<id>.*?)(?P=quote)', webpage)
             if match_obj:
-                res = self.url_result(match_obj.group("id"))
+                res = self.url_result(match_obj.group('id'))
                 return res
             # try to extract from twitter
-            blockquote_el = get_element_by_attribute("class", "twitter-tweet", webpage)
+            blockquote_el = get_element_by_attribute('class', 'twitter-tweet', webpage)
             if blockquote_el:
                 matches = re.findall(
                     r'<a[^>]+href=\s*(?P<quote>[\'"])(?P<link>.*?)(?P=quote)',
                     blockquote_el)
                 if matches:
                     for _, match in matches:
-                        if "/status/" in match:
+                        if '/status/' in match:
                             res = self.url_result(match)
                             return res
-            raise ExtractorError("No video found!")
+            raise ExtractorError('No video found!')
         if id not in jsonld:
-            jsonld["id"] = url
+            jsonld['id'] = url
         return jsonld

From 4392c4680c383b221b6aa26d25c6e4b5581a5ad6 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@example.com>
Date: Sat, 7 Oct 2023 01:28:34 +0000
Subject: [PATCH 618/871] Release 2023.10.07

Created by: Grub4K

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |  8 ++---
 .../ISSUE_TEMPLATE/2_site_support_request.yml |  8 ++---
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |  8 ++---
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |  8 ++---
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |  8 ++---
 .github/ISSUE_TEMPLATE/6_question.yml         |  8 ++---
 CONTRIBUTORS                                  |  6 ++++
 Changelog.md                                  | 29 +++++++++++++++++++
 supportedsites.md                             |  4 +--
 yt_dlp/version.py                             |  4 +--
 10 files changed, 63 insertions(+), 28 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index f0fc71d57..dacb41758 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -64,7 +64,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -72,8 +72,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.09.24, Current version: 2023.09.24
-        yt-dlp is up to date (2023.09.24)
+        Latest version: 2023.10.07, Current version: 2023.10.07
+        yt-dlp is up to date (2023.10.07)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index ac9a72a1c..ec6e298a1 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -76,7 +76,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -84,8 +84,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.09.24, Current version: 2023.09.24
-        yt-dlp is up to date (2023.09.24)
+        Latest version: 2023.10.07, Current version: 2023.10.07
+        yt-dlp is up to date (2023.10.07)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index 577e4d491..cf3cdd21f 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -72,7 +72,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -80,8 +80,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.09.24, Current version: 2023.09.24
-        yt-dlp is up to date (2023.09.24)
+        Latest version: 2023.10.07, Current version: 2023.10.07
+        yt-dlp is up to date (2023.10.07)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index 9529c1bd6..1bbcf6895 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -57,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -65,8 +65,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.09.24, Current version: 2023.09.24
-        yt-dlp is up to date (2023.09.24)
+        Latest version: 2023.10.07, Current version: 2023.10.07
+        yt-dlp is up to date (2023.10.07)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index b17a6e046..d3bc06e80 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -53,7 +53,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -61,7 +61,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.09.24, Current version: 2023.09.24
-        yt-dlp is up to date (2023.09.24)
+        Latest version: 2023.10.07, Current version: 2023.10.07
+        yt-dlp is up to date (2023.10.07)
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index 5345e8917..30311d5b5 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.09.24** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -59,7 +59,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.09.24 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -67,7 +67,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.09.24, Current version: 2023.09.24
-        yt-dlp is up to date (2023.09.24)
+        Latest version: 2023.10.07, Current version: 2023.10.07
+        yt-dlp is up to date (2023.10.07)
         <more lines>
       render: shell
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 72b9584ec..8eda41307 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -503,3 +503,9 @@ Yalab7
 zhallgato
 zhong-yiyu
 Zprokkel
+AS6939
+drzraf
+handlerug
+jiru
+madewokherd
+xofe
diff --git a/Changelog.md b/Changelog.md
index 04511927f..48dcbf102 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,35 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.10.07
+
+#### Extractor changes
+- **abc.net.au**: iview: [Improve `episode` extraction](https://github.com/yt-dlp/yt-dlp/commit/a9efb4b8d74f3583450ffda0ee57259a47d39c70) ([#8201](https://github.com/yt-dlp/yt-dlp/issues/8201)) by [xofe](https://github.com/xofe)
+- **erocast**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/47c598783c98c179e04dd12c2a3fee0f3dc53087) ([#8264](https://github.com/yt-dlp/yt-dlp/issues/8264)) by [madewokherd](https://github.com/madewokherd)
+- **gofile**: [Fix token cookie bug](https://github.com/yt-dlp/yt-dlp/commit/0730d5a966fa8a937d84bfb7f68be5198acb039b) by [bashonly](https://github.com/bashonly)
+- **iq.com**: [Fix extraction and subtitles](https://github.com/yt-dlp/yt-dlp/commit/35d9cbaf9638ccc9daf8a863063b2e7c135bc664) ([#8260](https://github.com/yt-dlp/yt-dlp/issues/8260)) by [AS6939](https://github.com/AS6939)
+- **lbry**
+    - [Add playlist support](https://github.com/yt-dlp/yt-dlp/commit/48cceec1ddb8649b5e771df8df79eb9c39c82b90) ([#8213](https://github.com/yt-dlp/yt-dlp/issues/8213)) by [bashonly](https://github.com/bashonly), [drzraf](https://github.com/drzraf), [Grub4K](https://github.com/Grub4K)
+    - [Extract `uploader_id`](https://github.com/yt-dlp/yt-dlp/commit/0e722f2f3ca42e634fd7b06ee70b16bf833ce132) ([#8244](https://github.com/yt-dlp/yt-dlp/issues/8244)) by [drzraf](https://github.com/drzraf)
+- **litv**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/91a670a4f7babe9c8aa2018f57d8c8952a6f49d8) ([#7785](https://github.com/yt-dlp/yt-dlp/issues/7785)) by [jiru](https://github.com/jiru)
+- **neteasemusic**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/f980df734cf5c0eaded2f7b38c6c60bccfeebb48) ([#8181](https://github.com/yt-dlp/yt-dlp/issues/8181)) by [c-basalt](https://github.com/c-basalt)
+- **nhk**: [Fix VOD extraction](https://github.com/yt-dlp/yt-dlp/commit/e831c80e8b2fc025b3b67d82974cc59e3526fdc8) ([#8249](https://github.com/yt-dlp/yt-dlp/issues/8249)) by [garret1317](https://github.com/garret1317)
+- **radiko**: [Improve extraction](https://github.com/yt-dlp/yt-dlp/commit/2ad3873f0dfa9285c91d2160e36c039e69d597c7) ([#8221](https://github.com/yt-dlp/yt-dlp/issues/8221)) by [garret1317](https://github.com/garret1317)
+- **substack**
+    - [Fix download cookies bug](https://github.com/yt-dlp/yt-dlp/commit/2f2dda3a7e85148773da3cdbc03ac9949ec1bc45) ([#8219](https://github.com/yt-dlp/yt-dlp/issues/8219)) by [handlerug](https://github.com/handlerug)
+    - [Fix embed extraction](https://github.com/yt-dlp/yt-dlp/commit/fbcc299bd8a19cf8b3c8805d6c268a9110230973) ([#8218](https://github.com/yt-dlp/yt-dlp/issues/8218)) by [handlerug](https://github.com/handlerug)
+- **theta**: [Remove extractors](https://github.com/yt-dlp/yt-dlp/commit/792f1e64f6a2beac51e85408d142b3118115c4fd) ([#8251](https://github.com/yt-dlp/yt-dlp/issues/8251)) by [alerikaisattera](https://github.com/alerikaisattera)
+- **wrestleuniversevod**: [Call API with device ID](https://github.com/yt-dlp/yt-dlp/commit/b095fd3fa9d58a65dc9b830bd63b9d909422aa86) ([#8272](https://github.com/yt-dlp/yt-dlp/issues/8272)) by [bashonly](https://github.com/bashonly)
+- **xhamster**: user: [Support creator urls](https://github.com/yt-dlp/yt-dlp/commit/cc8d8441524ec3442d7c0d3f8f33f15b66aa06f3) ([#8232](https://github.com/yt-dlp/yt-dlp/issues/8232)) by [Grub4K](https://github.com/Grub4K)
+- **youtube**
+    - [Fix `heatmap` extraction](https://github.com/yt-dlp/yt-dlp/commit/03e85ea99db76a2fddb65bf46f8819bda780aaf3) ([#8299](https://github.com/yt-dlp/yt-dlp/issues/8299)) by [bashonly](https://github.com/bashonly)
+    - [Raise a warning for `Incomplete Data` instead of an error](https://github.com/yt-dlp/yt-dlp/commit/eb5bdbfa70126c7d5355cc0954b63720522e462c) ([#8238](https://github.com/yt-dlp/yt-dlp/issues/8238)) by [coletdjnz](https://github.com/coletdjnz)
+
+#### Misc. changes
+- **cleanup**
+    - [Update extractor tests](https://github.com/yt-dlp/yt-dlp/commit/19c90e405b4137c06dfe6f9aaa02396df0da93e5) ([#7718](https://github.com/yt-dlp/yt-dlp/issues/7718)) by [trainman261](https://github.com/trainman261)
+    - Miscellaneous: [377e85a](https://github.com/yt-dlp/yt-dlp/commit/377e85a1797db9e98b78b38203ed9d4ded229991) by [dirkf](https://github.com/dirkf), [gamer191](https://github.com/gamer191), [Grub4K](https://github.com/Grub4K)
+
 ### 2023.09.24
 
 #### Important changes
diff --git a/supportedsites.md b/supportedsites.md
index 620e0f305..ecef4dc2d 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -422,6 +422,7 @@ # Supported sites
  - **eplus:inbound**: e+ (イープラス) overseas
  - **Epoch**
  - **Eporner**
+ - **Erocast**
  - **EroProfile**: [*eroprofile*](## "netrc machine")
  - **EroProfile:album**
  - **ertflix**: ERTFLIX videos
@@ -699,6 +700,7 @@ # Supported sites
  - **LastFMUser**
  - **lbry**
  - **lbry:channel**
+ - **lbry:playlist**
  - **LCI**
  - **Lcp**
  - **LcpPlay**
@@ -1474,8 +1476,6 @@ # Supported sites
  - **ThePlatformFeed**
  - **TheStar**
  - **TheSun**
- - **ThetaStream**
- - **ThetaVideo**
  - **TheWeatherChannel**
  - **ThisAmericanLife**
  - **ThisAV**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index 2a7c84b93..60c1c94cc 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.09.24'
+__version__ = '2023.10.07'
 
-RELEASE_GIT_HEAD = '088add9567d39b758737e4299a0e619fd89d2e8f'
+RELEASE_GIT_HEAD = '377e85a1797db9e98b78b38203ed9d4ded229991'
 
 VARIANT = None
 

From 9d7ded6419089c1bf252496073f73ad90ed71004 Mon Sep 17 00:00:00 2001
From: Awal Garg <awalgarg@gmail.com>
Date: Sun, 8 Oct 2023 01:57:23 +0200
Subject: [PATCH 619/871] [utils] `js_to_json`: Fix `Date` constructor parsing
 (#8295)

Authored by: awalgarg, Grub4K
---
 test/test_utils.py     | 7 ++++++-
 yt_dlp/utils/_utils.py | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index fd612ff86..77040f29c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1209,6 +1209,9 @@ def test_js_to_json_edgecases(self):
         on = js_to_json('\'"\\""\'')
         self.assertEqual(json.loads(on), '"""', msg='Unnecessary quote escape should be escaped')
 
+        on = js_to_json('[new Date("spam"), \'("eggs")\']')
+        self.assertEqual(json.loads(on), ['spam', '("eggs")'], msg='Date regex should match a single string')
+
     def test_js_to_json_malformed(self):
         self.assertEqual(js_to_json('42a1'), '42"a1"')
         self.assertEqual(js_to_json('42a-1'), '42"a"-1')
@@ -1220,11 +1223,13 @@ def test_js_to_json_template_literal(self):
         self.assertEqual(js_to_json('`${name}"${name}"`', {'name': '5'}), '"5\\"5\\""')
         self.assertEqual(js_to_json('`${name}`', {}), '"name"')
 
-    def test_js_to_json_map_array_constructors(self):
+    def test_js_to_json_common_constructors(self):
         self.assertEqual(json.loads(js_to_json('new Map([["a", 5]])')), {'a': 5})
         self.assertEqual(json.loads(js_to_json('Array(5, 10)')), [5, 10])
         self.assertEqual(json.loads(js_to_json('new Array(15,5)')), [15, 5])
         self.assertEqual(json.loads(js_to_json('new Map([Array(5, 10),new Array(15,5)])')), {'5': 10, '15': 5})
+        self.assertEqual(json.loads(js_to_json('new Date("123")')), "123")
+        self.assertEqual(json.loads(js_to_json('new Date(\'2023-10-19\')')), "2023-10-19")
 
     def test_extract_attributes(self):
         self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index ba6242380..3dc17bf59 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -2744,7 +2744,7 @@ def create_map(mobj):
     code = re.sub(r'(?:new\s+)?Array\((.*?)\)', r'[\g<1>]', code)
     code = re.sub(r'new Map\((\[.*?\])?\)', create_map, code)
     if not strict:
-        code = re.sub(r'new Date\((".+")\)', r'\g<1>', code)
+        code = re.sub(rf'new Date\(({STRING_RE})\)', r'\g<1>', code)
         code = re.sub(r'new \w+\((.*?)\)', lambda m: json.dumps(m.group(0)), code)
         code = re.sub(r'parseInt\([^\d]+(\d+)[^\d]+\)', r'\1', code)
         code = re.sub(r'\(function\([^)]*\)\s*\{[^}]*\}\s*\)\s*\(\s*(["\'][^)]*["\'])\s*\)', r'\1', code)

From 1c51c520f7b511ebd9e4eb7322285a8c31eedbbd Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 8 Oct 2023 02:01:01 +0200
Subject: [PATCH 620/871] [fd/fragment] Improve progress calculation (#8241)

This uses the download speed from all threads and also adds smoothing to speed and eta

Authored by: Grub4K
---
 yt_dlp/downloader/fragment.py |  48 ++++++---------
 yt_dlp/utils/progress.py      | 109 ++++++++++++++++++++++++++++++++++
 2 files changed, 128 insertions(+), 29 deletions(-)
 create mode 100644 yt_dlp/utils/progress.py

diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index b4b680dae..b4f003d37 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -14,6 +14,7 @@
 from ..networking.exceptions import HTTPError, IncompleteRead
 from ..utils import DownloadError, RetryManager, encodeFilename, traverse_obj
 from ..utils.networking import HTTPHeaderDict
+from ..utils.progress import ProgressCalculator
 
 
 class HttpQuietDownloader(HttpFD):
@@ -226,8 +227,7 @@ def _start_frag_download(self, ctx, info_dict):
         resume_len = ctx['complete_frags_downloaded_bytes']
         total_frags = ctx['total_frags']
         ctx_id = ctx.get('ctx_id')
-        # This dict stores the download progress, it's updated by the progress
-        # hook
+        # Stores the download progress, updated by the progress hook
         state = {
             'status': 'downloading',
             'downloaded_bytes': resume_len,
@@ -237,14 +237,8 @@ def _start_frag_download(self, ctx, info_dict):
             'tmpfilename': ctx['tmpfilename'],
         }
 
-        start = time.time()
-        ctx.update({
-            'started': start,
-            'fragment_started': start,
-            # Amount of fragment's bytes downloaded by the time of the previous
-            # frag progress hook invocation
-            'prev_frag_downloaded_bytes': 0,
-        })
+        ctx['started'] = time.time()
+        progress = ProgressCalculator(resume_len)
 
         def frag_progress_hook(s):
             if s['status'] not in ('downloading', 'finished'):
@@ -259,38 +253,35 @@ def frag_progress_hook(s):
             state['max_progress'] = ctx.get('max_progress')
             state['progress_idx'] = ctx.get('progress_idx')
 
-            time_now = time.time()
-            state['elapsed'] = time_now - start
+            state['elapsed'] = progress.elapsed
             frag_total_bytes = s.get('total_bytes') or 0
             s['fragment_info_dict'] = s.pop('info_dict', {})
+
+            # XXX: Fragment resume is not accounted for here
             if not ctx['live']:
                 estimated_size = (
                     (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes)
                     / (state['fragment_index'] + 1) * total_frags)
-                state['total_bytes_estimate'] = estimated_size
+                progress.total = estimated_size
+                progress.update(s.get('downloaded_bytes'))
+                state['total_bytes_estimate'] = progress.total
+            else:
+                progress.update(s.get('downloaded_bytes'))
 
             if s['status'] == 'finished':
                 state['fragment_index'] += 1
                 ctx['fragment_index'] = state['fragment_index']
-                state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
-                ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
-                ctx['speed'] = state['speed'] = self.calc_speed(
-                    ctx['fragment_started'], time_now, frag_total_bytes)
-                ctx['fragment_started'] = time.time()
-                ctx['prev_frag_downloaded_bytes'] = 0
-            else:
-                frag_downloaded_bytes = s['downloaded_bytes']
-                state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
-                ctx['speed'] = state['speed'] = self.calc_speed(
-                    ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx.get('frag_resume_len', 0))
-                if not ctx['live']:
-                    state['eta'] = self.calc_eta(state['speed'], estimated_size - state['downloaded_bytes'])
-                ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
+                progress.thread_reset()
+
+            state['downloaded_bytes'] = ctx['complete_frags_downloaded_bytes'] = progress.downloaded
+            state['speed'] = ctx['speed'] = progress.speed.smooth
+            state['eta'] = progress.eta.smooth
+
             self._hook_progress(state, info_dict)
 
         ctx['dl'].add_progress_hook(frag_progress_hook)
 
-        return start
+        return ctx['started']
 
     def _finish_frag_download(self, ctx, info_dict):
         ctx['dest_stream'].close()
@@ -500,7 +491,6 @@ def _download_fragment(fragment):
                 download_fragment(fragment, ctx_copy)
                 return fragment, fragment['frag_index'], ctx_copy.get('fragment_filename_sanitized')
 
-            self.report_warning('The download speed shown is only of one thread. This is a known issue')
             with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool:
                 try:
                     for fragment, frag_index, frag_filename in pool.map(_download_fragment, fragments):
diff --git a/yt_dlp/utils/progress.py b/yt_dlp/utils/progress.py
new file mode 100644
index 000000000..f254a3887
--- /dev/null
+++ b/yt_dlp/utils/progress.py
@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import bisect
+import threading
+import time
+
+
+class ProgressCalculator:
+    # Time to calculate the speed over (seconds)
+    SAMPLING_WINDOW = 3
+    # Minimum timeframe before to sample next downloaded bytes (seconds)
+    SAMPLING_RATE = 0.05
+    # Time before showing eta (seconds)
+    GRACE_PERIOD = 1
+
+    def __init__(self, initial: int):
+        self._initial = initial or 0
+        self.downloaded = self._initial
+
+        self.elapsed: float = 0
+        self.speed = SmoothValue(0, smoothing=0.7)
+        self.eta = SmoothValue(None, smoothing=0.9)
+
+        self._total = 0
+        self._start_time = time.monotonic()
+        self._last_update = self._start_time
+
+        self._lock = threading.Lock()
+        self._thread_sizes: dict[int, int] = {}
+
+        self._times = [self._start_time]
+        self._downloaded = [self.downloaded]
+
+    @property
+    def total(self):
+        return self._total
+
+    @total.setter
+    def total(self, value: int | None):
+        with self._lock:
+            if value is not None and value < self.downloaded:
+                value = self.downloaded
+
+            self._total = value
+
+    def thread_reset(self):
+        current_thread = threading.get_ident()
+        with self._lock:
+            self._thread_sizes[current_thread] = 0
+
+    def update(self, size: int | None):
+        if not size:
+            return
+
+        current_thread = threading.get_ident()
+
+        with self._lock:
+            last_size = self._thread_sizes.get(current_thread, 0)
+            self._thread_sizes[current_thread] = size
+            self._update(size - last_size)
+
+    def _update(self, size: int):
+        current_time = time.monotonic()
+
+        self.downloaded += size
+        self.elapsed = current_time - self._start_time
+        if self.total is not None and self.downloaded > self.total:
+            self._total = self.downloaded
+
+        if self._last_update + self.SAMPLING_RATE > current_time:
+            return
+        self._last_update = current_time
+
+        self._times.append(current_time)
+        self._downloaded.append(self.downloaded)
+
+        offset = bisect.bisect_left(self._times, current_time - self.SAMPLING_WINDOW)
+        del self._times[:offset]
+        del self._downloaded[:offset]
+        if len(self._times) < 2:
+            self.speed.reset()
+            self.eta.reset()
+            return
+
+        download_time = current_time - self._times[0]
+        if not download_time:
+            return
+
+        self.speed.set((self.downloaded - self._downloaded[0]) / download_time)
+        if self.total and self.speed.value and self.elapsed > self.GRACE_PERIOD:
+            self.eta.set((self.total - self.downloaded) / self.speed.value)
+        else:
+            self.eta.reset()
+
+
+class SmoothValue:
+    def __init__(self, initial: float | None, smoothing: float):
+        self.value = self.smooth = self._initial = initial
+        self._smoothing = smoothing
+
+    def set(self, value: float):
+        self.value = value
+        if self.smooth is None:
+            self.smooth = self.value
+        else:
+            self.smooth = (1 - self._smoothing) * value + self._smoothing * self.smooth
+
+    def reset(self):
+        self.value = self.smooth = self._initial

From b7098d46b552a9322c6cea39ba80be5229f922de Mon Sep 17 00:00:00 2001
From: naginatana <96737708+naginatana@users.noreply.github.com>
Date: Tue, 10 Oct 2023 01:46:16 +0800
Subject: [PATCH 621/871] [ie/youku] Improve tudou.com support (#8160)

Authored by: naginatana
---
 yt_dlp/extractor/youku.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youku.py b/yt_dlp/extractor/youku.py
index 7ecd9f183..e35176586 100644
--- a/yt_dlp/extractor/youku.py
+++ b/yt_dlp/extractor/youku.py
@@ -20,7 +20,7 @@ class YoukuIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:
             https?://(
-                (?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
+                (?:v|play(?:er)?)\.(?:youku|tudou)\.com/(?:v_show/id_|player\.php/sid/)|
                 video\.tudou\.com/v/)|
             youku:)
         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
@@ -87,6 +87,19 @@ class YoukuIE(InfoExtractor):
             'uploader_url': 'https://www.youku.com/profile/index/?uid=UNjU2MzY1MzM1Ng==',
             'tags': list,
         },
+    }, {
+        'url': 'https://play.tudou.com/v_show/id_XNjAxNjI2OTU3Ng==.html?',
+        'info_dict': {
+            'id': 'XNjAxNjI2OTU3Ng',
+            'ext': 'mp4',
+            'title': '阿斯塔意识到哈里杀了人，自己被骗了',
+            'thumbnail': 'https://m.ykimg.com/0541010164F732752794D4D7B70331D1',
+            'uploader_id': '88758207',
+            'tags': [],
+            'uploader_url': 'https://www.youku.com/profile/index/?uid=UMzU1MDMyODI4',
+            'uploader': '英美剧场',
+            'duration': 72.91,
+        },
     }]
 
     @staticmethod

From 09f815ad52843219a7ee3f2a0dddf6c250c91f0c Mon Sep 17 00:00:00 2001
From: Stefan Lobbenmeier <Stefan.Lobbenmeier@gmail.com>
Date: Mon, 9 Oct 2023 19:51:37 +0200
Subject: [PATCH 622/871] [ie/ArteTV] Support age-restricted content (#8301)

Closes #7782
Authored by: StefanLobbenmeier
---
 yt_dlp/extractor/arte.py | 39 ++++++++++++++++-----------------------
 1 file changed, 16 insertions(+), 23 deletions(-)

diff --git a/yt_dlp/extractor/arte.py b/yt_dlp/extractor/arte.py
index a19cd2a3a..139a3a729 100644
--- a/yt_dlp/extractor/arte.py
+++ b/yt_dlp/extractor/arte.py
@@ -48,17 +48,7 @@ class ArteTVIE(ArteTVBaseIE):
     }, {
         'note': 'No alt_title',
         'url': 'https://www.arte.tv/fr/videos/110371-000-A/la-chaleur-supplice-des-arbres-de-rue/',
-        'info_dict': {
-            'id': '110371-000-A',
-            'ext': 'mp4',
-            'upload_date': '20220718',
-            'duration': 154,
-            'timestamp': 1658162460,
-            'description': 'md5:5890f36fe7dccfadb8b7c0891de54786',
-            'title': 'La chaleur, supplice des arbres de rue',
-            'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/CPE2sQDtD8GLQgt8DuYHLf/940x530',
-        },
-        'params': {'skip_download': 'm3u8'}
+        'only_matching': True,
     }, {
         'url': 'https://api.arte.tv/api/player/v2/config/de/100605-013-A',
         'only_matching': True,
@@ -67,19 +57,20 @@ class ArteTVIE(ArteTVBaseIE):
         'only_matching': True,
     }, {
         'url': 'https://www.arte.tv/de/videos/110203-006-A/zaz/',
+        'only_matching': True,
+    }, {
+        'note': 'age-restricted',
+        'url': 'https://www.arte.tv/de/videos/006785-000-A/the-element-of-crime/',
         'info_dict': {
-            'id': '110203-006-A',
-            'chapters': 'count:16',
-            'description': 'md5:cf592f1df52fe52007e3f8eac813c084',
-            'alt_title': 'Zaz',
-            'title': 'Baloise Session 2022',
-            'timestamp': 1668445200,
-            'duration': 4054,
-            'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/ubQjmVCGyRx3hmBuZEK9QZ/940x530',
-            'upload_date': '20221114',
+            'id': '006785-000-A',
+            'description': 'md5:c2f94fdfefc8a280e4dab68ab96ab0ba',
+            'title': 'The Element of Crime',
+            'timestamp': 1696111200,
+            'duration': 5849,
+            'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/q82dTTfyuCXupPsGxXsd7B/940x530',
+            'upload_date': '20230930',
             'ext': 'mp4',
-        },
-        'expected_warnings': ['geo restricted']
+        }
     }]
 
     _GEO_BYPASS = True
@@ -136,7 +127,9 @@ def _real_extract(self, url):
         lang = mobj.group('lang') or mobj.group('lang_2')
         langauge_code = self._LANG_MAP.get(lang)
 
-        config = self._download_json(f'{self._API_BASE}/config/{lang}/{video_id}', video_id)
+        config = self._download_json(f'{self._API_BASE}/config/{lang}/{video_id}', video_id, headers={
+            'x-validated-age': '18'
+        })
 
         geoblocking = traverse_obj(config, ('data', 'attributes', 'restriction', 'geoblocking')) or {}
         if geoblocking.get('restrictedArea'):

From 88a99c87b680ae59002534a517e191f46c42cbd4 Mon Sep 17 00:00:00 2001
From: Midnight Veil <midnightveil@fea.st>
Date: Tue, 10 Oct 2023 04:55:46 +1100
Subject: [PATCH 623/871] [ie/tenplay] Add support for seasons (#7939)

Closes #7744
Authored by: midnightveil
---
 yt_dlp/extractor/_extractors.py |  5 ++-
 yt_dlp/extractor/tenplay.py     | 58 +++++++++++++++++++++++++++++++--
 2 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 55c3c2f8e..6717a6039 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1992,7 +1992,10 @@
     WeTvSeriesIE,
 )
 from .tennistv import TennisTVIE
-from .tenplay import TenPlayIE
+from .tenplay import (
+    TenPlayIE,
+    TenPlaySeasonIE,
+)
 from .testurl import TestURLIE
 from .tf1 import TF1IE
 from .tfo import TFOIE
diff --git a/yt_dlp/extractor/tenplay.py b/yt_dlp/extractor/tenplay.py
index c7097cf02..7ce7cbf84 100644
--- a/yt_dlp/extractor/tenplay.py
+++ b/yt_dlp/extractor/tenplay.py
@@ -1,9 +1,11 @@
-from datetime import datetime
 import base64
+import functools
+import itertools
+from datetime import datetime
 
 from .common import InfoExtractor
 from ..networking import HEADRequest
-from ..utils import int_or_none, urlencode_postdata
+from ..utils import int_or_none, traverse_obj, urlencode_postdata, urljoin
 
 
 class TenPlayIE(InfoExtractor):
@@ -113,3 +115,55 @@ def _real_extract(self, url):
             'uploader': 'Channel 10',
             'uploader_id': '2199827728001',
         }
+
+
+class TenPlaySeasonIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?10play\.com\.au/(?P<show>[^/?#]+)/episodes/(?P<season>[^/?#]+)/?(?:$|[?#])'
+    _TESTS = [{
+        'url': 'https://10play.com.au/masterchef/episodes/season-14',
+        'info_dict': {
+            'title': 'Season 14',
+            'id': 'MjMyOTIy',
+        },
+        'playlist_mincount': 64,
+    }, {
+        'url': 'https://10play.com.au/the-bold-and-the-beautiful-fast-tracked/episodes/season-2022',
+        'info_dict': {
+            'title': 'Season 2022',
+            'id': 'Mjc0OTIw',
+        },
+        'playlist_mincount': 256,
+    }]
+
+    def _entries(self, load_more_url, display_id=None):
+        skip_ids = []
+        for page in itertools.count(1):
+            episodes_carousel = self._download_json(
+                load_more_url, display_id, query={'skipIds[]': skip_ids},
+                note=f'Fetching episodes page {page}')
+
+            episodes_chunk = episodes_carousel['items']
+            skip_ids.extend(ep['id'] for ep in episodes_chunk)
+
+            for ep in episodes_chunk:
+                yield ep['cardLink']
+            if not episodes_carousel['hasMore']:
+                break
+
+    def _real_extract(self, url):
+        show, season = self._match_valid_url(url).group('show', 'season')
+        season_info = self._download_json(
+            f'https://10play.com.au/api/shows/{show}/episodes/{season}', f'{show}/{season}')
+
+        episodes_carousel = traverse_obj(season_info, (
+            'content', 0, 'components', (
+                lambda _, v: v['title'].lower() == 'episodes',
+                (..., {dict}),
+            )), get_all=False) or {}
+
+        playlist_id = episodes_carousel['tpId']
+
+        return self.playlist_from_matches(
+            self._entries(urljoin(url, episodes_carousel['loadMoreUrl']), playlist_id),
+            playlist_id, traverse_obj(season_info, ('content', 0, 'title', {str})),
+            getter=functools.partial(urljoin, url))

From 4de94b9e165bfd6421a692f5f2eabcdb08edcb71 Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Mon, 9 Oct 2023 19:00:26 +0100
Subject: [PATCH 624/871] [ie/nhk] Fix Japanese-language VOD extraction (#8309)

Closes #8303
Authored by: garret1317
---
 yt_dlp/extractor/nhk.py | 68 ++++++++++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 28 deletions(-)

diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py
index bcbc2279f..f6b5c501b 100644
--- a/yt_dlp/extractor/nhk.py
+++ b/yt_dlp/extractor/nhk.py
@@ -68,11 +68,12 @@ def _extract_formats_and_subtitles(self, vod_id):
 
     def _extract_episode_info(self, url, episode=None):
         fetch_episode = episode is None
-        lang, m_type, episode_id = NhkVodIE._match_valid_url(url).groups()
-        if len(episode_id) == 7:
+        lang, m_type, episode_id = NhkVodIE._match_valid_url(url).group('lang', 'type', 'id')
+        is_video = m_type == 'video'
+
+        if is_video:
             episode_id = episode_id[:4] + '-' + episode_id[4:]
 
-        is_video = m_type == 'video'
         if fetch_episode:
             episode = self._call_api(
                 episode_id, lang, is_video, True, episode_id[:4] == '9999')[0]
@@ -133,47 +134,46 @@ def get_clean_field(key):
 
 class NhkVodIE(NhkBaseIE):
     # the 7-character IDs can have alphabetic chars too: assume [a-z] rather than just [a-f], eg
-    _VALID_URL = r'%s%s(?P<id>[0-9a-z]{7}|[^/]+?-\d{8}-[0-9a-z]+)' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
+    _VALID_URL = [rf'{NhkBaseIE._BASE_URL_REGEX}/(?P<type>video)/(?P<id>[0-9a-z]+)',
+                  rf'{NhkBaseIE._BASE_URL_REGEX}/(?P<type>audio)/(?P<id>[^/?#]+?-\d{{8}}-[0-9a-z]+)']
     # Content available only for a limited period of time. Visit
     # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
     _TESTS = [{
-        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2061601/',
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2049126/',
         'info_dict': {
-            'id': 'yd8322ch',
+            'id': 'nw_vod_v_en_2049_126_20230413233000_01_1681398302',
             'ext': 'mp4',
-            'description': 'md5:109c8b05d67a62d0592f2b445d2cd898',
-            'title': 'GRAND SUMO Highlights - [Recap] May Tournament Day 1 (Opening Day)',
-            'upload_date': '20230514',
-            'timestamp': 1684083791,
-            'series': 'GRAND SUMO Highlights',
-            'episode': '[Recap] May Tournament Day 1 (Opening Day)',
-            'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1684084443/4028649.jpg?w=1920&h=1080',
+            'title': 'Japan Railway Journal - The Tohoku Shinkansen: Full Speed Ahead',
+            'description': 'md5:49f7c5b206e03868a2fdf0d0814b92f6',
+            'thumbnail': 'md5:51bcef4a21936e7fea1ff4e06353f463',
+            'episode': 'The Tohoku Shinkansen: Full Speed Ahead',
+            'series': 'Japan Railway Journal',
         },
     }, {
         # video clip
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
-        'md5': '7a90abcfe610ec22a6bfe15bd46b30ca',
+        'md5': '153c3016dfd252ba09726588149cf0e7',
         'info_dict': {
-            'id': 'a95j5iza',
+            'id': 'lpZXIwaDE6_Z-976CPsFdxyICyWUzlT5',
             'ext': 'mp4',
-            'title': "Dining with the Chef - Chef Saito's Family recipe: MENCHI-KATSU",
+            'title': 'Dining with the Chef - Chef Saito\'s Family recipe: MENCHI-KATSU',
             'description': 'md5:5aee4a9f9d81c26281862382103b0ea5',
-            'timestamp': 1565965194,
-            'upload_date': '20190816',
-            'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1567086278/3715195.jpg?w=1920&h=1080',
+            'thumbnail': 'md5:d6a4d9b6e9be90aaadda0bcce89631ed',
             'series': 'Dining with the Chef',
             'episode': 'Chef Saito\'s Family recipe: MENCHI-KATSU',
         },
     }, {
-        # audio clip
-        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/r_inventions-20201104-1/',
+        # radio
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/livinginjapan-20231001-1/',
         'info_dict': {
-            'id': 'r_inventions-20201104-1-en',
+            'id': 'livinginjapan-20231001-1-en',
             'ext': 'm4a',
-            'title': "Japan's Top Inventions - Miniature Video Cameras",
-            'description': 'md5:07ea722bdbbb4936fdd360b6a480c25b',
+            'title': 'Living in Japan - Tips for Travelers to Japan / Ramen Vending Machines',
+            'series': 'Living in Japan',
+            'description': 'md5:850611969932874b4a3309e0cae06c2f',
+            'thumbnail': 'md5:960622fb6e06054a4a1a0c97ea752545',
+            'episode': 'Tips for Travelers to Japan / Ramen Vending Machines'
         },
-        'skip': '404 Not Found',
     }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/',
         'only_matching': True,
@@ -199,6 +199,19 @@ class NhkVodIE(NhkBaseIE):
             'timestamp': 1623722008,
         },
         'skip': '404 Not Found',
+    }, {
+        # japanese-language, longer id than english
+        'url': 'https://www3.nhk.or.jp/nhkworld/ja/ondemand/video/0020271111/',
+        'info_dict': {
+            'id': 'nw_ja_v_jvod_ohayou_20231008',
+            'ext': 'mp4',
+            'title': 'おはよう日本（7時台） - 10月8日放送',
+            'series': 'おはよう日本（7時台）',
+            'episode': '10月8日放送',
+            'thumbnail': 'md5:d733b1c8e965ab68fb02b2d347d0e9b4',
+            'description': 'md5:9c1d6cbeadb827b955b20e99ab920ff0',
+        },
+        'skip': 'expires 2023-10-15',
     }]
 
     def _real_extract(self, url):
@@ -206,7 +219,7 @@ def _real_extract(self, url):
 
 
 class NhkVodProgramIE(NhkBaseIE):
-    _VALID_URL = r'%s/program%s(?P<id>[0-9a-z]+)(?:.+?\btype=(?P<episode_type>clip|(?:radio|tv)Episode))?' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
+    _VALID_URL = rf'{NhkBaseIE._BASE_URL_REGEX}/program{NhkBaseIE._TYPE_REGEX}(?P<id>\w+)(?:.+?\btype=(?P<episode_type>clip|(?:radio|tv)Episode))?'
     _TESTS = [{
         # video program episodes
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/sumo',
@@ -240,8 +253,7 @@ class NhkVodProgramIE(NhkBaseIE):
     }]
 
     def _real_extract(self, url):
-        lang, m_type, program_id, episode_type = self._match_valid_url(url).groups()
-
+        lang, m_type, program_id, episode_type = self._match_valid_url(url).group('lang', 'type', 'id', 'episode_type')
         episodes = self._call_api(
             program_id, lang, m_type == 'video', False, episode_type == 'clip')
 

From 84e26038d4002e763ea51ca1bdce4f7e63c540bf Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Mon, 9 Oct 2023 13:30:36 -0500
Subject: [PATCH 625/871] [utils] `write_xattr`: Use `os.setxattr` if available
 (#8205)

Closes #8193
Authored by: bashonly, Grub4K

Co-authored-by: Simon Sawicki <contact@grub4k.xyz>
---
 README.md              | 2 +-
 yt_dlp/utils/_utils.py | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index a0b69c9a1..a26482faa 100644
--- a/README.md
+++ b/README.md
@@ -280,7 +280,7 @@ ### Metadata
 
 * [**mutagen**](https://github.com/quodlibet/mutagen)\* - For `--embed-thumbnail` in certain formats. Licensed under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
 * [**AtomicParsley**](https://github.com/wez/atomicparsley) - For `--embed-thumbnail` in `mp4`/`m4a` files when `mutagen`/`ffmpeg` cannot. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
-* [**xattr**](https://github.com/xattr/xattr), [**pyxattr**](https://github.com/iustin/pyxattr) or [**setfattr**](http://savannah.nongnu.org/projects/attr) - For writing xattr metadata (`--xattr`) on **Linux**. Licensed under [MIT](https://github.com/xattr/xattr/blob/master/LICENSE.txt), [LGPL2.1](https://github.com/iustin/pyxattr/blob/master/COPYING) and [GPLv2+](http://git.savannah.nongnu.org/cgit/attr.git/tree/doc/COPYING) respectively
+* [**xattr**](https://github.com/xattr/xattr), [**pyxattr**](https://github.com/iustin/pyxattr) or [**setfattr**](http://savannah.nongnu.org/projects/attr) - For writing xattr metadata (`--xattr`) on **Mac** and **BSD**. Licensed under [MIT](https://github.com/xattr/xattr/blob/master/LICENSE.txt), [LGPL2.1](https://github.com/iustin/pyxattr/blob/master/COPYING) and [GPLv2+](http://git.savannah.nongnu.org/cgit/attr.git/tree/doc/COPYING) respectively
 
 ### Misc
 
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 3dc17bf59..10c7c4311 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -4441,10 +4441,12 @@ def write_xattr(path, key, value):
             raise XAttrMetadataError(e.errno, e.strerror)
         return
 
-    # UNIX Method 1. Use xattrs/pyxattrs modules
+    # UNIX Method 1. Use os.setxattr/xattrs/pyxattrs modules
 
     setxattr = None
-    if getattr(xattr, '_yt_dlp__identifier', None) == 'pyxattr':
+    if callable(getattr(os, 'setxattr', None)):
+        setxattr = os.setxattr
+    elif getattr(xattr, '_yt_dlp__identifier', None) == 'pyxattr':
         # Unicode arguments are not supported in pyxattr until version 0.5.0
         # See https://github.com/ytdl-org/youtube-dl/issues/5498
         if version_tuple(xattr.__version__) >= (0, 5, 0):

From feebf6d02fc9651331eee2af5e08e6112288163b Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Thu, 12 Oct 2023 12:20:52 +0200
Subject: [PATCH 626/871] [ie/youtube] Fix bug with `--extractor-retries inf`
 (#8328)

Authored by: Grub4K
---
 yt_dlp/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index b7ac3e9cc..c5be36636 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -947,7 +947,10 @@ def _extract_response(self, item_id, query, note='Downloading API JSON', headers
         icd_rm = next(icd_retries)
         main_retries = iter(self.RetryManager())
         main_rm = next(main_retries)
-        for _ in range(main_rm.retries + icd_rm.retries + 1):
+        # Manual retry loop for multiple RetryManagers
+        # The proper RetryManager MUST be advanced after an error
+        # and it's result MUST be checked if the manager is non fatal
+        while True:
             try:
                 response = self._call_api(
                     ep=ep, fatal=True, headers=headers,

From b9316642313bbc9e209ac0d2276d37ba60bceb49 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Fri, 13 Oct 2023 14:23:39 -0500
Subject: [PATCH 627/871] [ie/radiko] Fix bug with `downloader_options`

Closes #8333
Authored by: bashonly
---
 yt_dlp/extractor/radiko.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/radiko.py b/yt_dlp/extractor/radiko.py
index 8c8fb1a8f..c363d9ba5 100644
--- a/yt_dlp/extractor/radiko.py
+++ b/yt_dlp/extractor/radiko.py
@@ -154,7 +154,7 @@ def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token,
                     sf['preference'] = -100
                     sf['format_note'] = 'not preferred'
                 if not is_onair and timefree_int == 1 and time_to_skip:
-                    sf['downloader_options'] = {'ffmpeg_args': ['-ss', time_to_skip]}
+                    sf['downloader_options'] = {'ffmpeg_args': ['-ss', str(time_to_skip)]}
             formats.extend(subformats)
 
         return formats

From e030b6b6fba7b2f4614ad2ab9f7649d40a2dd305 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Fri, 13 Oct 2023 21:29:56 +0200
Subject: [PATCH 628/871] [ie/mbn] Add extractor (#8312)

Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/mbn.py         | 89 +++++++++++++++++++++++++++++++++
 2 files changed, 90 insertions(+)
 create mode 100644 yt_dlp/extractor/mbn.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 6717a6039..45073628c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1053,6 +1053,7 @@
 from .massengeschmacktv import MassengeschmackTVIE
 from .masters import MastersIE
 from .matchtv import MatchTVIE
+from .mbn import MBNIE
 from .mdr import MDRIE
 from .medaltv import MedalTVIE
 from .mediaite import MediaiteIE
diff --git a/yt_dlp/extractor/mbn.py b/yt_dlp/extractor/mbn.py
new file mode 100644
index 000000000..4917c4698
--- /dev/null
+++ b/yt_dlp/extractor/mbn.py
@@ -0,0 +1,89 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class MBNIE(InfoExtractor):
+    IE_DESC = 'mbn.co.kr (매일방송)'
+    _VALID_URL = r'https?://(?:www\.)?mbn\.co\.kr/vod/programContents/preview(?:list)?/\d+/\d+/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://mbn.co.kr/vod/programContents/previewlist/861/5433/1276155',
+        'md5': '85e1694e5b247c04d1386b7e3c90fd76',
+        'info_dict': {
+            'id': '1276155',
+            'ext': 'mp4',
+            'title': '결국 사로잡힌 권유리, 그녀를 목숨 걸고 구하려는 정일우!',
+            'duration': 3891,
+            'release_date': '20210703',
+            'thumbnail': 'http://img.vod.mbn.co.kr/mbnvod2img/861/2021/07/03/20210703230811_20_861_1276155_360_7_0.jpg',
+            'series': '보쌈 - 운명을 훔치다',
+            'episode': 'Episode 19',
+            'episode_number': 19,
+        },
+    }, {
+        'url': 'https://www.mbn.co.kr/vod/programContents/previewlist/835/5294/1084744',
+        'md5': 'fc65d3aac85e85e0b5056f4ef99cde4a',
+        'info_dict': {
+            'id': '1084744',
+            'ext': 'mp4',
+            'title': '김정은♥최원영, 제자리를 찾은 위험한 부부! ＂결혼은 투쟁이면서, 어려운 방식이야..＂',
+            'duration': 93,
+            'release_date': '20201124',
+            'thumbnail': 'http://img.vod.mbn.co.kr/mbnvod2img/835/2020/11/25/20201125000221_21_835_1084744_360_7_0.jpg',
+            'series': '나의 위험한 아내',
+        },
+    }, {
+        'url': 'https://www.mbn.co.kr/vod/programContents/preview/952/6088/1054797?next=1',
+        'md5': 'c711103c72aeac8323a5cf1751f10097',
+        'info_dict': {
+            'id': '1054797',
+            'ext': 'mp4',
+            'title': '[2차 티저] MBN 주말 미니시리즈 <완벽한 결혼의 정석> l 그녀에게 주어진 두 번째 인생',
+            'duration': 65,
+            'release_date': '20231028',
+            'thumbnail': 'http://img.vod.mbn.co.kr/vod2/952/2023/09/11/20230911130223_22_952_1054797_1080_7.jpg',
+            'series': '완벽한 결혼의 정석',
+        },
+    }]
+
+    def _real_extract(self, url):
+        content_id = self._match_id(url)
+        webpage = self._download_webpage(url, content_id)
+
+        content_cls_cd = self._search_regex(
+            r'"\?content_cls_cd=(\d+)&', webpage, 'content cls cd', fatal=False) or '20'
+        media_info = self._download_json(
+            'https://www.mbn.co.kr/player/mbnVodPlayer_2020.mbn', content_id,
+            note='Fetching playback data', query={
+                'content_cls_cd': content_cls_cd,
+                'content_id': content_id,
+                'relay_type': '1',
+            })
+
+        formats = []
+        for stream_url in traverse_obj(media_info, ('movie_list', ..., 'url', {url_or_none})):
+            stream_url = re.sub(r'/(?:chunk|play)list(?:_pd\d+)?\.m3u8', '/manifest.m3u8', stream_url)
+            final_url = url_or_none(self._download_webpage(
+                f'https://www.mbn.co.kr/player/mbnStreamAuth_new_vod.mbn?vod_url={stream_url}',
+                content_id, note='Fetching authenticated m3u8 url'))
+
+            formats.extend(self._extract_m3u8_formats(final_url, content_id, fatal=False))
+
+        return {
+            'id': content_id,
+            **traverse_obj(media_info, {
+                'title': ('movie_title', {str}),
+                'duration': ('play_sec', {int_or_none}),
+                'release_date': ('bcast_date', {lambda x: x.replace('.', '')}, {unified_strdate}),
+                'thumbnail': ('movie_start_Img', {url_or_none}),
+                'series': ('prog_nm', {str}),
+                'episode_number': ('ad_contentnumber', {int_or_none}),
+            }),
+            'formats': formats,
+        }

From b286ec68f1f28798b3e371f888a2ed97d399cf77 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Fri, 13 Oct 2023 21:30:24 +0200
Subject: [PATCH 629/871] [ie/jtbc] Add extractors (#8314)

Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/jtbc.py        | 156 ++++++++++++++++++++++++++++++++
 2 files changed, 160 insertions(+)
 create mode 100644 yt_dlp/extractor/jtbc.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 45073628c..ca4571182 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -896,6 +896,10 @@
 from .jove import JoveIE
 from .joj import JojIE
 from .jstream import JStreamIE
+from .jtbc import (
+    JTBCIE,
+    JTBCProgramIE,
+)
 from .jwplatform import JWPlatformIE
 from .kakao import KakaoIE
 from .kaltura import KalturaIE
diff --git a/yt_dlp/extractor/jtbc.py b/yt_dlp/extractor/jtbc.py
new file mode 100644
index 000000000..573f7492f
--- /dev/null
+++ b/yt_dlp/extractor/jtbc.py
@@ -0,0 +1,156 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class JTBCIE(InfoExtractor):
+    IE_DESC = 'jtbc.co.kr'
+    _VALID_URL = r'''(?x)
+        https?://(?:
+            vod\.jtbc\.co\.kr/player/(?:program|clip)
+            |tv\.jtbc\.co\.kr/(?:replay|trailer|clip)/pr\d+/pm\d+
+        )/(?P<id>(?:ep|vo)\d+)'''
+    _GEO_COUNTRIES = ['KR']
+
+    _TESTS = [{
+        'url': 'https://tv.jtbc.co.kr/replay/pr10011629/pm10067930/ep20216321/view',
+        'md5': 'e6ade71d8c8685bbfd6e6ce4167c6a6c',
+        'info_dict': {
+            'id': 'VO10721192',
+            'display_id': 'ep20216321',
+            'ext': 'mp4',
+            'title': '힘쎈여자 강남순 2회 다시보기',
+            'description': 'md5:043c1d9019100ce271dba09995dbd1e2',
+            'duration': 3770.0,
+            'release_date': '20231008',
+            'age_limit': 15,
+            'thumbnail': 'https://fs.jtbc.co.kr//joydata/CP00000001/prog/drama/stronggirlnamsoon/img/20231008_163541_522_1.jpg',
+            'series': '힘쎈여자 강남순',
+        },
+    }, {
+        'url': 'https://vod.jtbc.co.kr/player/program/ep20216733',
+        'md5': '217a6d190f115a75e4bda0ceaa4cd7f4',
+        'info_dict': {
+            'id': 'VO10721429',
+            'display_id': 'ep20216733',
+            'ext': 'mp4',
+            'title': '헬로 마이 닥터 친절한 진료실 149회 다시보기',
+            'description': 'md5:1d70788a982dd5de26874a92fcffddb8',
+            'duration': 2720.0,
+            'release_date': '20231009',
+            'age_limit': 15,
+            'thumbnail': 'https://fs.jtbc.co.kr//joydata/CP00000001/prog/culture/hellomydoctor/img/20231009_095002_528_1.jpg',
+            'series': '헬로 마이 닥터 친절한 진료실',
+        },
+    }, {
+        'url': 'https://vod.jtbc.co.kr/player/clip/vo10721270',
+        'md5': '05782e2dc22a9c548aebefe62ae4328a',
+        'info_dict': {
+            'id': 'VO10721270',
+            'display_id': 'vo10721270',
+            'ext': 'mp4',
+            'title': '뭉쳐야 찬다3 2회 예고편 - A매치로 향하는 마지막 관문💥',
+            'description': 'md5:d48b51a8655c84843b4ed8d0c39aae68',
+            'duration': 46.0,
+            'release_date': '20231015',
+            'age_limit': 15,
+            'thumbnail': 'https://fs.jtbc.co.kr//joydata/CP00000001/prog/enter/soccer3/img/20231008_210957_775_1.jpg',
+            'series': '뭉쳐야 찬다3',
+        },
+    }, {
+        'url': 'https://tv.jtbc.co.kr/trailer/pr10010392/pm10032526/vo10720912/view',
+        'md5': '367d480eb3ef54a9cd7a4b4d69c4b32d',
+        'info_dict': {
+            'id': 'VO10720912',
+            'display_id': 'vo10720912',
+            'ext': 'mp4',
+            'title': '아는 형님 404회 예고편 | 10월 14일(토) 저녁 8시 50분 방송!',
+            'description': 'md5:2743bb1079ceb85bb00060f2ad8f0280',
+            'duration': 148.0,
+            'release_date': '20231014',
+            'age_limit': 15,
+            'thumbnail': 'https://fs.jtbc.co.kr//joydata/CP00000001/prog/enter/jtbcbros/img/20231006_230023_802_1.jpg',
+            'series': '아는 형님',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        if display_id.startswith('vo'):
+            video_id = display_id.upper()
+        else:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._search_regex(r'data-vod="(VO\d+)"', webpage, 'vod id')
+
+        playback_data = self._download_json(
+            f'https://api.jtbc.co.kr/vod/{video_id}', video_id, note='Downloading VOD playback data')
+
+        subtitles = {}
+        for sub in traverse_obj(playback_data, ('tracks', lambda _, v: v['file'])):
+            subtitles.setdefault(sub.get('label', 'und'), []).append({'url': sub['file']})
+
+        formats = []
+        for stream_url in traverse_obj(playback_data, ('sources', 'HLS', ..., 'file', {url_or_none})):
+            stream_url = re.sub(r'/playlist(?:_pd\d+)?\.m3u8', '/index.m3u8', stream_url)
+            formats.extend(self._extract_m3u8_formats(stream_url, video_id, fatal=False))
+
+        metadata = self._download_json(
+            'https://now-api.jtbc.co.kr/v1/vod/detail', video_id,
+            note='Downloading mobile details', fatal=False, query={'vodFileId': video_id})
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            **traverse_obj(metadata, ('vodDetail', {
+                'title': 'vodTitleView',
+                'series': 'programTitle',
+                'age_limit': ('watchAge', {int_or_none}),
+                'release_date': ('broadcastDate', {lambda x: re.match(r'\d{8}', x.replace('.', ''))}, 0),
+                'description': 'episodeContents',
+                'thumbnail': ('imgFileUrl', {url_or_none}),
+            })),
+            'duration': parse_duration(playback_data.get('playTime')),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+
+class JTBCProgramIE(InfoExtractor):
+    IE_NAME = 'JTBC:program'
+    _VALID_URL = r'https?://(?:vod\.jtbc\.co\.kr/program|tv\.jtbc\.co\.kr/replay)/(?P<id>pr\d+)/(?:replay|pm\d+)/?(?:$|[?#])'
+
+    _TESTS = [{
+        'url': 'https://tv.jtbc.co.kr/replay/pr10010392/pm10032710',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'pr10010392',
+        },
+        'playlist_count': 398,
+    }, {
+        'url': 'https://vod.jtbc.co.kr/program/pr10011491/replay',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'pr10011491',
+        },
+        'playlist_count': 59,
+    }]
+
+    def _real_extract(self, url):
+        program_id = self._match_id(url)
+
+        vod_list = self._download_json(
+            'https://now-api.jtbc.co.kr/v1/vodClip/programHome/programReplayVodList', program_id,
+            note='Downloading program replay list', query={
+                'programId': program_id,
+                'rowCount': '10000',
+            })
+
+        entries = [self.url_result(f'https://vod.jtbc.co.kr/player/program/{video_id}', JTBCIE, video_id)
+                   for video_id in traverse_obj(vod_list, ('programReplayVodList', ..., 'episodeId'))]
+        return self.playlist_result(entries, program_id)

From 2acd1d555ef89851c73773776715d3de9a0e30b9 Mon Sep 17 00:00:00 2001
From: Riteo <riteo@posteo.net>
Date: Fri, 13 Oct 2023 22:01:39 +0200
Subject: [PATCH 630/871] [core] Ensure thumbnail output directory exists
 (#7985)

Closes #8203
Authored by: Riteo
---
 yt_dlp/YoutubeDL.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index f322b12a2..71d17ac01 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -4221,7 +4221,7 @@ def _write_subtitles(self, info_dict, filename):
         return ret
 
     def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None):
-        ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename) '''
+        ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename); or None if error '''
         write_all = self.params.get('write_all_thumbnails', False)
         thumbnails, ret = [], []
         if write_all or self.params.get('writethumbnail', False):
@@ -4237,6 +4237,9 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
             self.write_debug(f'Skipping writing {label} thumbnail')
             return ret
 
+        if not self._ensure_dir_exists(filename):
+            return None
+
         for idx, t in list(enumerate(thumbnails))[::-1]:
             thumb_ext = (f'{t["id"]}.' if multiple else '') + determine_ext(t['url'], 'jpg')
             thumb_display_id = f'{label} thumbnail {t["id"]}'

From b634ba742d8f38ce9ecfa0546485728b0c6c59d1 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 13 Oct 2023 17:15:35 -0500
Subject: [PATCH 631/871] [cleanup] Misc (#8338)

Authored by: bashonly, gamer191
---
 README.md                         |  3 +--
 yt_dlp/extractor/banbye.py        |  4 ++--
 yt_dlp/extractor/breitbart.py     |  2 +-
 yt_dlp/extractor/craftsy.py       |  2 +-
 yt_dlp/extractor/cybrary.py       |  4 ++--
 yt_dlp/extractor/fifa.py          |  2 +-
 yt_dlp/extractor/filmmodu.py      |  2 +-
 yt_dlp/extractor/itprotv.py       |  4 ++--
 yt_dlp/extractor/jable.py         |  4 ++--
 yt_dlp/extractor/kommunetv.py     |  2 +-
 yt_dlp/extractor/mainstreaming.py |  2 +-
 yt_dlp/extractor/mediaite.py      |  2 +-
 yt_dlp/extractor/mocha.py         |  2 +-
 yt_dlp/extractor/nfl.py           |  4 ++--
 yt_dlp/extractor/novaplay.py      |  2 +-
 yt_dlp/extractor/nubilesporn.py   |  2 +-
 yt_dlp/extractor/oftv.py          |  4 ++--
 yt_dlp/extractor/sina.py          |  2 +-
 yt_dlp/extractor/twitter.py       |  2 +-
 yt_dlp/extractor/utreon.py        |  2 +-
 yt_dlp/extractor/vk.py            |  4 ++--
 yt_dlp/extractor/weverse.py       | 12 ++++++------
 yt_dlp/extractor/wimtv.py         |  2 +-
 yt_dlp/extractor/xhamster.py      |  4 ++--
 yt_dlp/extractor/youtube.py       |  2 +-
 yt_dlp/extractor/zoom.py          |  2 +-
 yt_dlp/options.py                 |  2 +-
 27 files changed, 40 insertions(+), 41 deletions(-)

diff --git a/README.md b/README.md
index a26482faa..dd4652d43 100644
--- a/README.md
+++ b/README.md
@@ -89,7 +89,6 @@ # NEW FEATURES
     * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326) **\***
     * Supports some (but not all) age-gated content without cookies
     * Download livestreams from the start using `--live-from-start` (*experimental*)
-    * `255kbps` audio is extracted (if available) from YouTube Music when premium cookies are given
     * Channel URLs download all uploads of the channel, including shorts and live
 
 * **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[+KEYRING][:PROFILE][::CONTAINER]`
@@ -913,7 +912,7 @@ ## Authentication Options:
                                     Defaults to ~/.netrc
     --netrc-cmd NETRC_CMD           Command to execute to get the credentials
                                     for an extractor.
-    --video-password PASSWORD       Video password (vimeo, youku)
+    --video-password PASSWORD       Video-specific password
     --ap-mso MSO                    Adobe Pass multiple-system operator (TV
                                     provider) identifier, use --ap-list-mso for
                                     a list of available MSOs
diff --git a/yt_dlp/extractor/banbye.py b/yt_dlp/extractor/banbye.py
index e0fc93b97..dfcc82f02 100644
--- a/yt_dlp/extractor/banbye.py
+++ b/yt_dlp/extractor/banbye.py
@@ -31,7 +31,7 @@ def _extract_playlist(self, playlist_id):
 
 
 class BanByeIE(BanByeBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?banbye.com/(?:en/)?watch/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?banbye\.com/(?:en/)?watch/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://banbye.com/watch/v_ytfmvkVYLE8T',
         'md5': '2f4ea15c5ca259a73d909b2cfd558eb5',
@@ -120,7 +120,7 @@ def _real_extract(self, url):
 
 
 class BanByeChannelIE(BanByeBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?banbye.com/(?:en/)?channel/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?banbye\.com/(?:en/)?channel/(?P<id>\w+)'
     _TESTS = [{
         'url': 'https://banbye.com/channel/ch_wrealu24',
         'info_dict': {
diff --git a/yt_dlp/extractor/breitbart.py b/yt_dlp/extractor/breitbart.py
index ea0a59c86..b5abb7f19 100644
--- a/yt_dlp/extractor/breitbart.py
+++ b/yt_dlp/extractor/breitbart.py
@@ -2,7 +2,7 @@
 
 
 class BreitBartIE(InfoExtractor):
-    _VALID_URL = r'https?:\/\/(?:www\.)breitbart.com/videos/v/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?breitbart\.com/videos/v/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.breitbart.com/videos/v/5cOz1yup/?pl=Ij6NDOji',
         'md5': '0aa6d1d6e183ac5ca09207fe49f17ade',
diff --git a/yt_dlp/extractor/craftsy.py b/yt_dlp/extractor/craftsy.py
index 307bfb946..5d3733143 100644
--- a/yt_dlp/extractor/craftsy.py
+++ b/yt_dlp/extractor/craftsy.py
@@ -10,7 +10,7 @@
 
 
 class CraftsyIE(InfoExtractor):
-    _VALID_URL = r'https?://www.craftsy.com/class/(?P<id>[a-z0-9_-]+)/'
+    _VALID_URL = r'https?://www\.craftsy\.com/class/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://www.craftsy.com/class/the-midnight-quilt-show-season-5/',
         'info_dict': {
diff --git a/yt_dlp/extractor/cybrary.py b/yt_dlp/extractor/cybrary.py
index 73f2439b3..aeffe93b4 100644
--- a/yt_dlp/extractor/cybrary.py
+++ b/yt_dlp/extractor/cybrary.py
@@ -45,7 +45,7 @@ def _get_vimeo_id(self, activity_id):
 
 
 class CybraryIE(CybraryBaseIE):
-    _VALID_URL = r'https?://app.cybrary.it/immersive/(?P<enrollment>[0-9]+)/activity/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://app\.cybrary\.it/immersive/(?P<enrollment>[0-9]+)/activity/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'https://app.cybrary.it/immersive/12487950/activity/63102',
         'md5': '9ae12d37e555cb2ed554223a71a701d0',
@@ -110,7 +110,7 @@ def _real_extract(self, url):
 
 
 class CybraryCourseIE(CybraryBaseIE):
-    _VALID_URL = r'https://app.cybrary.it/browse/course/(?P<id>[\w-]+)/?(?:$|[#?])'
+    _VALID_URL = r'https://app\.cybrary\.it/browse/course/(?P<id>[\w-]+)/?(?:$|[#?])'
     _TESTS = [{
         'url': 'https://app.cybrary.it/browse/course/az-500-microsoft-azure-security-technologies',
         'info_dict': {
diff --git a/yt_dlp/extractor/fifa.py b/yt_dlp/extractor/fifa.py
index 8b4db3a8a..f604cbd40 100644
--- a/yt_dlp/extractor/fifa.py
+++ b/yt_dlp/extractor/fifa.py
@@ -8,7 +8,7 @@
 
 
 class FifaIE(InfoExtractor):
-    _VALID_URL = r'https?://www.fifa.com/fifaplus/(?P<locale>\w{2})/watch/([^#?]+/)?(?P<id>\w+)'
+    _VALID_URL = r'https?://www\.fifa\.com/fifaplus/(?P<locale>\w{2})/watch/([^#?]+/)?(?P<id>\w+)'
     _TESTS = [{
         'url': 'https://www.fifa.com/fifaplus/en/watch/7on10qPcnyLajDDU3ntg6y',
         'info_dict': {
diff --git a/yt_dlp/extractor/filmmodu.py b/yt_dlp/extractor/filmmodu.py
index 9eb550eed..1e793560d 100644
--- a/yt_dlp/extractor/filmmodu.py
+++ b/yt_dlp/extractor/filmmodu.py
@@ -3,7 +3,7 @@
 
 
 class FilmmoduIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www.)?filmmodu.org/(?P<id>[^/]+-(?:turkce-dublaj-izle|altyazili-izle))'
+    _VALID_URL = r'https?://(?:www\.)?filmmodu\.org/(?P<id>[^/]+-(?:turkce-dublaj-izle|altyazili-izle))'
     _TESTS = [{
         'url': 'https://www.filmmodu.org/f9-altyazili-izle',
         'md5': 'aeefd955c2a508a5bdaa3bcec8eeb0d4',
diff --git a/yt_dlp/extractor/itprotv.py b/yt_dlp/extractor/itprotv.py
index 4ac12603a..b9d5c196d 100644
--- a/yt_dlp/extractor/itprotv.py
+++ b/yt_dlp/extractor/itprotv.py
@@ -31,7 +31,7 @@ def _check_if_logged_in(self, webpage):
 
 
 class ITProTVIE(ITProTVBaseIE):
-    _VALID_URL = r'https://app.itpro.tv/course/(?P<course>[\w-]+)/(?P<id>[\w-]+)'
+    _VALID_URL = r'https://app\.itpro\.tv/course/(?P<course>[\w-]+)/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://app.itpro.tv/course/guided-tour/introductionitprotv',
         'md5': 'bca4a28c2667fd1a63052e71a94bb88c',
@@ -102,7 +102,7 @@ def _real_extract(self, url):
 
 
 class ITProTVCourseIE(ITProTVBaseIE):
-    _VALID_URL = r'https?://app.itpro.tv/course/(?P<id>[\w-]+)/?(?:$|[#?])'
+    _VALID_URL = r'https?://app\.itpro\.tv/course/(?P<id>[\w-]+)/?(?:$|[#?])'
     _TESTS = [
         {
             'url': 'https://app.itpro.tv/course/guided-tour',
diff --git a/yt_dlp/extractor/jable.py b/yt_dlp/extractor/jable.py
index 84c3225e4..71fed49ea 100644
--- a/yt_dlp/extractor/jable.py
+++ b/yt_dlp/extractor/jable.py
@@ -10,7 +10,7 @@
 
 
 class JableIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?jable.tv/videos/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?jable\.tv/videos/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://jable.tv/videos/pppd-812/',
         'md5': 'f1537283a9bc073c31ff86ca35d9b2a6',
@@ -64,7 +64,7 @@ def _real_extract(self, url):
 
 
 class JablePlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?jable.tv/(?:categories|models|tags)/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?jable\.tv/(?:categories|models|tags)/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://jable.tv/models/kaede-karen/',
         'info_dict': {
diff --git a/yt_dlp/extractor/kommunetv.py b/yt_dlp/extractor/kommunetv.py
index e21e556be..a30905b57 100644
--- a/yt_dlp/extractor/kommunetv.py
+++ b/yt_dlp/extractor/kommunetv.py
@@ -3,7 +3,7 @@
 
 
 class KommunetvIE(InfoExtractor):
-    _VALID_URL = r'https://(\w+).kommunetv.no/archive/(?P<id>\w+)'
+    _VALID_URL = r'https://\w+\.kommunetv\.no/archive/(?P<id>\w+)'
     _TEST = {
         'url': 'https://oslo.kommunetv.no/archive/921',
         'md5': '5f102be308ee759be1e12b63d5da4bbc',
diff --git a/yt_dlp/extractor/mainstreaming.py b/yt_dlp/extractor/mainstreaming.py
index fe5589d59..fd9bba8bc 100644
--- a/yt_dlp/extractor/mainstreaming.py
+++ b/yt_dlp/extractor/mainstreaming.py
@@ -13,7 +13,7 @@
 
 
 class MainStreamingIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:webtools-?)?(?P<host>[A-Za-z0-9-]*\.msvdn.net)/(?:embed|amp_embed|content)/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:webtools-?)?(?P<host>[A-Za-z0-9-]*\.msvdn\.net)/(?:embed|amp_embed|content)/(?P<id>\w+)'
     _EMBED_REGEX = [rf'<iframe[^>]+?src=["\']?(?P<url>{_VALID_URL})["\']?']
     IE_DESC = 'MainStreaming Player'
 
diff --git a/yt_dlp/extractor/mediaite.py b/yt_dlp/extractor/mediaite.py
index ab253920b..32887cbde 100644
--- a/yt_dlp/extractor/mediaite.py
+++ b/yt_dlp/extractor/mediaite.py
@@ -2,7 +2,7 @@
 
 
 class MediaiteIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mediaite.com(?!/category)(?:/[\w-]+){2}'
+    _VALID_URL = r'https?://(?:www\.)?mediaite\.com(?!/category)(?:/[\w-]+){2}'
     _TESTS = [{
         'url': 'https://www.mediaite.com/sports/bill-burr-roasts-nfl-for-promoting-black-lives-matter-while-scheduling-more-games-after-all-the-sht-they-know-about-cte/',
         'info_dict': {
diff --git a/yt_dlp/extractor/mocha.py b/yt_dlp/extractor/mocha.py
index 5f72b810b..2fbc0e911 100644
--- a/yt_dlp/extractor/mocha.py
+++ b/yt_dlp/extractor/mocha.py
@@ -3,7 +3,7 @@
 
 
 class MochaVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://video.mocha.com.vn/(?P<video_slug>[\w-]+)'
+    _VALID_URL = r'https?://video\.mocha\.com\.vn/(?P<video_slug>[\w-]+)'
     _TESTS = [{
         'url': 'http://video.mocha.com.vn/chuyen-meo-gia-su-tu-thong-diep-cuoc-song-v18694039',
         'info_dict': {
diff --git a/yt_dlp/extractor/nfl.py b/yt_dlp/extractor/nfl.py
index bd060dba9..3f83cd20e 100644
--- a/yt_dlp/extractor/nfl.py
+++ b/yt_dlp/extractor/nfl.py
@@ -247,7 +247,7 @@ def _real_extract(self, url):
 
 class NFLPlusReplayIE(NFLBaseIE):
     IE_NAME = 'nfl.com:plus:replay'
-    _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/games/(?P<slug>[\w-]+)(?:/(?P<id>\d+))?'
+    _VALID_URL = r'https?://(?:www\.)?nfl\.com/plus/games/(?P<slug>[\w-]+)(?:/(?P<id>\d+))?'
     _TESTS = [{
         'url': 'https://www.nfl.com/plus/games/giants-at-vikings-2022-post-1/1572108',
         'info_dict': {
@@ -342,7 +342,7 @@ def entries():
 
 class NFLPlusEpisodeIE(NFLBaseIE):
     IE_NAME = 'nfl.com:plus:episode'
-    _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/episodes/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?nfl\.com/plus/episodes/(?P<id>[\w-]+)'
     _TESTS = [{
         'note': 'Subscription required',
         'url': 'https://www.nfl.com/plus/episodes/kurt-s-qb-insider-conference-championships',
diff --git a/yt_dlp/extractor/novaplay.py b/yt_dlp/extractor/novaplay.py
index 92d1d136c..d8849cd88 100644
--- a/yt_dlp/extractor/novaplay.py
+++ b/yt_dlp/extractor/novaplay.py
@@ -3,7 +3,7 @@
 
 
 class NovaPlayIE(InfoExtractor):
-    _VALID_URL = r'https://play.nova\.bg/video/.*/(?P<id>\d+)'
+    _VALID_URL = r'https://play\.nova\.bg/video/[^?#]+/(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'https://play.nova.bg/video/ochakvaite/season-0/ochakvaite-2022-07-22-sybudi-se-sat/606627',
diff --git a/yt_dlp/extractor/nubilesporn.py b/yt_dlp/extractor/nubilesporn.py
index d4f1d9d67..1d630f547 100644
--- a/yt_dlp/extractor/nubilesporn.py
+++ b/yt_dlp/extractor/nubilesporn.py
@@ -19,7 +19,7 @@
 class NubilesPornIE(InfoExtractor):
     _NETRC_MACHINE = 'nubiles-porn'
     _VALID_URL = r'''(?x)
-        https://members.nubiles-porn.com/video/watch/(?P<id>\d+)
+        https://members\.nubiles-porn\.com/video/watch/(?P<id>\d+)
         (?:/(?P<display_id>[\w\-]+-s(?P<season>\d+)e(?P<episode>\d+)))?
     '''
 
diff --git a/yt_dlp/extractor/oftv.py b/yt_dlp/extractor/oftv.py
index 3ae7278fb..4cac51846 100644
--- a/yt_dlp/extractor/oftv.py
+++ b/yt_dlp/extractor/oftv.py
@@ -4,7 +4,7 @@
 
 
 class OfTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?of.tv/video/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?of\.tv/video/(?P<id>\w+)'
     _TESTS = [{
         'url': 'https://of.tv/video/627d7d95b353db0001dadd1a',
         'md5': 'cb9cd5db3bb9ee0d32bfd7e373d6ef0a',
@@ -34,7 +34,7 @@ def _real_extract(self, url):
 
 
 class OfTVPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?of.tv/creators/(?P<id>[a-zA-Z0-9-]+)/.?'
+    _VALID_URL = r'https?://(?:www\.)?of\.tv/creators/(?P<id>[a-zA-Z0-9-]+)/?(?:$|[?#])'
     _TESTS = [{
         'url': 'https://of.tv/creators/this-is-fire/',
         'playlist_count': 8,
diff --git a/yt_dlp/extractor/sina.py b/yt_dlp/extractor/sina.py
index 984281188..eeb9ebb44 100644
--- a/yt_dlp/extractor/sina.py
+++ b/yt_dlp/extractor/sina.py
@@ -11,7 +11,7 @@
 
 
 class SinaIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://(?:.*?\.)?video\.sina\.com\.cn/
+    _VALID_URL = r'''(?x)https?://(?:[^/?#]+\.)?video\.sina\.com\.cn/
                         (?:
                             (?:view/|.*\#)(?P<id>\d+)|
                             .+?/(?P<pseudo_id>[^/?#]+)(?:\.s?html)|
diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 4065acbaa..b6386214d 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1741,7 +1741,7 @@ def _real_extract(self, url):
 
 class TwitterShortenerIE(TwitterBaseIE):
     IE_NAME = 'twitter:shortener'
-    _VALID_URL = r'https?://t.co/(?P<id>[^?]+)|tco:(?P<eid>[^?]+)'
+    _VALID_URL = r'https?://t\.co/(?P<id>[^?#]+)|tco:(?P<eid>[^?#]+)'
     _BASE_URL = 'https://t.co/'
 
     def _real_extract(self, url):
diff --git a/yt_dlp/extractor/utreon.py b/yt_dlp/extractor/utreon.py
index 90c10c051..8a9169101 100644
--- a/yt_dlp/extractor/utreon.py
+++ b/yt_dlp/extractor/utreon.py
@@ -10,7 +10,7 @@
 
 
 class UtreonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?utreon.com/v/(?P<id>[a-zA-Z0-9_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?utreon\.com/v/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://utreon.com/v/z_I7ikQbuDw',
         'info_dict': {
diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py
index 915422817..c12e87362 100644
--- a/yt_dlp/extractor/vk.py
+++ b/yt_dlp/extractor/vk.py
@@ -97,12 +97,12 @@ class VKIE(VKBaseIE):
                         (?:
                             (?:
                                 (?:(?:m|new)\.)?vk\.com/video_|
-                                (?:www\.)?daxab.com/
+                                (?:www\.)?daxab\.com/
                             )
                             ext\.php\?(?P<embed_query>.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+).*)|
                             (?:
                                 (?:(?:m|new)\.)?vk\.com/(?:.+?\?.*?z=)?(?:video|clip)|
-                                (?:www\.)?daxab.com/embed/
+                                (?:www\.)?daxab\.com/embed/
                             )
                             (?P<videoid>-?\d+_\d+)(?:.*\blist=(?P<list_id>([\da-f]+)|(ln-[\da-zA-Z]+)))?
                         )
diff --git a/yt_dlp/extractor/weverse.py b/yt_dlp/extractor/weverse.py
index bbf62856a..47f36806b 100644
--- a/yt_dlp/extractor/weverse.py
+++ b/yt_dlp/extractor/weverse.py
@@ -182,7 +182,7 @@ def _extract_live_status(self, data):
 
 
 class WeverseIE(WeverseBaseIE):
-    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/live/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse\.io/(?P<artist>[^/?#]+)/live/(?P<id>[\d-]+)'
     _TESTS = [{
         'url': 'https://weverse.io/billlie/live/0-107323480',
         'md5': '1fa849f00181eef9100d3c8254c47979',
@@ -344,7 +344,7 @@ def _real_extract(self, url):
 
 
 class WeverseMediaIE(WeverseBaseIE):
-    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/media/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse\.io/(?P<artist>[^/?#]+)/media/(?P<id>[\d-]+)'
     _TESTS = [{
         'url': 'https://weverse.io/billlie/media/4-116372884',
         'md5': '8efc9cfd61b2f25209eb1a5326314d28',
@@ -420,7 +420,7 @@ def _real_extract(self, url):
 
 
 class WeverseMomentIE(WeverseBaseIE):
-    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/moment/(?P<uid>[\da-f]+)/post/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse\.io/(?P<artist>[^/?#]+)/moment/(?P<uid>[\da-f]+)/post/(?P<id>[\d-]+)'
     _TESTS = [{
         'url': 'https://weverse.io/secretnumber/moment/66a07e164b56a696ee71c99315ffe27b/post/1-117229444',
         'md5': '87733ac19a54081b7dfc2442036d282b',
@@ -516,7 +516,7 @@ def _real_extract(self, url):
 
 
 class WeverseLiveTabIE(WeverseTabBaseIE):
-    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/live/?(?:[?#]|$)'
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse\.io/(?P<id>[^/?#]+)/live/?(?:[?#]|$)'
     _TESTS = [{
         'url': 'https://weverse.io/billlie/live/',
         'playlist_mincount': 55,
@@ -534,7 +534,7 @@ class WeverseLiveTabIE(WeverseTabBaseIE):
 
 
 class WeverseMediaTabIE(WeverseTabBaseIE):
-    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/media(?:/|/all|/new)?(?:[?#]|$)'
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse\.io/(?P<id>[^/?#]+)/media(?:/|/all|/new)?(?:[?#]|$)'
     _TESTS = [{
         'url': 'https://weverse.io/billlie/media/',
         'playlist_mincount': 231,
@@ -558,7 +558,7 @@ class WeverseMediaTabIE(WeverseTabBaseIE):
 
 
 class WeverseLiveIE(WeverseBaseIE):
-    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/?(?:[?#]|$)'
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse\.io/(?P<id>[^/?#]+)/?(?:[?#]|$)'
     _TESTS = [{
         'url': 'https://weverse.io/purplekiss',
         'info_dict': {
diff --git a/yt_dlp/extractor/wimtv.py b/yt_dlp/extractor/wimtv.py
index 571112390..f9bf092df 100644
--- a/yt_dlp/extractor/wimtv.py
+++ b/yt_dlp/extractor/wimtv.py
@@ -11,7 +11,7 @@ class WimTVIE(InfoExtractor):
     _player = None
     _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
     _VALID_URL = r'''(?x:
-        https?://platform.wim.tv/
+        https?://platform\.wim\.tv/
         (?:
             (?:embed/)?\?
             |\#/webtv/.+?/
diff --git a/yt_dlp/extractor/xhamster.py b/yt_dlp/extractor/xhamster.py
index aec1f20bb..01ac5ddb6 100644
--- a/yt_dlp/extractor/xhamster.py
+++ b/yt_dlp/extractor/xhamster.py
@@ -24,7 +24,7 @@ class XHamsterIE(InfoExtractor):
     _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster\d+\.com|xhday\.com|xhvid\.com)'
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:.+?\.)?%s/
+                        (?:[^/?#]+\.)?%s/
                         (?:
                             movies/(?P<id>[\dA-Za-z]+)/(?P<display_id>[^/]*)\.html|
                             videos/(?P<display_id_2>[^/]*)-(?P<id_2>[\dA-Za-z]+)
@@ -372,7 +372,7 @@ def get_height(s):
 
 
 class XHamsterEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?%s/xembed\.php\?video=(?P<id>\d+)' % XHamsterIE._DOMAINS
+    _VALID_URL = r'https?://(?:[^/?#]+\.)?%s/xembed\.php\?video=(?P<id>\d+)' % XHamsterIE._DOMAINS
     _EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?xhamster\.com/xembed\.php\?video=\d+)\1']
     _TEST = {
         'url': 'http://xhamster.com/xembed.php?video=3328539',
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index c5be36636..ac28ed7d2 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -949,7 +949,7 @@ def _extract_response(self, item_id, query, note='Downloading API JSON', headers
         main_rm = next(main_retries)
         # Manual retry loop for multiple RetryManagers
         # The proper RetryManager MUST be advanced after an error
-        # and it's result MUST be checked if the manager is non fatal
+        # and its result MUST be checked if the manager is non fatal
         while True:
             try:
                 response = self._call_api(
diff --git a/yt_dlp/extractor/zoom.py b/yt_dlp/extractor/zoom.py
index 1e41d0434..329ba1415 100644
--- a/yt_dlp/extractor/zoom.py
+++ b/yt_dlp/extractor/zoom.py
@@ -13,7 +13,7 @@
 
 class ZoomIE(InfoExtractor):
     IE_NAME = 'zoom'
-    _VALID_URL = r'(?P<base_url>https?://(?:[^.]+\.)?zoom.us/)rec(?:ording)?/(?P<type>play|share)/(?P<id>[A-Za-z0-9_.-]+)'
+    _VALID_URL = r'(?P<base_url>https?://(?:[^.]+\.)?zoom\.us/)rec(?:ording)?/(?P<type>play|share)/(?P<id>[\w.-]+)'
     _TESTS = [{
         'url': 'https://economist.zoom.us/rec/play/dUk_CNBETmZ5VA2BwEl-jjakPpJ3M1pcfVYAPRsoIbEByGsLjUZtaa4yCATQuOL3der8BlTwxQePl_j0.EImBkXzTIaPvdZO5',
         'md5': 'ab445e8c911fddc4f9adc842c2c5d434',
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 163809706..85a6402a6 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -727,7 +727,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
     authentication.add_option(
         '--video-password',
         dest='videopassword', metavar='PASSWORD',
-        help='Video password (vimeo, youku)')
+        help='Video-specific password')
     authentication.add_option(
         '--ap-mso',
         dest='ap_mso', metavar='MSO',

From b73c4093187cffddcb6fbc4bfbdc0fea244ff1e9 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@example.com>
Date: Fri, 13 Oct 2023 22:22:31 +0000
Subject: [PATCH 632/871] Release 2023.10.13

Created by: bashonly

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      |  8 +++----
 .../ISSUE_TEMPLATE/2_site_support_request.yml |  8 +++----
 .../ISSUE_TEMPLATE/3_site_feature_request.yml |  8 +++----
 .github/ISSUE_TEMPLATE/4_bug_report.yml       |  8 +++----
 .github/ISSUE_TEMPLATE/5_feature_request.yml  |  8 +++----
 .github/ISSUE_TEMPLATE/6_question.yml         |  8 +++----
 CONTRIBUTORS                                  |  4 ++++
 Changelog.md                                  | 24 +++++++++++++++++++
 supportedsites.md                             |  4 ++++
 yt_dlp/version.py                             |  4 ++--
 10 files changed, 58 insertions(+), 26 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index dacb41758..6c713e5a8 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -64,7 +64,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -72,8 +72,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.07, Current version: 2023.10.07
-        yt-dlp is up to date (2023.10.07)
+        Latest version: 2023.10.13, Current version: 2023.10.13
+        yt-dlp is up to date (2023.10.13)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index ec6e298a1..e20036ce8 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -76,7 +76,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -84,8 +84,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.07, Current version: 2023.10.07
-        yt-dlp is up to date (2023.10.07)
+        Latest version: 2023.10.13, Current version: 2023.10.13
+        yt-dlp is up to date (2023.10.13)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index cf3cdd21f..a9845b6b8 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -72,7 +72,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -80,8 +80,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.07, Current version: 2023.10.07
-        yt-dlp is up to date (2023.10.07)
+        Latest version: 2023.10.13, Current version: 2023.10.13
+        yt-dlp is up to date (2023.10.13)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index 1bbcf6895..d3d60a11e 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -57,7 +57,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -65,8 +65,8 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.07, Current version: 2023.10.07
-        yt-dlp is up to date (2023.10.07)
+        Latest version: 2023.10.13, Current version: 2023.10.13
+        yt-dlp is up to date (2023.10.13)
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index d3bc06e80..57de148d0 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -53,7 +53,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -61,7 +61,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.07, Current version: 2023.10.07
-        yt-dlp is up to date (2023.10.07)
+        Latest version: 2023.10.13, Current version: 2023.10.13
+        yt-dlp is up to date (2023.10.13)
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index 30311d5b5..7b55a7427 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.07** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -59,7 +59,7 @@ body:
         [debug] Command-line config: ['-vU', 'test:youtube']
         [debug] Portable config "yt-dlp.conf": ['-i']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.07 [9d339c4] (win32_exe)
+        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
         [debug] Checking exe version: ffmpeg -bsfs
         [debug] Checking exe version: ffprobe -bsfs
@@ -67,7 +67,7 @@ body:
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
         [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.07, Current version: 2023.10.07
-        yt-dlp is up to date (2023.10.07)
+        Latest version: 2023.10.13, Current version: 2023.10.13
+        yt-dlp is up to date (2023.10.13)
         <more lines>
       render: shell
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 8eda41307..3035ee296 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -509,3 +509,7 @@ handlerug
 jiru
 madewokherd
 xofe
+awalgarg
+midnightveil
+naginatana
+Riteo
diff --git a/Changelog.md b/Changelog.md
index 48dcbf102..6f45eab2f 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,30 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.10.13
+
+#### Core changes
+- [Ensure thumbnail output directory exists](https://github.com/yt-dlp/yt-dlp/commit/2acd1d555ef89851c73773776715d3de9a0e30b9) ([#7985](https://github.com/yt-dlp/yt-dlp/issues/7985)) by [Riteo](https://github.com/Riteo)
+- **utils**
+    - `js_to_json`: [Fix `Date` constructor parsing](https://github.com/yt-dlp/yt-dlp/commit/9d7ded6419089c1bf252496073f73ad90ed71004) ([#8295](https://github.com/yt-dlp/yt-dlp/issues/8295)) by [awalgarg](https://github.com/awalgarg), [Grub4K](https://github.com/Grub4K)
+    - `write_xattr`: [Use `os.setxattr` if available](https://github.com/yt-dlp/yt-dlp/commit/84e26038d4002e763ea51ca1bdce4f7e63c540bf) ([#8205](https://github.com/yt-dlp/yt-dlp/issues/8205)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+
+#### Extractor changes
+- **artetv**: [Support age-restricted content](https://github.com/yt-dlp/yt-dlp/commit/09f815ad52843219a7ee3f2a0dddf6c250c91f0c) ([#8301](https://github.com/yt-dlp/yt-dlp/issues/8301)) by [StefanLobbenmeier](https://github.com/StefanLobbenmeier)
+- **jtbc**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/b286ec68f1f28798b3e371f888a2ed97d399cf77) ([#8314](https://github.com/yt-dlp/yt-dlp/issues/8314)) by [seproDev](https://github.com/seproDev)
+- **mbn**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/e030b6b6fba7b2f4614ad2ab9f7649d40a2dd305) ([#8312](https://github.com/yt-dlp/yt-dlp/issues/8312)) by [seproDev](https://github.com/seproDev)
+- **nhk**: [Fix Japanese-language VOD extraction](https://github.com/yt-dlp/yt-dlp/commit/4de94b9e165bfd6421a692f5f2eabcdb08edcb71) ([#8309](https://github.com/yt-dlp/yt-dlp/issues/8309)) by [garret1317](https://github.com/garret1317)
+- **radiko**: [Fix bug with `downloader_options`](https://github.com/yt-dlp/yt-dlp/commit/b9316642313bbc9e209ac0d2276d37ba60bceb49) by [bashonly](https://github.com/bashonly)
+- **tenplay**: [Add support for seasons](https://github.com/yt-dlp/yt-dlp/commit/88a99c87b680ae59002534a517e191f46c42cbd4) ([#7939](https://github.com/yt-dlp/yt-dlp/issues/7939)) by [midnightveil](https://github.com/midnightveil)
+- **youku**: [Improve tudou.com support](https://github.com/yt-dlp/yt-dlp/commit/b7098d46b552a9322c6cea39ba80be5229f922de) ([#8160](https://github.com/yt-dlp/yt-dlp/issues/8160)) by [naginatana](https://github.com/naginatana)
+- **youtube**: [Fix bug with `--extractor-retries inf`](https://github.com/yt-dlp/yt-dlp/commit/feebf6d02fc9651331eee2af5e08e6112288163b) ([#8328](https://github.com/yt-dlp/yt-dlp/issues/8328)) by [Grub4K](https://github.com/Grub4K)
+
+#### Downloader changes
+- **fragment**: [Improve progress calculation](https://github.com/yt-dlp/yt-dlp/commit/1c51c520f7b511ebd9e4eb7322285a8c31eedbbd) ([#8241](https://github.com/yt-dlp/yt-dlp/issues/8241)) by [Grub4K](https://github.com/Grub4K)
+
+#### Misc. changes
+- **cleanup**: Miscellaneous: [b634ba7](https://github.com/yt-dlp/yt-dlp/commit/b634ba742d8f38ce9ecfa0546485728b0c6c59d1) by [bashonly](https://github.com/bashonly), [gamer191](https://github.com/gamer191)
+
 ### 2023.10.07
 
 #### Extractor changes
diff --git a/supportedsites.md b/supportedsites.md
index ecef4dc2d..0ab61d68d 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -657,6 +657,8 @@ # Supported sites
  - **Joj**
  - **Jove**
  - **JStream**
+ - **JTBC**: jtbc.co.kr
+ - **JTBC:program**
  - **JWPlatform**
  - **Kakao**
  - **Kaltura**
@@ -766,6 +768,7 @@ # Supported sites
  - **massengeschmack.tv**
  - **Masters**
  - **MatchTV**
+ - **MBN**: mbn.co.kr (매일방송)
  - **MDR**: MDR.DE and KiKA
  - **MedalTV**
  - **media.ccc.de**
@@ -1468,6 +1471,7 @@ # Supported sites
  - **Tempo**
  - **TennisTV**: [*tennistv*](## "netrc machine")
  - **TenPlay**: [*10play*](## "netrc machine")
+ - **TenPlaySeason**
  - **TF1**
  - **TFO**
  - **TheHoleTv**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index 60c1c94cc..9d0096316 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.10.07'
+__version__ = '2023.10.13'
 
-RELEASE_GIT_HEAD = '377e85a1797db9e98b78b38203ed9d4ded229991'
+RELEASE_GIT_HEAD = 'b634ba742d8f38ce9ecfa0546485728b0c6c59d1'
 
 VARIANT = None
 

From 700444c23ddb65f618c2abd942acdc0c58c650b1 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Fri, 13 Oct 2023 18:02:06 -0500
Subject: [PATCH 633/871] [ci] Run core tests with dependencies

Authored by: bashonly, coletdjnz
---
 .github/workflows/core.yml   | 2 +-
 devscripts/make_changelog.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 7fcf11dfa..7acaee1e8 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -33,7 +33,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pytest
-      run: pip install pytest
+      run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: False
       run: |
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index 9ff65db14..d0e893e58 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -56,6 +56,7 @@ def subgroup_lookup(cls):
                 },
                 cls.MISC: {
                     'build',
+                    'ci',
                     'cleanup',
                     'devscripts',
                     'docs',

From 8a8b54523addf46dfd50ef599761a81bc22362e6 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 14 Oct 2023 12:33:00 +1300
Subject: [PATCH 634/871] [rh:requests] Add handler for `requests` HTTP library
 (#3668)

Adds support for HTTPS proxies and persistent connections (keep-alive)

Closes https://github.com/yt-dlp/yt-dlp/issues/1890
Resolves https://github.com/yt-dlp/yt-dlp/issues/4070
Resolves https://github.com/ytdl-org/youtube-dl/issues/32549
Resolves https://github.com/ytdl-org/youtube-dl/issues/14523
Resolves https://github.com/ytdl-org/youtube-dl/issues/13734

Authored by: coletdjnz, Grub4K, bashonly
---
 .github/workflows/core.yml          |   2 +-
 README.md                           |   4 +-
 requirements.txt                    |   2 +
 setup.py                            |   9 +-
 test/test_networking.py             | 168 +++++++++---
 test/test_socks.py                  |  36 +--
 yt_dlp/YoutubeDL.py                 |   7 +-
 yt_dlp/__pyinstaller/hook-yt_dlp.py |   4 +-
 yt_dlp/dependencies/__init__.py     |   9 +
 yt_dlp/networking/__init__.py       |  10 +
 yt_dlp/networking/_helper.py        |  20 +-
 yt_dlp/networking/_requests.py      | 398 ++++++++++++++++++++++++++++
 yt_dlp/networking/_urllib.py        |  26 +-
 yt_dlp/options.py                   |   3 +-
 14 files changed, 619 insertions(+), 79 deletions(-)
 create mode 100644 yt_dlp/networking/_requests.py

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 7acaee1e8..049faf373 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -32,7 +32,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install pytest
+    - name: Install dependencies
       run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: False
diff --git a/README.md b/README.md
index dd4652d43..3b7432474 100644
--- a/README.md
+++ b/README.md
@@ -157,6 +157,7 @@ ### Differences in default behavior
 * yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
 * yt-dlp tries to parse the external downloader outputs into the standard progress output if possible (Currently implemented: [~~aria2c~~](https://github.com/yt-dlp/yt-dlp/issues/5931)). You can use `--compat-options no-external-downloader-progress` to get the downloader output as-is
 * yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filter` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this
+* yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests.
 
 For ease of use, a few more compat options are available:
 
@@ -164,7 +165,7 @@ ### Differences in default behavior
 * `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter`
 * `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter`
 * `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date`
-* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress`. Use this to enable all future compat options
+* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler`. Use this to enable all future compat options
 
 
 # INSTALLATION
@@ -274,6 +275,7 @@ ### Networking
 * [**certifi**](https://github.com/certifi/python-certifi)\* - Provides Mozilla's root certificate bundle. Licensed under [MPLv2](https://github.com/certifi/python-certifi/blob/master/LICENSE)
 * [**brotli**](https://github.com/google/brotli)\* or [**brotlicffi**](https://github.com/python-hyper/brotlicffi) - [Brotli](https://en.wikipedia.org/wiki/Brotli) content encoding support. Both licensed under MIT <sup>[1](https://github.com/google/brotli/blob/master/LICENSE) [2](https://github.com/python-hyper/brotlicffi/blob/master/LICENSE) </sup>
 * [**websockets**](https://github.com/aaugustin/websockets)\* - For downloading over websocket. Licensed under [BSD-3-Clause](https://github.com/aaugustin/websockets/blob/main/LICENSE)
+* [**requests**](https://github.com/psf/requests)\* - HTTP library. For HTTPS proxy and persistent connections support. Licensed under [Apache-2.0](https://github.com/psf/requests/blob/main/LICENSE)
 
 ### Metadata
 
diff --git a/requirements.txt b/requirements.txt
index dde37120f..112c30aeb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,5 @@ websockets
 brotli; platform_python_implementation=='CPython'
 brotlicffi; platform_python_implementation!='CPython'
 certifi
+requests>=2.31.0,<3
+urllib3>=1.26.17,<3
\ No newline at end of file
diff --git a/setup.py b/setup.py
index a2f9f55c3..1740db27d 100644
--- a/setup.py
+++ b/setup.py
@@ -62,7 +62,14 @@ def py2exe_params():
             'compressed': 1,
             'optimize': 2,
             'dist_dir': './dist',
-            'excludes': ['Crypto', 'Cryptodome'],  # py2exe cannot import Crypto
+            'excludes': [
+                # py2exe cannot import Crypto
+                'Crypto',
+                'Cryptodome',
+                # py2exe appears to confuse this with our socks library.
+                # We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
+                'urllib3.contrib.socks'
+            ],
             'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
             # Modules that are only imported dynamically must be added here
             'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
diff --git a/test/test_networking.py b/test/test_networking.py
index 5308c8d6f..2b45deac7 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -28,7 +28,7 @@
 
 from test.helper import FakeYDL, http_server_port
 from yt_dlp.cookies import YoutubeDLCookieJar
-from yt_dlp.dependencies import brotli
+from yt_dlp.dependencies import brotli, requests, urllib3
 from yt_dlp.networking import (
     HEADRequest,
     PUTRequest,
@@ -43,6 +43,7 @@
     HTTPError,
     IncompleteRead,
     NoSupportingHandlers,
+    ProxyError,
     RequestError,
     SSLError,
     TransportError,
@@ -305,7 +306,7 @@ def setup_class(cls):
 
 
 class TestHTTPRequestHandler(TestRequestHandlerBase):
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_verify_cert(self, handler):
         with handler() as rh:
             with pytest.raises(CertificateVerifyError):
@@ -316,7 +317,7 @@ def test_verify_cert(self, handler):
             assert r.status == 200
             r.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_ssl_error(self, handler):
         # HTTPS server with too old TLS version
         # XXX: is there a better way to test this than to create a new server?
@@ -334,7 +335,7 @@ def test_ssl_error(self, handler):
                 validate_and_send(rh, Request(f'https://127.0.0.1:{https_port}/headers'))
             assert not issubclass(exc_info.type, CertificateVerifyError)
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_percent_encode(self, handler):
         with handler() as rh:
             # Unicode characters should be encoded with uppercase percent-encoding
@@ -346,7 +347,7 @@ def test_percent_encode(self, handler):
             assert res.status == 200
             res.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_remove_dot_segments(self, handler):
         with handler() as rh:
             # This isn't a comprehensive test,
@@ -361,14 +362,14 @@ def test_remove_dot_segments(self, handler):
             assert res.url == f'http://127.0.0.1:{self.http_port}/headers'
             res.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_unicode_path_redirection(self, handler):
         with handler() as rh:
             r = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/302-non-ascii-redirect'))
             assert r.url == f'http://127.0.0.1:{self.http_port}/%E4%B8%AD%E6%96%87.html'
             r.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_raise_http_error(self, handler):
         with handler() as rh:
             for bad_status in (400, 500, 599, 302):
@@ -378,7 +379,7 @@ def test_raise_http_error(self, handler):
             # Should not raise an error
             validate_and_send(rh, Request('http://127.0.0.1:%d/gen_200' % self.http_port)).close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_response_url(self, handler):
         with handler() as rh:
             # Response url should be that of the last url in redirect chain
@@ -389,7 +390,7 @@ def test_response_url(self, handler):
             assert res2.url == f'http://127.0.0.1:{self.http_port}/gen_200'
             res2.close()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_redirect(self, handler):
         with handler() as rh:
             def do_req(redirect_status, method, assert_no_content=False):
@@ -444,7 +445,7 @@ def do_req(redirect_status, method, assert_no_content=False):
                 with pytest.raises(HTTPError):
                     do_req(code, 'GET')
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_request_cookie_header(self, handler):
         # We should accept a Cookie header being passed as in normal headers and handle it appropriately.
         with handler() as rh:
@@ -476,19 +477,19 @@ def test_request_cookie_header(self, handler):
             assert b'Cookie: test=ytdlp' not in data
             assert b'Cookie: test=test' in data
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_redirect_loop(self, handler):
         with handler() as rh:
             with pytest.raises(HTTPError, match='redirect loop'):
                 validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/redirect_loop'))
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_incompleteread(self, handler):
         with handler(timeout=2) as rh:
             with pytest.raises(IncompleteRead):
                 validate_and_send(rh, Request('http://127.0.0.1:%d/incompleteread' % self.http_port)).read()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_cookies(self, handler):
         cookiejar = YoutubeDLCookieJar()
         cookiejar.set_cookie(http.cookiejar.Cookie(
@@ -505,7 +506,7 @@ def test_cookies(self, handler):
                 rh, Request(f'http://127.0.0.1:{self.http_port}/headers', extensions={'cookiejar': cookiejar})).read()
             assert b'Cookie: test=ytdlp' in data
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_headers(self, handler):
 
         with handler(headers=HTTPHeaderDict({'test1': 'test', 'test2': 'test2'})) as rh:
@@ -521,7 +522,7 @@ def test_headers(self, handler):
             assert b'Test2: test2' not in data
             assert b'Test3: test3' in data
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_timeout(self, handler):
         with handler() as rh:
             # Default timeout is 20 seconds, so this should go through
@@ -537,7 +538,7 @@ def test_timeout(self, handler):
             validate_and_send(
                 rh, Request(f'http://127.0.0.1:{self.http_port}/timeout_1', extensions={'timeout': 4}))
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_source_address(self, handler):
         source_address = f'127.0.0.{random.randint(5, 255)}'
         with handler(source_address=source_address) as rh:
@@ -545,13 +546,13 @@ def test_source_address(self, handler):
                 rh, Request(f'http://127.0.0.1:{self.http_port}/source_address')).read().decode()
             assert source_address == data
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_gzip_trailing_garbage(self, handler):
         with handler() as rh:
             data = validate_and_send(rh, Request(f'http://localhost:{self.http_port}/trailing_garbage')).read().decode()
             assert data == '<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     @pytest.mark.skipif(not brotli, reason='brotli support is not installed')
     def test_brotli(self, handler):
         with handler() as rh:
@@ -562,7 +563,7 @@ def test_brotli(self, handler):
             assert res.headers.get('Content-Encoding') == 'br'
             assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_deflate(self, handler):
         with handler() as rh:
             res = validate_and_send(
@@ -572,7 +573,7 @@ def test_deflate(self, handler):
             assert res.headers.get('Content-Encoding') == 'deflate'
             assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_gzip(self, handler):
         with handler() as rh:
             res = validate_and_send(
@@ -582,7 +583,7 @@ def test_gzip(self, handler):
             assert res.headers.get('Content-Encoding') == 'gzip'
             assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_multiple_encodings(self, handler):
         with handler() as rh:
             for pair in ('gzip,deflate', 'deflate, gzip', 'gzip, gzip', 'deflate, deflate'):
@@ -593,7 +594,7 @@ def test_multiple_encodings(self, handler):
                 assert res.headers.get('Content-Encoding') == pair
                 assert res.read() == b'<html><video src="/vid.mp4" /></html>'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_unsupported_encoding(self, handler):
         with handler() as rh:
             res = validate_and_send(
@@ -603,7 +604,7 @@ def test_unsupported_encoding(self, handler):
             assert res.headers.get('Content-Encoding') == 'unsupported'
             assert res.read() == b'raw'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_read(self, handler):
         with handler() as rh:
             res = validate_and_send(
@@ -633,7 +634,7 @@ def setup_class(cls):
         cls.geo_proxy_thread.daemon = True
         cls.geo_proxy_thread.start()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_http_proxy(self, handler):
         http_proxy = f'http://127.0.0.1:{self.proxy_port}'
         geo_proxy = f'http://127.0.0.1:{self.geo_port}'
@@ -659,7 +660,7 @@ def test_http_proxy(self, handler):
             assert res != f'normal: {real_url}'
             assert 'Accept' in res
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_noproxy(self, handler):
         with handler(proxies={'proxy': f'http://127.0.0.1:{self.proxy_port}'}) as rh:
             # NO_PROXY
@@ -669,7 +670,7 @@ def test_noproxy(self, handler):
                     'utf-8')
                 assert 'Accept' in nop_response
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_allproxy(self, handler):
         url = 'http://foo.com/bar'
         with handler() as rh:
@@ -677,7 +678,7 @@ def test_allproxy(self, handler):
                 'utf-8')
             assert response == f'normal: {url}'
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_http_proxy_with_idn(self, handler):
         with handler(proxies={
             'http': f'http://127.0.0.1:{self.proxy_port}',
@@ -715,27 +716,27 @@ def _run_test(self, handler, **handler_kwargs):
         ) as rh:
             validate_and_send(rh, Request(f'https://127.0.0.1:{self.port}/video.html')).read().decode()
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_certificate_combined_nopass(self, handler):
         self._run_test(handler, client_cert={
             'client_certificate': os.path.join(self.certdir, 'clientwithkey.crt'),
         })
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_certificate_nocombined_nopass(self, handler):
         self._run_test(handler, client_cert={
             'client_certificate': os.path.join(self.certdir, 'client.crt'),
             'client_certificate_key': os.path.join(self.certdir, 'client.key'),
         })
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_certificate_combined_pass(self, handler):
         self._run_test(handler, client_cert={
             'client_certificate': os.path.join(self.certdir, 'clientwithencryptedkey.crt'),
             'client_certificate_password': 'foobar',
         })
 
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_certificate_nocombined_pass(self, handler):
         self._run_test(handler, client_cert={
             'client_certificate': os.path.join(self.certdir, 'client.crt'),
@@ -819,6 +820,75 @@ def test_httplib_validation_errors(self, handler, req, match, version_check):
             assert not isinstance(exc_info.value, TransportError)
 
 
+class TestRequestsRequestHandler(TestRequestHandlerBase):
+    @pytest.mark.parametrize('raised,expected', [
+        (lambda: requests.exceptions.ConnectTimeout(), TransportError),
+        (lambda: requests.exceptions.ReadTimeout(), TransportError),
+        (lambda: requests.exceptions.Timeout(), TransportError),
+        (lambda: requests.exceptions.ConnectionError(), TransportError),
+        (lambda: requests.exceptions.ProxyError(), ProxyError),
+        (lambda: requests.exceptions.SSLError('12[CERTIFICATE_VERIFY_FAILED]34'), CertificateVerifyError),
+        (lambda: requests.exceptions.SSLError(), SSLError),
+        (lambda: requests.exceptions.InvalidURL(), RequestError),
+        (lambda: requests.exceptions.InvalidHeader(), RequestError),
+        # catch-all: https://github.com/psf/requests/blob/main/src/requests/adapters.py#L535
+        (lambda: urllib3.exceptions.HTTPError(), TransportError),
+        (lambda: requests.exceptions.RequestException(), RequestError)
+        #  (lambda: requests.exceptions.TooManyRedirects(), HTTPError) - Needs a response object
+    ])
+    @pytest.mark.parametrize('handler', ['Requests'], indirect=True)
+    def test_request_error_mapping(self, handler, monkeypatch, raised, expected):
+        with handler() as rh:
+            def mock_get_instance(*args, **kwargs):
+                class MockSession:
+                    def request(self, *args, **kwargs):
+                        raise raised()
+                return MockSession()
+
+            monkeypatch.setattr(rh, '_get_instance', mock_get_instance)
+
+            with pytest.raises(expected) as exc_info:
+                rh.send(Request('http://fake'))
+
+            assert exc_info.type is expected
+
+    @pytest.mark.parametrize('raised,expected,match', [
+        (lambda: urllib3.exceptions.SSLError(), SSLError, None),
+        (lambda: urllib3.exceptions.TimeoutError(), TransportError, None),
+        (lambda: urllib3.exceptions.ReadTimeoutError(None, None, None), TransportError, None),
+        (lambda: urllib3.exceptions.ProtocolError(), TransportError, None),
+        (lambda: urllib3.exceptions.DecodeError(), TransportError, None),
+        (lambda: urllib3.exceptions.HTTPError(), TransportError, None),  # catch-all
+        (
+            lambda: urllib3.exceptions.ProtocolError('error', http.client.IncompleteRead(partial=b'abc', expected=4)),
+            IncompleteRead,
+            '3 bytes read, 4 more expected'
+        ),
+        (
+            lambda: urllib3.exceptions.IncompleteRead(partial=3, expected=5),
+            IncompleteRead,
+            '3 bytes read, 5 more expected'
+        ),
+    ])
+    @pytest.mark.parametrize('handler', ['Requests'], indirect=True)
+    def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
+        from urllib3.response import HTTPResponse as Urllib3Response
+        from requests.models import Response as RequestsResponse
+        from yt_dlp.networking._requests import RequestsResponseAdapter
+        requests_res = RequestsResponse()
+        requests_res.raw = Urllib3Response(body=b'', status=200)
+        res = RequestsResponseAdapter(requests_res)
+
+        def mock_read(*args, **kwargs):
+            raise raised()
+        monkeypatch.setattr(res.fp, 'read', mock_read)
+
+        with pytest.raises(expected, match=match) as exc_info:
+            res.read()
+
+        assert exc_info.type is expected
+
+
 def run_validation(handler, error, req, **handler_kwargs):
     with handler(**handler_kwargs) as rh:
         if error:
@@ -855,6 +925,10 @@ class HTTPSupportedRH(ValidationRH):
             ('file', UnsupportedRequest, {}),
             ('file', False, {'enable_file_urls': True}),
         ]),
+        ('Requests', [
+            ('http', False, {}),
+            ('https', False, {}),
+        ]),
         (NoCheckRH, [('http', False, {})]),
         (ValidationRH, [('http', UnsupportedRequest, {})])
     ]
@@ -870,6 +944,14 @@ class HTTPSupportedRH(ValidationRH):
             ('socks5h', False),
             ('socks', UnsupportedRequest),
         ]),
+        ('Requests', [
+            ('http', False),
+            ('https', False),
+            ('socks4', False),
+            ('socks4a', False),
+            ('socks5', False),
+            ('socks5h', False),
+        ]),
         (NoCheckRH, [('http', False)]),
         (HTTPSupportedRH, [('http', UnsupportedRequest)]),
     ]
@@ -880,6 +962,10 @@ class HTTPSupportedRH(ValidationRH):
             ('all', False),
             ('unrelated', False),
         ]),
+        ('Requests', [
+            ('all', False),
+            ('unrelated', False),
+        ]),
         (NoCheckRH, [('all', False)]),
         (HTTPSupportedRH, [('all', UnsupportedRequest)]),
         (HTTPSupportedRH, [('no', UnsupportedRequest)]),
@@ -894,6 +980,13 @@ class HTTPSupportedRH(ValidationRH):
             ({'timeout': 'notatimeout'}, AssertionError),
             ({'unsupported': 'value'}, UnsupportedRequest),
         ]),
+        ('Requests', [
+            ({'cookiejar': 'notacookiejar'}, AssertionError),
+            ({'cookiejar': YoutubeDLCookieJar()}, False),
+            ({'timeout': 1}, False),
+            ({'timeout': 'notatimeout'}, AssertionError),
+            ({'unsupported': 'value'}, UnsupportedRequest),
+        ]),
         (NoCheckRH, [
             ({'cookiejar': 'notacookiejar'}, False),
             ({'somerandom': 'test'}, False),  # but any extension is allowed through
@@ -909,7 +1002,7 @@ class HTTPSupportedRH(ValidationRH):
     def test_url_scheme(self, handler, scheme, fail, handler_kwargs):
         run_validation(handler, fail, Request(f'{scheme}://'), **(handler_kwargs or {}))
 
-    @pytest.mark.parametrize('handler,fail', [('Urllib', False)], indirect=['handler'])
+    @pytest.mark.parametrize('handler,fail', [('Urllib', False), ('Requests', False)], indirect=['handler'])
     def test_no_proxy(self, handler, fail):
         run_validation(handler, fail, Request('http://', proxies={'no': '127.0.0.1,github.com'}))
         run_validation(handler, fail, Request('http://'), proxies={'no': '127.0.0.1,github.com'})
@@ -932,13 +1025,13 @@ def test_proxy_scheme(self, handler, scheme, fail):
         run_validation(handler, fail, Request('http://', proxies={'http': f'{scheme}://example.com'}))
         run_validation(handler, fail, Request('http://'), proxies={'http': f'{scheme}://example.com'})
 
-    @pytest.mark.parametrize('handler', ['Urllib', HTTPSupportedRH], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', HTTPSupportedRH, 'Requests'], indirect=True)
     def test_empty_proxy(self, handler):
         run_validation(handler, False, Request('http://', proxies={'http': None}))
         run_validation(handler, False, Request('http://'), proxies={'http': None})
 
     @pytest.mark.parametrize('proxy_url', ['//example.com', 'example.com', '127.0.0.1', '/a/b/c'])
-    @pytest.mark.parametrize('handler', ['Urllib'], indirect=True)
+    @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_invalid_proxy_url(self, handler, proxy_url):
         run_validation(handler, UnsupportedRequest, Request('http://', proxies={'http': proxy_url}))
 
@@ -1242,6 +1335,13 @@ def test_urllib_file_urls(self):
             rh = self.build_handler(ydl, UrllibRH)
             assert rh.enable_file_urls is True
 
+    def test_compat_opt_prefer_urllib(self):
+        # This assumes urllib only has a preference when this compat opt is given
+        with FakeYDL({'compat_opts': ['prefer-legacy-http-handler']}) as ydl:
+            director = ydl.build_request_director([UrllibRH])
+            assert len(director.preferences) == 1
+            assert director.preferences.pop()(UrllibRH, None)
+
 
 class TestRequest:
 
diff --git a/test/test_socks.py b/test/test_socks.py
index 211ee814d..d8ac88dad 100644
--- a/test/test_socks.py
+++ b/test/test_socks.py
@@ -263,7 +263,7 @@ def ctx(request):
 
 
 class TestSocks4Proxy:
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks4_no_auth(self, handler, ctx):
         with handler() as rh:
             with ctx.socks_server(Socks4ProxyHandler) as server_address:
@@ -271,7 +271,7 @@ def test_socks4_no_auth(self, handler, ctx):
                     rh, proxies={'all': f'socks4://{server_address}'})
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks4_auth(self, handler, ctx):
         with handler() as rh:
             with ctx.socks_server(Socks4ProxyHandler, user_id='user') as server_address:
@@ -281,7 +281,7 @@ def test_socks4_auth(self, handler, ctx):
                     rh, proxies={'all': f'socks4://user:@{server_address}'})
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks4a_ipv4_target(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
@@ -289,7 +289,7 @@ def test_socks4a_ipv4_target(self, handler, ctx):
                 assert response['version'] == 4
                 assert (response['ipv4_address'] == '127.0.0.1') != (response['domain_address'] == '127.0.0.1')
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks4a_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
@@ -298,7 +298,7 @@ def test_socks4a_domain_target(self, handler, ctx):
                 assert response['ipv4_address'] is None
                 assert response['domain_address'] == 'localhost'
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
@@ -308,7 +308,7 @@ def test_ipv4_client_source_address(self, handler, ctx):
                 assert response['client_address'][0] == source_address
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     @pytest.mark.parametrize('reply_code', [
         Socks4CD.REQUEST_REJECTED_OR_FAILED,
         Socks4CD.REQUEST_REJECTED_CANNOT_CONNECT_TO_IDENTD,
@@ -320,7 +320,7 @@ def test_socks4_errors(self, handler, ctx, reply_code):
                 with pytest.raises(ProxyError):
                     ctx.socks_info_request(rh)
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_ipv6_socks4_proxy(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler, bind_ip='::1') as server_address:
             with handler(proxies={'all': f'socks4://{server_address}'}) as rh:
@@ -329,7 +329,7 @@ def test_ipv6_socks4_proxy(self, handler, ctx):
                 assert response['ipv4_address'] == '127.0.0.1'
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_timeout(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler, sleep=2) as server_address:
             with handler(proxies={'all': f'socks4://{server_address}'}, timeout=0.5) as rh:
@@ -339,7 +339,7 @@ def test_timeout(self, handler, ctx):
 
 class TestSocks5Proxy:
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_no_auth(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -347,7 +347,7 @@ def test_socks5_no_auth(self, handler, ctx):
                 assert response['auth_methods'] == [0x0]
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_user_pass(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler, auth=('test', 'testpass')) as server_address:
             with handler() as rh:
@@ -360,7 +360,7 @@ def test_socks5_user_pass(self, handler, ctx):
                 assert response['auth_methods'] == [Socks5Auth.AUTH_NONE, Socks5Auth.AUTH_USER_PASS]
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_ipv4_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -368,7 +368,7 @@ def test_socks5_ipv4_target(self, handler, ctx):
                 assert response['ipv4_address'] == '127.0.0.1'
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -376,7 +376,7 @@ def test_socks5_domain_target(self, handler, ctx):
                 assert (response['ipv4_address'] == '127.0.0.1') != (response['ipv6_address'] == '::1')
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5h_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
@@ -385,7 +385,7 @@ def test_socks5h_domain_target(self, handler, ctx):
                 assert response['domain_address'] == 'localhost'
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5h_ip_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
@@ -394,7 +394,7 @@ def test_socks5h_ip_target(self, handler, ctx):
                 assert response['domain_address'] is None
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_socks5_ipv6_destination(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -402,7 +402,7 @@ def test_socks5_ipv6_destination(self, handler, ctx):
                 assert response['ipv6_address'] == '::1'
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_ipv6_socks5_proxy(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler, bind_ip='::1') as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -413,7 +413,7 @@ def test_ipv6_socks5_proxy(self, handler, ctx):
 
     # XXX: is there any feasible way of testing IPv6 source addresses?
     # Same would go for non-proxy source_address test...
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
@@ -422,7 +422,7 @@ def test_ipv4_client_source_address(self, handler, ctx):
                 assert response['client_address'][0] == source_address
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
     @pytest.mark.parametrize('reply_code', [
         Socks5Reply.GENERAL_FAILURE,
         Socks5Reply.CONNECTION_NOT_ALLOWED,
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 71d17ac01..8e11646d3 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3968,7 +3968,7 @@ def get_encoding(stream):
         })) or 'none'))
 
         write_debug(f'Proxy map: {self.proxies}')
-        # write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}')
+        write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}')
         for plugin_type, plugins in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}.items():
             display_list = ['%s%s' % (
                 klass.__name__, '' if klass.__name__ == name else f' as {name}')
@@ -4057,6 +4057,9 @@ def urlopen(self, req):
                     raise RequestError(
                         'file:// URLs are disabled by default in yt-dlp for security reasons. '
                         'Use --enable-file-urls to enable at your own risk.', cause=ue) from ue
+                if 'unsupported proxy type: "https"' in ue.msg.lower():
+                    raise RequestError(
+                        'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests')
             raise
         except SSLError as e:
             if 'UNSAFE_LEGACY_RENEGOTIATION_DISABLED' in str(e):
@@ -4099,6 +4102,8 @@ def build_request_director(self, handlers, preferences=None):
                 }),
             ))
         director.preferences.update(preferences or [])
+        if 'prefer-legacy-http-handler' in self.params['compat_opts']:
+            director.preferences.add(lambda rh, _: 500 if rh.RH_KEY == 'Urllib' else 0)
         return director
 
     def encode(self, s):
diff --git a/yt_dlp/__pyinstaller/hook-yt_dlp.py b/yt_dlp/__pyinstaller/hook-yt_dlp.py
index 88c2b8b28..c7f2c0ceb 100644
--- a/yt_dlp/__pyinstaller/hook-yt_dlp.py
+++ b/yt_dlp/__pyinstaller/hook-yt_dlp.py
@@ -21,7 +21,9 @@ def get_hidden_imports():
     yield from ('yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated')
     yield from ('yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated')
     yield pycryptodome_module()
-    yield from collect_submodules('websockets')
+    # Only `websockets` is required, others are collected just in case
+    for module in ('websockets', 'requests', 'urllib3'):
+        yield from collect_submodules(module)
     # These are auto-detected, but explicitly add them just in case
     yield from ('mutagen', 'brotli', 'certifi')
 
diff --git a/yt_dlp/dependencies/__init__.py b/yt_dlp/dependencies/__init__.py
index b56e4f5cc..ef83739a3 100644
--- a/yt_dlp/dependencies/__init__.py
+++ b/yt_dlp/dependencies/__init__.py
@@ -58,6 +58,15 @@
     # See https://github.com/yt-dlp/yt-dlp/issues/2633
     websockets = None
 
+try:
+    import urllib3
+except ImportError:
+    urllib3 = None
+
+try:
+    import requests
+except ImportError:
+    requests = None
 
 try:
     import xattr  # xattr or pyxattr
diff --git a/yt_dlp/networking/__init__.py b/yt_dlp/networking/__init__.py
index 5b1599a6d..aa8d0eabe 100644
--- a/yt_dlp/networking/__init__.py
+++ b/yt_dlp/networking/__init__.py
@@ -1,4 +1,6 @@
 # flake8: noqa: F401
+import warnings
+
 from .common import (
     HEADRequest,
     PUTRequest,
@@ -11,3 +13,11 @@
 # isort: split
 # TODO: all request handlers should be safely imported
 from . import _urllib
+from ..utils import bug_reports_message
+
+try:
+    from . import _requests
+except ImportError:
+    pass
+except Exception as e:
+    warnings.warn(f'Failed to import "requests" request handler: {e}' + bug_reports_message())
diff --git a/yt_dlp/networking/_helper.py b/yt_dlp/networking/_helper.py
index 4c9dbf25d..a6fa3550b 100644
--- a/yt_dlp/networking/_helper.py
+++ b/yt_dlp/networking/_helper.py
@@ -11,7 +11,7 @@
 
 from .exceptions import RequestError, UnsupportedRequest
 from ..dependencies import certifi
-from ..socks import ProxyType
+from ..socks import ProxyType, sockssocket
 from ..utils import format_field, traverse_obj
 
 if typing.TYPE_CHECKING:
@@ -224,6 +224,24 @@ def _socket_connect(ip_addr, timeout, source_address):
         raise
 
 
+def create_socks_proxy_socket(dest_addr, proxy_args, proxy_ip_addr, timeout, source_address):
+    af, socktype, proto, canonname, sa = proxy_ip_addr
+    sock = sockssocket(af, socktype, proto)
+    try:
+        connect_proxy_args = proxy_args.copy()
+        connect_proxy_args.update({'addr': sa[0], 'port': sa[1]})
+        sock.setproxy(**connect_proxy_args)
+        if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:  # noqa: E721
+            sock.settimeout(timeout)
+        if source_address:
+            sock.bind(source_address)
+        sock.connect(dest_addr)
+        return sock
+    except socket.error:
+        sock.close()
+        raise
+
+
 def create_connection(
     address,
     timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py
new file mode 100644
index 000000000..27974357a
--- /dev/null
+++ b/yt_dlp/networking/_requests.py
@@ -0,0 +1,398 @@
+import contextlib
+import functools
+import http.client
+import logging
+import re
+import socket
+import warnings
+
+from ..dependencies import brotli, requests, urllib3
+from ..utils import bug_reports_message, int_or_none, variadic
+
+if requests is None:
+    raise ImportError('requests module is not installed')
+
+if urllib3 is None:
+    raise ImportError('urllib3 module is not installed')
+
+urllib3_version = tuple(int_or_none(x, default=0) for x in urllib3.__version__.split('.'))
+
+if urllib3_version < (1, 26, 17):
+    raise ImportError('Only urllib3 >= 1.26.17 is supported')
+
+if requests.__build__ < 0x023100:
+    raise ImportError('Only requests >= 2.31.0 is supported')
+
+import requests.adapters
+import requests.utils
+import urllib3.connection
+import urllib3.exceptions
+
+from ._helper import (
+    InstanceStoreMixin,
+    add_accept_encoding_header,
+    create_connection,
+    create_socks_proxy_socket,
+    get_redirect_method,
+    make_socks_proxy_opts,
+    select_proxy,
+)
+from .common import (
+    Features,
+    RequestHandler,
+    Response,
+    register_preference,
+    register_rh,
+)
+from .exceptions import (
+    CertificateVerifyError,
+    HTTPError,
+    IncompleteRead,
+    ProxyError,
+    RequestError,
+    SSLError,
+    TransportError,
+)
+from ..socks import ProxyError as SocksProxyError
+
+SUPPORTED_ENCODINGS = [
+    'gzip', 'deflate'
+]
+
+if brotli is not None:
+    SUPPORTED_ENCODINGS.append('br')
+
+"""
+Override urllib3's behavior to not convert lower-case percent-encoded characters
+to upper-case during url normalization process.
+
+RFC3986 defines that the lower or upper case percent-encoded hexidecimal characters are equivalent
+and normalizers should convert them to uppercase for consistency [1].
+
+However, some sites may have an incorrect implementation where they provide
+a percent-encoded url that is then compared case-sensitively.[2]
+
+While this is a very rare case, since urllib does not do this normalization step, it
+is best to avoid it in requests too for compatability reasons.
+
+1: https://tools.ietf.org/html/rfc3986#section-2.1
+2: https://github.com/streamlink/streamlink/pull/4003
+"""
+
+
+class Urllib3PercentREOverride:
+    def __init__(self, r: re.Pattern):
+        self.re = r
+
+    # pass through all other attribute calls to the original re
+    def __getattr__(self, item):
+        return self.re.__getattribute__(item)
+
+    def subn(self, repl, string, *args, **kwargs):
+        return string, self.re.subn(repl, string, *args, **kwargs)[1]
+
+
+# urllib3 >= 1.25.8 uses subn:
+# https://github.com/urllib3/urllib3/commit/a2697e7c6b275f05879b60f593c5854a816489f0
+import urllib3.util.url  # noqa: E305
+
+if hasattr(urllib3.util.url, 'PERCENT_RE'):
+    urllib3.util.url.PERCENT_RE = Urllib3PercentREOverride(urllib3.util.url.PERCENT_RE)
+elif hasattr(urllib3.util.url, '_PERCENT_RE'):  # urllib3 >= 2.0.0
+    urllib3.util.url._PERCENT_RE = Urllib3PercentREOverride(urllib3.util.url._PERCENT_RE)
+else:
+    warnings.warn('Failed to patch PERCENT_RE in urllib3 (does the attribute exist?)' + bug_reports_message())
+
+"""
+Workaround for issue in urllib.util.ssl_.py: ssl_wrap_context does not pass
+server_hostname to SSLContext.wrap_socket if server_hostname is an IP,
+however this is an issue because we set check_hostname to True in our SSLContext.
+
+Monkey-patching IS_SECURETRANSPORT forces ssl_wrap_context to pass server_hostname regardless.
+
+This has been fixed in urllib3 2.0+.
+See: https://github.com/urllib3/urllib3/issues/517
+"""
+
+if urllib3_version < (2, 0, 0):
+    with contextlib.suppress():
+        urllib3.util.IS_SECURETRANSPORT = urllib3.util.ssl_.IS_SECURETRANSPORT = True
+
+
+# Requests will not automatically handle no_proxy by default
+# due to buggy no_proxy handling with proxy dict [1].
+# 1. https://github.com/psf/requests/issues/5000
+requests.adapters.select_proxy = select_proxy
+
+
+class RequestsResponseAdapter(Response):
+    def __init__(self, res: requests.models.Response):
+        super().__init__(
+            fp=res.raw, headers=res.headers, url=res.url,
+            status=res.status_code, reason=res.reason)
+
+        self._requests_response = res
+
+    def read(self, amt: int = None):
+        try:
+            # Interact with urllib3 response directly.
+            return self.fp.read(amt, decode_content=True)
+
+        # See urllib3.response.HTTPResponse.read() for exceptions raised on read
+        except urllib3.exceptions.SSLError as e:
+            raise SSLError(cause=e) from e
+
+        except urllib3.exceptions.IncompleteRead as e:
+            # urllib3 IncompleteRead.partial is always an integer
+            raise IncompleteRead(partial=e.partial, expected=e.expected) from e
+
+        except urllib3.exceptions.ProtocolError as e:
+            # http.client.IncompleteRead may be contained within ProtocolError
+            # See urllib3.response.HTTPResponse._error_catcher()
+            ir_err = next(
+                (err for err in (e.__context__, e.__cause__, *variadic(e.args))
+                 if isinstance(err, http.client.IncompleteRead)), None)
+            if ir_err is not None:
+                raise IncompleteRead(partial=len(ir_err.partial), expected=ir_err.expected) from e
+            raise TransportError(cause=e) from e
+
+        except urllib3.exceptions.HTTPError as e:
+            # catch-all for any other urllib3 response exceptions
+            raise TransportError(cause=e) from e
+
+
+class RequestsHTTPAdapter(requests.adapters.HTTPAdapter):
+    def __init__(self, ssl_context=None, proxy_ssl_context=None, source_address=None, **kwargs):
+        self._pm_args = {}
+        if ssl_context:
+            self._pm_args['ssl_context'] = ssl_context
+        if source_address:
+            self._pm_args['source_address'] = (source_address, 0)
+        self._proxy_ssl_context = proxy_ssl_context or ssl_context
+        super().__init__(**kwargs)
+
+    def init_poolmanager(self, *args, **kwargs):
+        return super().init_poolmanager(*args, **kwargs, **self._pm_args)
+
+    def proxy_manager_for(self, proxy, **proxy_kwargs):
+        extra_kwargs = {}
+        if not proxy.lower().startswith('socks') and self._proxy_ssl_context:
+            extra_kwargs['proxy_ssl_context'] = self._proxy_ssl_context
+        return super().proxy_manager_for(proxy, **proxy_kwargs, **self._pm_args, **extra_kwargs)
+
+    def cert_verify(*args, **kwargs):
+        # lean on SSLContext for cert verification
+        pass
+
+
+class RequestsSession(requests.sessions.Session):
+    """
+    Ensure unified redirect method handling with our urllib redirect handler.
+    """
+    def rebuild_method(self, prepared_request, response):
+        new_method = get_redirect_method(prepared_request.method, response.status_code)
+
+        # HACK: requests removes headers/body on redirect unless code was a 307/308.
+        if new_method == prepared_request.method:
+            response._real_status_code = response.status_code
+            response.status_code = 308
+
+        prepared_request.method = new_method
+
+    def rebuild_auth(self, prepared_request, response):
+        # HACK: undo status code change from rebuild_method, if applicable.
+        # rebuild_auth runs after requests would remove headers/body based on status code
+        if hasattr(response, '_real_status_code'):
+            response.status_code = response._real_status_code
+            del response._real_status_code
+        return super().rebuild_auth(prepared_request, response)
+
+
+class Urllib3LoggingFilter(logging.Filter):
+
+    def filter(self, record):
+        # Ignore HTTP request messages since HTTPConnection prints those
+        if record.msg == '%s://%s:%s "%s %s %s" %s %s':
+            return False
+        return True
+
+
+class Urllib3LoggingHandler(logging.Handler):
+    """Redirect urllib3 logs to our logger"""
+    def __init__(self, logger, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._logger = logger
+
+    def emit(self, record):
+        try:
+            msg = self.format(record)
+            if record.levelno >= logging.ERROR:
+                self._logger.error(msg)
+            else:
+                self._logger.stdout(msg)
+
+        except Exception:
+            self.handleError(record)
+
+
+@register_rh
+class RequestsRH(RequestHandler, InstanceStoreMixin):
+
+    """Requests RequestHandler
+    https://github.com/psf/requests
+    """
+    _SUPPORTED_URL_SCHEMES = ('http', 'https')
+    _SUPPORTED_ENCODINGS = tuple(SUPPORTED_ENCODINGS)
+    _SUPPORTED_PROXY_SCHEMES = ('http', 'https', 'socks4', 'socks4a', 'socks5', 'socks5h')
+    _SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY)
+    RH_NAME = 'requests'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # Forward urllib3 debug messages to our logger
+        logger = logging.getLogger('urllib3')
+        handler = Urllib3LoggingHandler(logger=self._logger)
+        handler.setFormatter(logging.Formatter('requests: %(message)s'))
+        handler.addFilter(Urllib3LoggingFilter())
+        logger.addHandler(handler)
+        logger.setLevel(logging.WARNING)
+
+        if self.verbose:
+            # Setting this globally is not ideal, but is easier than hacking with urllib3.
+            # It could technically be problematic for scripts embedding yt-dlp.
+            # However, it is unlikely debug traffic is used in that context in a way this will cause problems.
+            urllib3.connection.HTTPConnection.debuglevel = 1
+            logger.setLevel(logging.DEBUG)
+        # this is expected if we are using --no-check-certificate
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    def close(self):
+        self._clear_instances()
+
+    def _check_extensions(self, extensions):
+        super()._check_extensions(extensions)
+        extensions.pop('cookiejar', None)
+        extensions.pop('timeout', None)
+
+    def _create_instance(self, cookiejar):
+        session = RequestsSession()
+        http_adapter = RequestsHTTPAdapter(
+            ssl_context=self._make_sslcontext(),
+            source_address=self.source_address,
+            max_retries=urllib3.util.retry.Retry(False),
+        )
+        session.adapters.clear()
+        session.headers = requests.models.CaseInsensitiveDict({'Connection': 'keep-alive'})
+        session.mount('https://', http_adapter)
+        session.mount('http://', http_adapter)
+        session.cookies = cookiejar
+        session.trust_env = False  # no need, we already load proxies from env
+        return session
+
+    def _send(self, request):
+
+        headers = self._merge_headers(request.headers)
+        add_accept_encoding_header(headers, SUPPORTED_ENCODINGS)
+
+        max_redirects_exceeded = False
+
+        session = self._get_instance(
+            cookiejar=request.extensions.get('cookiejar') or self.cookiejar)
+
+        try:
+            requests_res = session.request(
+                method=request.method,
+                url=request.url,
+                data=request.data,
+                headers=headers,
+                timeout=float(request.extensions.get('timeout') or self.timeout),
+                proxies=request.proxies or self.proxies,
+                allow_redirects=True,
+                stream=True
+            )
+
+        except requests.exceptions.TooManyRedirects as e:
+            max_redirects_exceeded = True
+            requests_res = e.response
+
+        except requests.exceptions.SSLError as e:
+            if 'CERTIFICATE_VERIFY_FAILED' in str(e):
+                raise CertificateVerifyError(cause=e) from e
+            raise SSLError(cause=e) from e
+
+        except requests.exceptions.ProxyError as e:
+            raise ProxyError(cause=e) from e
+
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
+            raise TransportError(cause=e) from e
+
+        except urllib3.exceptions.HTTPError as e:
+            # Catch any urllib3 exceptions that may leak through
+            raise TransportError(cause=e) from e
+
+        except requests.exceptions.RequestException as e:
+            # Miscellaneous Requests exceptions. May not necessary be network related e.g. InvalidURL
+            raise RequestError(cause=e) from e
+
+        res = RequestsResponseAdapter(requests_res)
+
+        if not 200 <= res.status < 300:
+            raise HTTPError(res, redirect_loop=max_redirects_exceeded)
+
+        return res
+
+
+@register_preference(RequestsRH)
+def requests_preference(rh, request):
+    return 100
+
+
+# Use our socks proxy implementation with requests to avoid an extra dependency.
+class SocksHTTPConnection(urllib3.connection.HTTPConnection):
+    def __init__(self, _socks_options, *args, **kwargs):  # must use _socks_options to pass PoolKey checks
+        self._proxy_args = _socks_options
+        super().__init__(*args, **kwargs)
+
+    def _new_conn(self):
+        try:
+            return create_connection(
+                address=(self._proxy_args['addr'], self._proxy_args['port']),
+                timeout=self.timeout,
+                source_address=self.source_address,
+                _create_socket_func=functools.partial(
+                    create_socks_proxy_socket, (self.host, self.port), self._proxy_args))
+        except (socket.timeout, TimeoutError) as e:
+            raise urllib3.exceptions.ConnectTimeoutError(
+                self, f'Connection to {self.host} timed out. (connect timeout={self.timeout})') from e
+        except SocksProxyError as e:
+            raise urllib3.exceptions.ProxyError(str(e), e) from e
+        except (OSError, socket.error) as e:
+            raise urllib3.exceptions.NewConnectionError(
+                self, f'Failed to establish a new connection: {e}') from e
+
+
+class SocksHTTPSConnection(SocksHTTPConnection, urllib3.connection.HTTPSConnection):
+    pass
+
+
+class SocksHTTPConnectionPool(urllib3.HTTPConnectionPool):
+    ConnectionCls = SocksHTTPConnection
+
+
+class SocksHTTPSConnectionPool(urllib3.HTTPSConnectionPool):
+    ConnectionCls = SocksHTTPSConnection
+
+
+class SocksProxyManager(urllib3.PoolManager):
+
+    def __init__(self, socks_proxy, username=None, password=None, num_pools=10, headers=None, **connection_pool_kw):
+        connection_pool_kw['_socks_options'] = make_socks_proxy_opts(socks_proxy)
+        super().__init__(num_pools, headers, **connection_pool_kw)
+        self.pool_classes_by_scheme = {
+            'http': SocksHTTPConnectionPool,
+            'https': SocksHTTPSConnectionPool
+        }
+
+
+requests.adapters.SOCKSProxyManager = SocksProxyManager
diff --git a/yt_dlp/networking/_urllib.py b/yt_dlp/networking/_urllib.py
index 9e2bf33e4..68bab2b08 100644
--- a/yt_dlp/networking/_urllib.py
+++ b/yt_dlp/networking/_urllib.py
@@ -3,7 +3,6 @@
 import functools
 import http.client
 import io
-import socket
 import ssl
 import urllib.error
 import urllib.parse
@@ -24,6 +23,7 @@
     InstanceStoreMixin,
     add_accept_encoding_header,
     create_connection,
+    create_socks_proxy_socket,
     get_redirect_method,
     make_socks_proxy_opts,
     select_proxy,
@@ -40,7 +40,6 @@
 )
 from ..dependencies import brotli
 from ..socks import ProxyError as SocksProxyError
-from ..socks import sockssocket
 from ..utils import update_url_query
 from ..utils.networking import normalize_url
 
@@ -190,25 +189,12 @@ class SocksConnection(base_class):
         _create_connection = create_connection
 
         def connect(self):
-            def sock_socket_connect(ip_addr, timeout, source_address):
-                af, socktype, proto, canonname, sa = ip_addr
-                sock = sockssocket(af, socktype, proto)
-                try:
-                    connect_proxy_args = proxy_args.copy()
-                    connect_proxy_args.update({'addr': sa[0], 'port': sa[1]})
-                    sock.setproxy(**connect_proxy_args)
-                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:  # noqa: E721
-                        sock.settimeout(timeout)
-                    if source_address:
-                        sock.bind(source_address)
-                    sock.connect((self.host, self.port))
-                    return sock
-                except socket.error:
-                    sock.close()
-                    raise
             self.sock = create_connection(
-                (proxy_args['addr'], proxy_args['port']), timeout=self.timeout,
-                source_address=self.source_address, _create_socket_func=sock_socket_connect)
+                (proxy_args['addr'], proxy_args['port']),
+                timeout=self.timeout,
+                source_address=self.source_address,
+                _create_socket_func=functools.partial(
+                    create_socks_proxy_socket, (self.host, self.port), proxy_args))
             if isinstance(self, http.client.HTTPSConnection):
                 self.sock = self._context.wrap_socket(self.sock, server_hostname=self.host)
 
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 85a6402a6..4254974fc 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -471,11 +471,12 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
                 'no-attach-info-json', 'embed-thumbnail-atomicparsley', 'no-external-downloader-progress',
                 'embed-metadata', 'seperate-video-versions', 'no-clean-infojson', 'no-keep-subs', 'no-certifi',
                 'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-youtube-prefer-utc-upload-date',
+                'prefer-legacy-http-handler'
             }, 'aliases': {
                 'youtube-dl': ['all', '-multistreams', '-playlist-match-filter'],
                 'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter'],
                 '2021': ['2022', 'no-certifi', 'filename-sanitization', 'no-youtube-prefer-utc-upload-date'],
-                '2022': ['no-external-downloader-progress', 'playlist-match-filter'],
+                '2022': ['no-external-downloader-progress', 'playlist-match-filter', 'prefer-legacy-http-handler'],
             }
         }, help=(
             'Options that can help keep compatibility with youtube-dl or youtube-dlc '

From 4e38e2ae9d7380015349e6aee59c78bb3938befd Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 15 Oct 2023 10:54:38 +0200
Subject: [PATCH 635/871] [rh:requests] Handle both `bytes` and `int` for
 `IncompleteRead.partial` (Fix 8a8b54523addf46dfd50ef599761a81bc22362e6)
 (#8348)

Authored by: bashonly, coletdjnz, Grub4K
---
 test/test_networking.py         |  2 +-
 yt_dlp/networking/_requests.py  | 11 +++++------
 yt_dlp/networking/exceptions.py |  2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index 2b45deac7..689161fb2 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -865,7 +865,7 @@ def request(self, *args, **kwargs):
             '3 bytes read, 4 more expected'
         ),
         (
-            lambda: urllib3.exceptions.IncompleteRead(partial=3, expected=5),
+            lambda: urllib3.exceptions.ProtocolError('error', urllib3.exceptions.IncompleteRead(partial=3, expected=5)),
             IncompleteRead,
             '3 bytes read, 5 more expected'
         ),
diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py
index 27974357a..fe3f60b0b 100644
--- a/yt_dlp/networking/_requests.py
+++ b/yt_dlp/networking/_requests.py
@@ -142,18 +142,17 @@ def read(self, amt: int = None):
         except urllib3.exceptions.SSLError as e:
             raise SSLError(cause=e) from e
 
-        except urllib3.exceptions.IncompleteRead as e:
-            # urllib3 IncompleteRead.partial is always an integer
-            raise IncompleteRead(partial=e.partial, expected=e.expected) from e
-
         except urllib3.exceptions.ProtocolError as e:
-            # http.client.IncompleteRead may be contained within ProtocolError
+            # IncompleteRead is always contained within ProtocolError
             # See urllib3.response.HTTPResponse._error_catcher()
             ir_err = next(
                 (err for err in (e.__context__, e.__cause__, *variadic(e.args))
                  if isinstance(err, http.client.IncompleteRead)), None)
             if ir_err is not None:
-                raise IncompleteRead(partial=len(ir_err.partial), expected=ir_err.expected) from e
+                # `urllib3.exceptions.IncompleteRead` is subclass of `http.client.IncompleteRead`
+                # but uses an `int` for its `partial` property.
+                partial = ir_err.partial if isinstance(ir_err.partial, int) else len(ir_err.partial)
+                raise IncompleteRead(partial=partial, expected=ir_err.expected) from e
             raise TransportError(cause=e) from e
 
         except urllib3.exceptions.HTTPError as e:
diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py
index f58dc246e..12441901c 100644
--- a/yt_dlp/networking/exceptions.py
+++ b/yt_dlp/networking/exceptions.py
@@ -75,7 +75,7 @@ def __repr__(self):
 
 
 class IncompleteRead(TransportError):
-    def __init__(self, partial: int, expected: int = None, **kwargs):
+    def __init__(self, partial: int, expected: int | None = None, **kwargs):
         self.partial = partial
         self.expected = expected
         msg = f'{partial} bytes read'

From a40e0b37dfc8c26916b0e01aa3f29f3bc42250b6 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 22 Oct 2023 18:05:22 -0500
Subject: [PATCH 636/871] [core] Only ensure playlist thumbnail dir if writing
 thumbs (#8373)

Bugfix for 2acd1d555ef89851c73773776715d3de9a0e30b9

Closes #8372
Authored by: bashonly
---
 yt_dlp/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 8e11646d3..d066efdc7 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -4242,7 +4242,7 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
             self.write_debug(f'Skipping writing {label} thumbnail')
             return ret
 
-        if not self._ensure_dir_exists(filename):
+        if thumbnails and not self._ensure_dir_exists(filename):
             return None
 
         for idx, t in list(enumerate(thumbnails))[::-1]:

From 7b8b1cf5eb8bf44ce70bc24e1f56f0dba2737e98 Mon Sep 17 00:00:00 2001
From: saintliao <saintliao@gmail.com>
Date: Sat, 28 Oct 2023 07:59:13 +0800
Subject: [PATCH 637/871] [ie/twitcasting] Fix livestream extraction (#8427)

Closes #8431
Authored by: JC-Chung, saintliao

Co-authored-by: JC-Chung <52159296+JC-Chung@users.noreply.github.com>
---
 yt_dlp/extractor/twitcasting.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py
index 540e217fd..32a38c86e 100644
--- a/yt_dlp/extractor/twitcasting.py
+++ b/yt_dlp/extractor/twitcasting.py
@@ -142,7 +142,7 @@ def _real_extract(self, url):
             'https://twitcasting.tv/streamserver.php?target=%s&mode=client' % uploader_id, video_id,
             'Downloading live info', fatal=False)
 
-        is_live = 'data-status="online"' in webpage
+        is_live = any(f'data-{x}' in webpage for x in ['is-onlive="true"', 'live-type="live"', 'status="online"'])
         if not traverse_obj(stream_server_data, 'llfmp4') and is_live:
             self.raise_login_required(method='cookies')
 

From 8e02a4dcc800f9444e9d461edc41edd7b662f435 Mon Sep 17 00:00:00 2001
From: Bart Broere <mail@bartbroere.eu>
Date: Sat, 28 Oct 2023 02:00:12 +0200
Subject: [PATCH 638/871] [ie/npo] Send `POST` request to streams API endpoint
 (#8413)

Closes #6398
Authored by: bartbroere
---
 yt_dlp/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/npo.py b/yt_dlp/extractor/npo.py
index 40fee24d0..4d5ff50de 100644
--- a/yt_dlp/extractor/npo.py
+++ b/yt_dlp/extractor/npo.py
@@ -245,7 +245,7 @@ def _real_extract(self, url):
                     'quality': 'npoplus',
                     'tokenId': player_token,
                     'streamType': 'broadcast',
-                })
+                }, data=b'')  # endpoint requires POST
             if not streams:
                 continue
             stream = streams.get('stream')

From 177f0d963e4b9db749805c482e6f288354c8be84 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 27 Oct 2023 19:01:31 -0500
Subject: [PATCH 639/871] [ie/QDance] Update `_VALID_URL` (#8426)

Authored by: bashonly
---
 yt_dlp/extractor/qdance.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/qdance.py b/yt_dlp/extractor/qdance.py
index d817677f0..934ebbfd7 100644
--- a/yt_dlp/extractor/qdance.py
+++ b/yt_dlp/extractor/qdance.py
@@ -15,7 +15,7 @@
 
 class QDanceIE(InfoExtractor):
     _NETRC_MACHINE = 'qdance'
-    _VALID_URL = r'https?://(?:www\.)?q-dance\.com/network/(?:library|live)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?q-dance\.com/network/(?:library|live)/(?P<id>[\w-]+)'
     _TESTS = [{
         'note': 'vod',
         'url': 'https://www.q-dance.com/network/library/146542138',
@@ -53,6 +53,27 @@ class QDanceIE(InfoExtractor):
             'channel_id': 'qdancenetwork.video_149170353',
         },
         'skip': 'Completed livestream',
+    }, {
+        'note': 'vod with alphanumeric id',
+        'url': 'https://www.q-dance.com/network/library/WhDleSIWSfeT3Q9ObBKBeA',
+        'info_dict': {
+            'id': 'WhDleSIWSfeT3Q9ObBKBeA',
+            'ext': 'mp4',
+            'title': 'Aftershock I Defqon.1 Weekend Festival 2023 I Sunday I BLUE',
+            'display_id': 'naam-i-defqon-1-weekend-festival-2023-i-dag-i-podium',
+            'description': 'Relive Defqon.1 Path of the Warrior with Aftershock at the BLUE 🔥',
+            'series': 'Defqon.1',
+            'series_id': '31840378',
+            'season': 'Defqon.1 Weekend Festival 2023',
+            'season_id': '141735599',
+            'duration': 3507,
+            'availability': 'premium_only',
+            'thumbnail': 'https://images.q-dance.network/1698158361-230625-135716-defqon-1-aftershock.jpg',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.q-dance.com/network/library/-uRFKXwmRZGVnve7av9uqA',
+        'only_matching': True,
     }]
 
     _access_token = None

From 4ce2f29a50fcfb9920e6f2ffe42192945a2bad7e Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 27 Oct 2023 19:35:37 -0500
Subject: [PATCH 640/871] [ie/generic] Improve direct video link ext detection
 (#8340)

Closes #8265
Authored by: bashonly
---
 yt_dlp/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 5e1240c13..ac7cc673f 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -34,6 +34,7 @@
     unified_timestamp,
     unsmuggle_url,
     update_url_query,
+    urlhandle_detect_ext,
     url_or_none,
     urljoin,
     variadic,
@@ -2459,7 +2460,7 @@ def _real_extract(self, url):
             self.report_detected('direct video link')
             headers = smuggled_data.get('http_headers', {})
             format_id = str(m.group('format_id'))
-            ext = determine_ext(url)
+            ext = determine_ext(url, default_ext=None) or urlhandle_detect_ext(full_response)
             subtitles = {}
             if format_id.endswith('mpegurl') or ext == 'm3u8':
                 formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4', headers=headers)
@@ -2471,6 +2472,7 @@ def _real_extract(self, url):
                 formats = [{
                     'format_id': format_id,
                     'url': url,
+                    'ext': ext,
                     'vcodec': 'none' if m.group('type') == 'audio' else None
                 }]
                 info_dict['direct'] = True

From 39abae23546160fa98ac2b0c91e3d69fa965b573 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 28 Oct 2023 03:01:31 -0500
Subject: [PATCH 641/871] [ie/youtube] Deprioritize iOS client formats (#8337)

Authored by: bashonly
---
 README.md                   | 2 +-
 yt_dlp/extractor/youtube.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3b7432474..d464519f0 100644
--- a/README.md
+++ b/README.md
@@ -1801,7 +1801,7 @@ # EXTRACTOR ARGUMENTS
 #### youtube
 * `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube.py](https://github.com/yt-dlp/yt-dlp/blob/c26f9b991a0681fd3ea548d535919cec1fbbd430/yt_dlp/extractor/youtube.py#L381-L390) for list of supported content language codes
 * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
-* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb`, `mweb_embedscreen` and `tv_embedded` (agegate bypass) with no variants. By default, `ios,android,web` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly, the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
+* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb`, `mweb_embedscreen` and `tv_embedded` (agegate bypass) with no variants. By default, `android,web,ios` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly, the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
 * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
 * `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp.
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index ac28ed7d2..ded0f9150 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3619,7 +3619,7 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
 
     def _get_requested_clients(self, url, smuggled_data):
         requested_clients = []
-        default = ['ios', 'android', 'web']
+        default = ['android', 'web', 'ios']
         allowed_clients = sorted(
             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)

From ef79d20dc9d27ac002a7196f073b37f2f2721aed Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 28 Oct 2023 03:02:13 -0500
Subject: [PATCH 642/871] [ie/youtube] Check newly uploaded iOS HLS formats
 (#8336)

Closes #7747
Authored by: bashonly
---
 yt_dlp/YoutubeDL.py         | 2 +-
 yt_dlp/extractor/youtube.py | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index d066efdc7..f10167826 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2338,7 +2338,7 @@ def _check_formats(formats):
                 return
 
             for f in formats:
-                if f.get('has_drm'):
+                if f.get('has_drm') or f.get('__needs_testing'):
                     yield from self._check_formats([f])
                 else:
                     yield f
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index ded0f9150..ddbd614d6 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4560,6 +4560,14 @@ def process_language(container, base_url, lang_code, sub_name, query):
                 self._parse_time_text(self._get_text(vpir, 'dateText'))) or upload_date
         info['upload_date'] = upload_date
 
+        if upload_date and live_status not in ('is_live', 'post_live', 'is_upcoming'):
+            # Newly uploaded videos' HLS formats are potentially problematic and need to be checked
+            upload_datetime = datetime_from_str(upload_date).replace(tzinfo=datetime.timezone.utc)
+            if upload_datetime >= datetime_from_str('today-1day'):
+                for fmt in info['formats']:
+                    if fmt.get('protocol') == 'm3u8_native':
+                        fmt['__needs_testing'] = True
+
         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
             v = info.get(s_k)
             if v:

From 464327acdb353ceb91d2115163a5a9621b22fe0d Mon Sep 17 00:00:00 2001
From: Shubham <shubh622005@gmail.com>
Date: Sat, 28 Oct 2023 21:20:08 +0530
Subject: [PATCH 643/871] [ie/polskieradio:audition] Fix playlist extraction
 (#8459)

Closes #8419
Authored by: shubhexists
---
 yt_dlp/extractor/polskieradio.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/polskieradio.py b/yt_dlp/extractor/polskieradio.py
index 5bf92b9b5..e0b22fffd 100644
--- a/yt_dlp/extractor/polskieradio.py
+++ b/yt_dlp/extractor/polskieradio.py
@@ -262,14 +262,14 @@ def _call_lp3(self, path, query, video_id, note):
             query=query, headers={'x-api-key': '9bf6c5a2-a7d0-4980-9ed7-a3f7291f2a81'})
 
     def _entries(self, playlist_id, has_episodes, has_articles):
-        for i in itertools.count(1) if has_episodes else []:
+        for i in itertools.count(0) if has_episodes else []:
             page = self._call_lp3(
                 'AudioArticle/GetListByCategoryId', {
                     'categoryId': playlist_id,
                     'PageSize': 10,
                     'skip': i,
                     'format': 400,
-                }, playlist_id, f'Downloading episode list page {i}')
+                }, playlist_id, f'Downloading episode list page {i + 1}')
             if not traverse_obj(page, 'data'):
                 break
             for episode in page['data']:
@@ -281,14 +281,14 @@ def _entries(self, playlist_id, has_episodes, has_articles):
                     'timestamp': parse_iso8601(episode.get('datePublic')),
                 }
 
-        for i in itertools.count(1) if has_articles else []:
+        for i in itertools.count(0) if has_articles else []:
             page = self._call_lp3(
                 'Article/GetListByCategoryId', {
                     'categoryId': playlist_id,
                     'PageSize': 9,
                     'skip': i,
                     'format': 400,
-                }, playlist_id, f'Downloading article list page {i}')
+                }, playlist_id, f'Downloading article list page {i + 1}')
             if not traverse_obj(page, 'data'):
                 break
             for article in page['data']:

From 4a601c9eff9fb42e24a4c8da3fa03628e035b35b Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sat, 28 Oct 2023 17:53:24 +0200
Subject: [PATCH 644/871] [ie/weverse] Fix login error handling (#8458)

Authored by: seproDev
---
 yt_dlp/extractor/weverse.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/weverse.py b/yt_dlp/extractor/weverse.py
index 47f36806b..c94ca9db9 100644
--- a/yt_dlp/extractor/weverse.py
+++ b/yt_dlp/extractor/weverse.py
@@ -45,10 +45,10 @@ def _perform_login(self, username, password):
             'x-acc-trace-id': str(uuid.uuid4()),
             'x-clog-user-device-id': str(uuid.uuid4()),
         }
-        check_username = self._download_json(
-            f'{self._ACCOUNT_API_BASE}/signup/email/status', None,
-            note='Checking username', query={'email': username}, headers=headers)
-        if not check_username.get('hasPassword'):
+        valid_username = traverse_obj(self._download_json(
+            f'{self._ACCOUNT_API_BASE}/signup/email/status', None, note='Checking username',
+            query={'email': username}, headers=headers, expected_status=(400, 404)), 'hasPassword')
+        if not valid_username:
             raise ExtractorError('Invalid username provided', expected=True)
 
         headers['content-type'] = 'application/json'

From 21b25281c51523620706b11bfc1c4a889858e1f2 Mon Sep 17 00:00:00 2001
From: CrendKing <975235+CrendKing@users.noreply.github.com>
Date: Tue, 7 Nov 2023 08:18:19 -0800
Subject: [PATCH 645/871] [fd/aria2c] Remove duplicate `--file-allocation=none`
 (#8332)

Authored by: CrendKing
---
 yt_dlp/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index 4ce8a3bf7..ce5eeb0a9 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -335,7 +335,7 @@ def _make_cmd(self, tmpfilename, info_dict):
         cmd += ['--auto-file-renaming=false']
 
         if 'fragments' in info_dict:
-            cmd += ['--file-allocation=none', '--uri-selector=inorder']
+            cmd += ['--uri-selector=inorder']
             url_list_file = '%s.frag.urls' % tmpfilename
             url_list = []
             for frag_index, fragment in enumerate(info_dict['fragments']):

From fd8fcf8f4f7555be807fbafa5565586f565374ee Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Tue, 7 Nov 2023 14:55:12 -0600
Subject: [PATCH 646/871] Revert 39abae23546160fa98ac2b0c91e3d69fa965b573

The iOS client is not subject to integrity checks and is likely to be a more stable choice going forward

Authored by: bashonly
---
 README.md                   | 2 +-
 yt_dlp/extractor/youtube.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index d464519f0..3b7432474 100644
--- a/README.md
+++ b/README.md
@@ -1801,7 +1801,7 @@ # EXTRACTOR ARGUMENTS
 #### youtube
 * `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube.py](https://github.com/yt-dlp/yt-dlp/blob/c26f9b991a0681fd3ea548d535919cec1fbbd430/yt_dlp/extractor/youtube.py#L381-L390) for list of supported content language codes
 * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
-* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb`, `mweb_embedscreen` and `tv_embedded` (agegate bypass) with no variants. By default, `android,web,ios` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly, the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
+* `player_client`: Clients to extract video data from. The main clients are `web`, `android` and `ios` with variants `_music`, `_embedded`, `_embedscreen`, `_creator` (e.g. `web_embedded`); and `mweb`, `mweb_embedscreen` and `tv_embedded` (agegate bypass) with no variants. By default, `ios,android,web` is used, but `tv_embedded` and `creator` variants are added as required for age-gated videos. Similarly, the music variants are added for `music.youtube.com` urls. You can use `all` to use all the clients, and `default` for the default clients.
 * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
 * `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp.
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index ddbd614d6..adbac8e95 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3619,7 +3619,7 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
 
     def _get_requested_clients(self, url, smuggled_data):
         requested_clients = []
-        default = ['android', 'web', 'ios']
+        default = ['ios', 'android', 'web']
         allowed_clients = sorted(
             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)

From 2622c804d1a5accc3045db398e0fc52074f4bdb3 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 7 Nov 2023 15:28:34 -0600
Subject: [PATCH 647/871] [fd/dash] Force native downloader for
 `--live-from-start` (#8339)

Closes #8212
Authored by: bashonly
---
 yt_dlp/downloader/dash.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/downloader/dash.py b/yt_dlp/downloader/dash.py
index 4328d739c..afc79b6ca 100644
--- a/yt_dlp/downloader/dash.py
+++ b/yt_dlp/downloader/dash.py
@@ -15,12 +15,15 @@ class DashSegmentsFD(FragmentFD):
     FD_NAME = 'dashsegments'
 
     def real_download(self, filename, info_dict):
-        if info_dict.get('is_live') and set(info_dict['protocol'].split('+')) != {'http_dash_segments_generator'}:
-            self.report_error('Live DASH videos are not supported')
+        if 'http_dash_segments_generator' in info_dict['protocol'].split('+'):
+            real_downloader = None  # No external FD can support --live-from-start
+        else:
+            if info_dict.get('is_live'):
+                self.report_error('Live DASH videos are not supported')
+            real_downloader = get_suitable_downloader(
+                info_dict, self.params, None, protocol='dash_frag_urls', to_stdout=(filename == '-'))
 
         real_start = time.time()
-        real_downloader = get_suitable_downloader(
-            info_dict, self.params, None, protocol='dash_frag_urls', to_stdout=(filename == '-'))
 
         requested_formats = [{**info_dict, **fmt} for fmt in info_dict.get('requested_formats', [])]
         args = []

From 595ea4a99b726b8fe9463e7853b7053978d0544e Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 7 Nov 2023 16:48:15 -0600
Subject: [PATCH 648/871] [core] Fix format sorting with `--load-info-json`
 (#8521)

Closes #7971
Authored by: bashonly
---
 yt_dlp/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index f10167826..d5c0a2422 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -3550,7 +3550,7 @@ def sanitize_info(info_dict, remove_private_keys=False):
             reject = lambda k, v: v is None or k.startswith('__') or k in {
                 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
                 'entries', 'filepath', '_filename', 'filename', 'infojson_filename', 'original_url',
-                'playlist_autonumber', '_format_sort_fields',
+                'playlist_autonumber',
             }
         else:
             reject = lambda k, v: False

From 10025b715ea01489557eb2c5a3cc04d361fcdb52 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 7 Nov 2023 17:10:01 -0600
Subject: [PATCH 649/871] [core] Add `--compat-option manifest-filesize-approx`
 (#8356)

Closes #7623
Authored by: bashonly
---
 README.md           | 1 +
 yt_dlp/YoutubeDL.py | 3 ++-
 yt_dlp/options.py   | 8 ++++----
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 3b7432474..52f8bf799 100644
--- a/README.md
+++ b/README.md
@@ -157,6 +157,7 @@ ### Differences in default behavior
 * yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
 * yt-dlp tries to parse the external downloader outputs into the standard progress output if possible (Currently implemented: [~~aria2c~~](https://github.com/yt-dlp/yt-dlp/issues/5931)). You can use `--compat-options no-external-downloader-progress` to get the downloader output as-is
 * yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filter` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this
+* yt-dlp versions between 2021.11.10 and 2023.06.21 estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values
 * yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests.
 
 For ease of use, a few more compat options are available:
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index d5c0a2422..fb8e89443 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2764,7 +2764,8 @@ def is_wellformed(f):
                 format['dynamic_range'] = 'SDR'
             if format.get('aspect_ratio') is None:
                 format['aspect_ratio'] = try_call(lambda: round(format['width'] / format['height'], 2))
-            if (not format.get('manifest_url')  # For fragmented formats, "tbr" is often max bitrate and not average
+            # For fragmented formats, "tbr" is often max bitrate and not average
+            if (('manifest-filesize-approx' in self.params['compat_opts'] or not format.get('manifest_url'))
                     and info_dict.get('duration') and format.get('tbr')
                     and not format.get('filesize') and not format.get('filesize_approx')):
                 format['filesize_approx'] = int(info_dict['duration'] * format['tbr'] * (1024 / 8))
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 4254974fc..e9d927717 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -471,12 +471,12 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
                 'no-attach-info-json', 'embed-thumbnail-atomicparsley', 'no-external-downloader-progress',
                 'embed-metadata', 'seperate-video-versions', 'no-clean-infojson', 'no-keep-subs', 'no-certifi',
                 'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-youtube-prefer-utc-upload-date',
-                'prefer-legacy-http-handler'
+                'prefer-legacy-http-handler', 'manifest-filesize-approx'
             }, 'aliases': {
-                'youtube-dl': ['all', '-multistreams', '-playlist-match-filter'],
-                'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter'],
+                'youtube-dl': ['all', '-multistreams', '-playlist-match-filter', '-manifest-filesize-approx'],
+                'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter', '-manifest-filesize-approx'],
                 '2021': ['2022', 'no-certifi', 'filename-sanitization', 'no-youtube-prefer-utc-upload-date'],
-                '2022': ['no-external-downloader-progress', 'playlist-match-filter', 'prefer-legacy-http-handler'],
+                '2022': ['no-external-downloader-progress', 'playlist-match-filter', 'prefer-legacy-http-handler', 'manifest-filesize-approx'],
             }
         }, help=(
             'Options that can help keep compatibility with youtube-dl or youtube-dlc '

From 7d337ca977d73a0a6c07ab481ed8faa8f6ff8726 Mon Sep 17 00:00:00 2001
From: HitomaruKonpaku <HitomaruKonpaku@users.noreply.github.com>
Date: Sat, 11 Nov 2023 08:34:22 +0700
Subject: [PATCH 650/871] [ie/twitter:broadcast] Improve metadata extraction
 (#8383)

Authored by: HitomaruKonpaku
---
 yt_dlp/extractor/periscope.py |  5 +++--
 yt_dlp/extractor/twitter.py   | 40 ++++++++++++++++++++++++++++++++---
 2 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/periscope.py b/yt_dlp/extractor/periscope.py
index 84bcf1573..dcd021926 100644
--- a/yt_dlp/extractor/periscope.py
+++ b/yt_dlp/extractor/periscope.py
@@ -24,12 +24,13 @@ def _parse_broadcast_data(self, broadcast, video_id):
 
         thumbnails = [{
             'url': broadcast[image],
-        } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
+        } for image in ('image_url', 'image_url_medium', 'image_url_small') if broadcast.get(image)]
 
         return {
             'id': broadcast.get('id') or video_id,
             'title': title,
-            'timestamp': parse_iso8601(broadcast.get('created_at')),
+            'timestamp': parse_iso8601(broadcast.get('created_at')) or int_or_none(
+                broadcast.get('created_at_ms'), scale=1000),
             'uploader': uploader,
             'uploader_id': broadcast.get('user_id') or broadcast.get('username'),
             'thumbnails': thumbnails,
diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index b6386214d..7bd78eb48 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1563,7 +1563,7 @@ class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE):
     IE_NAME = 'twitter:broadcast'
     _VALID_URL = TwitterBaseIE._BASE_REGEX + r'i/broadcasts/(?P<id>[0-9a-zA-Z]{13})'
 
-    _TEST = {
+    _TESTS = [{
         # untitled Periscope video
         'url': 'https://twitter.com/i/broadcasts/1yNGaQLWpejGj',
         'info_dict': {
@@ -1571,11 +1571,42 @@ class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE):
             'ext': 'mp4',
             'title': 'Andrea May Sahouri - Periscope Broadcast',
             'uploader': 'Andrea May Sahouri',
-            'uploader_id': '1PXEdBZWpGwKe',
+            'uploader_id': 'andreamsahouri',
+            'uploader_url': 'https://twitter.com/andreamsahouri',
+            'timestamp': 1590973638,
+            'upload_date': '20200601',
             'thumbnail': r're:^https?://[^?#]+\.jpg\?token=',
             'view_count': int,
         },
-    }
+    }, {
+        'url': 'https://twitter.com/i/broadcasts/1ZkKzeyrPbaxv',
+        'info_dict': {
+            'id': '1ZkKzeyrPbaxv',
+            'ext': 'mp4',
+            'title': 'Starship | SN10 | High-Altitude Flight Test',
+            'uploader': 'SpaceX',
+            'uploader_id': 'SpaceX',
+            'uploader_url': 'https://twitter.com/SpaceX',
+            'timestamp': 1614812942,
+            'upload_date': '20210303',
+            'thumbnail': r're:^https?://[^?#]+\.jpg\?token=',
+            'view_count': int,
+        },
+    }, {
+        'url': 'https://twitter.com/i/broadcasts/1OyKAVQrgzwGb',
+        'info_dict': {
+            'id': '1OyKAVQrgzwGb',
+            'ext': 'mp4',
+            'title': 'Starship Flight Test',
+            'uploader': 'SpaceX',
+            'uploader_id': 'SpaceX',
+            'uploader_url': 'https://twitter.com/SpaceX',
+            'timestamp': 1681993964,
+            'upload_date': '20230420',
+            'thumbnail': r're:^https?://[^?#]+\.jpg\?token=',
+            'view_count': int,
+        },
+    }]
 
     def _real_extract(self, url):
         broadcast_id = self._match_id(url)
@@ -1585,6 +1616,9 @@ def _real_extract(self, url):
         if not broadcast:
             raise ExtractorError('Broadcast no longer exists', expected=True)
         info = self._parse_broadcast_data(broadcast, broadcast_id)
+        info['title'] = broadcast.get('status') or info.get('title')
+        info['uploader_id'] = broadcast.get('twitter_username') or info.get('uploader_id')
+        info['uploader_url'] = format_field(broadcast, 'twitter_username', 'https://twitter.com/%s', default=None)
         media_key = broadcast['media_key']
         source = self._call_api(
             f'live_video_stream/status/{media_key}', media_key)['source']

From 3906de07551fedb00b789345bf24cc27d6ddf128 Mon Sep 17 00:00:00 2001
From: Tom <tomaviv57@gmail.com>
Date: Sat, 11 Nov 2023 21:51:54 +0200
Subject: [PATCH 651/871] [ie/zoom] Extract combined view formats (#7847)

Authored by: Mipsters
---
 yt_dlp/extractor/zoom.py | 40 ++++++++++++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/zoom.py b/yt_dlp/extractor/zoom.py
index 329ba1415..e2bf81729 100644
--- a/yt_dlp/extractor/zoom.py
+++ b/yt_dlp/extractor/zoom.py
@@ -2,10 +2,12 @@
 from ..utils import (
     ExtractorError,
     int_or_none,
-    str_or_none,
     js_to_json,
     parse_filesize,
+    parse_resolution,
+    str_or_none,
     traverse_obj,
+    url_basename,
     urlencode_postdata,
     urljoin,
 )
@@ -41,6 +43,18 @@ class ZoomIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Timea Andrea Lelik\'s Personal Meeting Room',
         },
+        'skip': 'This recording has expired',
+    }, {
+        # view_with_share URL
+        'url': 'https://cityofdetroit.zoom.us/rec/share/VjE-5kW3xmgbEYqR5KzRgZ1OFZvtMtiXk5HyRJo5kK4m5PYE6RF4rF_oiiO_9qaM.UTAg1MI7JSnF3ZjX',
+        'md5': 'bdc7867a5934c151957fb81321b3c024',
+        'info_dict': {
+            'id': 'VjE-5kW3xmgbEYqR5KzRgZ1OFZvtMtiXk5HyRJo5kK4m5PYE6RF4rF_oiiO_9qaM.UTAg1MI7JSnF3ZjX',
+            'ext': 'mp4',
+            'title': 'February 2022 Detroit Revenue Estimating Conference',
+            'duration': 7299,
+            'formats': 'mincount:3',
+        },
     }]
 
     def _get_page_data(self, webpage, video_id):
@@ -72,6 +86,7 @@ def _get_real_webpage(self, url, base_url, video_id, url_type):
 
     def _real_extract(self, url):
         base_url, url_type, video_id = self._match_valid_url(url).group('base_url', 'type', 'id')
+        query = {}
 
         if url_type == 'share':
             webpage = self._get_real_webpage(url, base_url, video_id, 'share')
@@ -80,6 +95,7 @@ def _real_extract(self, url):
                 f'{base_url}nws/recording/1.0/play/share-info/{meeting_id}',
                 video_id, note='Downloading share info JSON')['result']['redirectUrl']
             url = urljoin(base_url, redirect_path)
+            query['continueMode'] = 'true'
 
         webpage = self._get_real_webpage(url, base_url, video_id, 'play')
         file_id = self._get_page_data(webpage, video_id)['fileId']
@@ -88,7 +104,7 @@ def _real_extract(self, url):
             raise ExtractorError('Unable to extract file ID')
 
         data = self._download_json(
-            f'{base_url}nws/recording/1.0/play/info/{file_id}', video_id,
+            f'{base_url}nws/recording/1.0/play/info/{file_id}', video_id, query=query,
             note='Downloading play info JSON')['result']
 
         subtitles = {}
@@ -104,10 +120,10 @@ def _real_extract(self, url):
         if data.get('viewMp4Url'):
             formats.append({
                 'format_note': 'Camera stream',
-                'url': str_or_none(data.get('viewMp4Url')),
+                'url': data['viewMp4Url'],
                 'width': int_or_none(traverse_obj(data, ('viewResolvtions', 0))),
                 'height': int_or_none(traverse_obj(data, ('viewResolvtions', 1))),
-                'format_id': str_or_none(traverse_obj(data, ('recording', 'id'))),
+                'format_id': 'view',
                 'ext': 'mp4',
                 'filesize_approx': parse_filesize(str_or_none(traverse_obj(data, ('recording', 'fileSizeInMB')))),
                 'preference': 0
@@ -116,14 +132,26 @@ def _real_extract(self, url):
         if data.get('shareMp4Url'):
             formats.append({
                 'format_note': 'Screen share stream',
-                'url': str_or_none(data.get('shareMp4Url')),
+                'url': data['shareMp4Url'],
                 'width': int_or_none(traverse_obj(data, ('shareResolvtions', 0))),
                 'height': int_or_none(traverse_obj(data, ('shareResolvtions', 1))),
-                'format_id': str_or_none(traverse_obj(data, ('shareVideo', 'id'))),
+                'format_id': 'share',
                 'ext': 'mp4',
                 'preference': -1
             })
 
+        view_with_share_url = data.get('viewMp4WithshareUrl')
+        if view_with_share_url:
+            formats.append({
+                **parse_resolution(self._search_regex(
+                    r'_(\d+x\d+)\.mp4', url_basename(view_with_share_url), 'resolution', default=None)),
+                'format_note': 'Screen share with camera',
+                'url': view_with_share_url,
+                'format_id': 'view_with_share',
+                'ext': 'mp4',
+                'preference': 1
+            })
+
         return {
             'id': video_id,
             'title': str_or_none(traverse_obj(data, ('meet', 'topic'))),

From 25a4bd345a0dcfece6fef752d4537eb403da94d9 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sat, 11 Nov 2023 20:53:10 +0100
Subject: [PATCH 652/871] [ie/sbs.co.kr] Add extractors (#8326)

Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py |   5 +
 yt_dlp/extractor/sbscokr.py     | 200 ++++++++++++++++++++++++++++++++
 2 files changed, 205 insertions(+)
 create mode 100644 yt_dlp/extractor/sbscokr.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ca4571182..cf6744487 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1758,6 +1758,11 @@
 from .sapo import SapoIE
 from .savefrom import SaveFromIE
 from .sbs import SBSIE
+from .sbscokr import (
+    SBSCoKrIE,
+    SBSCoKrAllvodProgramIE,
+    SBSCoKrProgramsVodIE,
+)
 from .screen9 import Screen9IE
 from .screencast import ScreencastIE
 from .screencastify import ScreencastifyIE
diff --git a/yt_dlp/extractor/sbscokr.py b/yt_dlp/extractor/sbscokr.py
new file mode 100644
index 000000000..001d19ee1
--- /dev/null
+++ b/yt_dlp/extractor/sbscokr.py
@@ -0,0 +1,200 @@
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+    parse_iso8601,
+    parse_resolution,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class SBSCoKrIE(InfoExtractor):
+    IE_NAME = 'sbs.co.kr'
+    _VALID_URL = [r'https?://allvod\.sbs\.co\.kr/allvod/vod(?:Package)?EndPage\.do\?(?:[^#]+&)?mdaId=(?P<id>\d+)',
+                  r'https?://programs\.sbs\.co\.kr/(?:enter|drama|culture|sports|plus|mtv|kth)/[a-z0-9]+/(?:vod|clip|movie)/\d+/(?P<id>(?:OC)?\d+)']
+
+    _TESTS = [{
+        'url': 'https://programs.sbs.co.kr/enter/dongsang2/clip/52007/OC467706746?div=main_pop_clip',
+        'md5': 'c3f6d45e1fb5682039d94cda23c36f19',
+        'info_dict': {
+            'id': 'OC467706746',
+            'ext': 'mp4',
+            'title': '‘아슬아슬’ 박군♥한영의 새 집 인테리어 대첩♨',
+            'description': 'md5:6a71eb1979ee4a94ea380310068ccab4',
+            'thumbnail': 'https://img2.sbs.co.kr/ops_clip_img/2023/10/10/34c4c0f9-a9a5-4ff6-a92e-9bb4b5f6fa65915w1280.jpg',
+            'release_timestamp': 1696889400,
+            'release_date': '20231009',
+            'view_count': int,
+            'like_count': int,
+            'duration': 238,
+            'age_limit': 15,
+            'series': '동상이몽2_너는 내 운명',
+            'episode': '레이디제인, ‘혼전임신설’ ‘3개월’ 앞당긴 결혼식 비하인드 스토리 최초 공개!',
+            'episode_number': 311,
+        },
+    }, {
+        'url': 'https://allvod.sbs.co.kr/allvod/vodPackageEndPage.do?mdaId=22000489324&combiId=PA000000284&packageType=A&isFreeYN=',
+        'md5': 'bf46b2e89fda7ae7de01f5743cef7236',
+        'info_dict': {
+            'id': '22000489324',
+            'ext': 'mp4',
+            'title': '[다시보기] 트롤리 15회',
+            'description': 'md5:0e55d74bef1ac55c61ae90c73ac485f4',
+            'thumbnail': 'https://img2.sbs.co.kr/img/sbs_cms/WE/2023/02/14/arC1676333794938-1280-720.jpg',
+            'release_timestamp': 1676325600,
+            'release_date': '20230213',
+            'view_count': int,
+            'like_count': int,
+            'duration': 5931,
+            'age_limit': 15,
+            'series': '트롤리',
+            'episode': '이거 다 거짓말이야',
+            'episode_number': 15,
+        },
+    }, {
+        'url': 'https://programs.sbs.co.kr/enter/fourman/vod/69625/22000508948',
+        'md5': '41e8ae4cc6c8424f4e4d76661a4becbf',
+        'info_dict': {
+            'id': '22000508948',
+            'ext': 'mp4',
+            'title': '[다시보기] 신발 벗고 돌싱포맨 104회',
+            'description': 'md5:c6a247383c4dd661e4b956bf4d3b586e',
+            'thumbnail': 'https://img2.sbs.co.kr/img/sbs_cms/WE/2023/08/30/2vb1693355446261-1280-720.jpg',
+            'release_timestamp': 1693342800,
+            'release_date': '20230829',
+            'view_count': int,
+            'like_count': int,
+            'duration': 7036,
+            'age_limit': 15,
+            'series': '신발 벗고 돌싱포맨',
+            'episode': '돌싱포맨 저격수들 등장!',
+            'episode_number': 104,
+        },
+    }]
+
+    def _call_api(self, video_id, rscuse=''):
+        return self._download_json(
+            f'https://api.play.sbs.co.kr/1.0/sbs_vodall/{video_id}', video_id,
+            note=f'Downloading m3u8 information {rscuse}',
+            query={
+                'platform': 'pcweb',
+                'protocol': 'download',
+                'absolute_show': 'Y',
+                'service': 'program',
+                'ssl': 'Y',
+                'rscuse': rscuse,
+            })
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        details = self._call_api(video_id)
+        source = traverse_obj(details, ('vod', 'source', 'mediasource', {dict})) or {}
+
+        formats = []
+        for stream in traverse_obj(details, (
+            'vod', 'source', 'mediasourcelist', lambda _, v: v['mediaurl'] or v['mediarscuse']
+        ), default=[source]):
+            if not stream.get('mediaurl'):
+                new_source = traverse_obj(
+                    self._call_api(video_id, rscuse=stream['mediarscuse']),
+                    ('vod', 'source', 'mediasource', {dict})) or {}
+                if new_source.get('mediarscuse') == source.get('mediarscuse') or not new_source.get('mediaurl'):
+                    continue
+                stream = new_source
+            formats.append({
+                'url': stream['mediaurl'],
+                'format_id': stream.get('mediarscuse'),
+                'format_note': stream.get('medianame'),
+                **parse_resolution(stream.get('quality')),
+                'preference': int_or_none(stream.get('mediarscuse'))
+            })
+
+        caption_url = traverse_obj(details, ('vod', 'source', 'subtitle', {url_or_none}))
+
+        return {
+            'id': video_id,
+            **traverse_obj(details, ('vod', {
+                'title': ('info', 'title'),
+                'duration': ('info', 'duration', {int_or_none}),
+                'view_count': ('info', 'viewcount', {int_or_none}),
+                'like_count': ('info', 'likecount', {int_or_none}),
+                'description': ('info', 'synopsis', {clean_html}),
+                'episode': ('info', 'content', ('contenttitle', 'title')),
+                'episode_number': ('info', 'content', 'number', {int_or_none}),
+                'series': ('info', 'program', 'programtitle'),
+                'age_limit': ('info', 'targetage', {int_or_none}),
+                'release_timestamp': ('info', 'broaddate', {parse_iso8601}),
+                'thumbnail': ('source', 'thumbnail', 'origin', {url_or_none}),
+            }), get_all=False),
+            'formats': formats,
+            'subtitles': {'ko': [{'url': caption_url}]} if caption_url else None,
+        }
+
+
+class SBSCoKrAllvodProgramIE(InfoExtractor):
+    IE_NAME = 'sbs.co.kr:allvod_program'
+    _VALID_URL = r'https?://allvod\.sbs\.co\.kr/allvod/vod(?:Free)?ProgramDetail\.do\?(?:[^#]+&)?pgmId=(?P<id>P?\d+)'
+
+    _TESTS = [{
+        'url': 'https://allvod.sbs.co.kr/allvod/vodFreeProgramDetail.do?type=legend&pgmId=22000010159&listOrder=vodCntAsc',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': '22000010159',
+        },
+        'playlist_count': 18,
+    }, {
+        'url': 'https://allvod.sbs.co.kr/allvod/vodProgramDetail.do?pgmId=P460810577',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'P460810577',
+        },
+        'playlist_count': 13,
+    }]
+
+    def _real_extract(self, url):
+        program_id = self._match_id(url)
+
+        details = self._download_json(
+            'https://allvod.sbs.co.kr/allvod/vodProgramDetail/vodProgramDetailAjax.do',
+            program_id, note='Downloading program details',
+            query={
+                'pgmId': program_id,
+                'currentCount': '10000',
+            })
+
+        return self.playlist_result(
+            [self.url_result(f'https://allvod.sbs.co.kr/allvod/vodEndPage.do?mdaId={video_id}', SBSCoKrIE)
+             for video_id in traverse_obj(details, ('list', ..., 'mdaId'))], program_id)
+
+
+class SBSCoKrProgramsVodIE(InfoExtractor):
+    IE_NAME = 'sbs.co.kr:programs_vod'
+    _VALID_URL = r'https?://programs\.sbs\.co\.kr/(?:enter|drama|culture|sports|plus|mtv)/(?P<id>[a-z0-9]+)/vods'
+
+    _TESTS = [{
+        'url': 'https://programs.sbs.co.kr/culture/morningwide/vods/65007',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': '00000210215',
+        },
+        'playlist_mincount': 9782,
+    }, {
+        'url': 'https://programs.sbs.co.kr/enter/dongsang2/vods/52006',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': '22000010476',
+        },
+        'playlist_mincount': 312,
+    }]
+
+    def _real_extract(self, url):
+        program_slug = self._match_id(url)
+
+        program_id = self._download_json(
+            f'https://static.apis.sbs.co.kr/program-api/1.0/menu/{program_slug}', program_slug,
+            note='Downloading program menu data')['program']['programid']
+
+        return self.url_result(
+            f'https://allvod.sbs.co.kr/allvod/vodProgramDetail.do?pgmId={program_id}', SBSCoKrAllvodProgramIE)

From cb480e390d85fb3a598c1b6d5eef3438ce729fc9 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 11 Nov 2023 13:53:59 -0600
Subject: [PATCH 653/871] [ie/thisav] Remove (#8346)

Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |  1 -
 yt_dlp/extractor/thisav.py      | 66 ---------------------------------
 yt_dlp/extractor/unsupported.py |  4 ++
 3 files changed, 4 insertions(+), 67 deletions(-)
 delete mode 100644 yt_dlp/extractor/thisav.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index cf6744487..b4d50e52b 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2019,7 +2019,6 @@
 from .thesun import TheSunIE
 from .theweatherchannel import TheWeatherChannelIE
 from .thisamericanlife import ThisAmericanLifeIE
-from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
 from .thisvid import (
     ThisVidIE,
diff --git a/yt_dlp/extractor/thisav.py b/yt_dlp/extractor/thisav.py
deleted file mode 100644
index b1cd57d1f..000000000
--- a/yt_dlp/extractor/thisav.py
+++ /dev/null
@@ -1,66 +0,0 @@
-from .common import InfoExtractor
-from ..utils import remove_end
-
-
-class ThisAVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?thisav\.com/video/(?P<id>[0-9]+)/.*'
-    _TESTS = [{
-        # jwplayer
-        'url': 'http://www.thisav.com/video/47734/%98%26sup1%3B%83%9E%83%82---just-fit.html',
-        'md5': '0480f1ef3932d901f0e0e719f188f19b',
-        'info_dict': {
-            'id': '47734',
-            'ext': 'flv',
-            'title': '高樹マリア - Just fit',
-            'uploader': 'dj7970',
-            'uploader_id': 'dj7970'
-        }
-    }, {
-        # html5 media
-        'url': 'http://www.thisav.com/video/242352/nerdy-18yo-big-ass-tattoos-and-glasses.html',
-        'md5': 'ba90c076bd0f80203679e5b60bf523ee',
-        'info_dict': {
-            'id': '242352',
-            'ext': 'mp4',
-            'title': 'Nerdy 18yo Big Ass Tattoos and Glasses',
-            'uploader': 'cybersluts',
-            'uploader_id': 'cybersluts',
-        },
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-        title = remove_end(self._html_extract_title(webpage), ' - 視頻 - ThisAV.com-世界第一中文成人娛樂網站')
-        video_url = self._html_search_regex(
-            r"addVariable\('file','([^']+)'\);", webpage, 'video url', default=None)
-        if video_url:
-            info_dict = {
-                'formats': [{
-                    'url': video_url,
-                }],
-            }
-        else:
-            entries = self._parse_html5_media_entries(url, webpage, video_id)
-            if entries:
-                info_dict = entries[0]
-            else:
-                info_dict = self._extract_jwplayer_data(
-                    webpage, video_id, require_title=False)
-        uploader = self._html_search_regex(
-            r': <a href="http://www\.thisav\.com/user/[0-9]+/(?:[^"]+)">([^<]+)</a>',
-            webpage, 'uploader name', fatal=False)
-        uploader_id = self._html_search_regex(
-            r': <a href="http://www\.thisav\.com/user/[0-9]+/([^"]+)">(?:[^<]+)</a>',
-            webpage, 'uploader id', fatal=False)
-
-        info_dict.update({
-            'id': video_id,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'title': title,
-        })
-
-        return info_dict
diff --git a/yt_dlp/extractor/unsupported.py b/yt_dlp/extractor/unsupported.py
index bbcbf3acb..d610baecb 100644
--- a/yt_dlp/extractor/unsupported.py
+++ b/yt_dlp/extractor/unsupported.py
@@ -164,11 +164,15 @@ class KnownPiracyIE(UnsupportedInfoExtractor):
         r'viewsb\.com',
         r'filemoon\.sx',
         r'hentai\.animestigma\.com',
+        r'thisav\.com',
     )
 
     _TESTS = [{
         'url': 'http://dood.to/e/5s1wmbdacezb',
         'only_matching': True,
+    }, {
+        'url': 'https://thisav.com/en/terms',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9b5bedf13a3323074daceb0ec6ebb3cc6e0b9684 Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Sun, 12 Nov 2023 03:54:53 +0800
Subject: [PATCH 654/871] [ie/brilliantpala] Fix cookies support (#8352)

Authored by: pzhlkj6612
---
 yt_dlp/extractor/brilliantpala.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/brilliantpala.py b/yt_dlp/extractor/brilliantpala.py
index 6fd5b8148..0bf8622c1 100644
--- a/yt_dlp/extractor/brilliantpala.py
+++ b/yt_dlp/extractor/brilliantpala.py
@@ -21,10 +21,10 @@ def _initialize_pre_login(self):
 
     def _get_logged_in_username(self, url, video_id):
         webpage, urlh = self._download_webpage_handle(url, video_id)
-        if self._LOGIN_API == urlh.url:
+        if urlh.url.startswith(self._LOGIN_API):
             self.raise_login_required()
         return self._html_search_regex(
-            r'"username"\s*:\s*"(?P<username>[^"]+)"', webpage, 'stream page info', 'username')
+            r'"username"\s*:\s*"(?P<username>[^"]+)"', webpage, 'logged-in username')
 
     def _perform_login(self, username, password):
         login_form = self._hidden_inputs(self._download_webpage(

From 3ff494f6f41c27549420fa88be27555bd449ffdc Mon Sep 17 00:00:00 2001
From: Martin Pecka <peci1@seznam.cz>
Date: Sat, 11 Nov 2023 20:56:29 +0100
Subject: [PATCH 655/871] [ie/NovaEmbed] Improve `_VALID_URL` (#8368)

Authored by: peci1
---
 yt_dlp/extractor/nova.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/nova.py b/yt_dlp/extractor/nova.py
index bd0c4ebe3..8a7dfceeb 100644
--- a/yt_dlp/extractor/nova.py
+++ b/yt_dlp/extractor/nova.py
@@ -13,7 +13,7 @@
 
 
 class NovaEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://media\.cms\.nova\.cz/embed/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://media(?:tn)?\.cms\.nova\.cz/embed/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://media.cms.nova.cz/embed/8o0n0r?autoplay=1',
         'info_dict': {
@@ -37,6 +37,16 @@ class NovaEmbedIE(InfoExtractor):
             'duration': 114,
         },
         'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://mediatn.cms.nova.cz/embed/EU5ELEsmOHt?autoplay=1',
+        'info_dict': {
+            'id': 'EU5ELEsmOHt',
+            'ext': 'mp4',
+            'title': 'Haptické křeslo, bionická ruka nebo roboti. Reportérka se podívala na Týden inovací',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 1780,
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):

From 05adfd883a4f2ecae0267e670a62a2e45c351aeb Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sat, 11 Nov 2023 20:57:56 +0100
Subject: [PATCH 656/871] [ie/ondemandkorea] Overhaul extractor (#8386)

Closes #8374
Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py   |   5 +-
 yt_dlp/extractor/ondemandkorea.py | 196 +++++++++++++++++++++---------
 2 files changed, 142 insertions(+), 59 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b4d50e52b..731450725 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1387,7 +1387,10 @@
 from .oktoberfesttv import OktoberfestTVIE
 from .olympics import OlympicsReplayIE
 from .on24 import On24IE
-from .ondemandkorea import OnDemandKoreaIE
+from .ondemandkorea import (
+    OnDemandKoreaIE,
+    OnDemandKoreaProgramIE,
+)
 from .onefootball import OneFootballIE
 from .onenewsnz import OneNewsNZIE
 from .oneplace import OnePlacePodcastIE
diff --git a/yt_dlp/extractor/ondemandkorea.py b/yt_dlp/extractor/ondemandkorea.py
index dd7d1d7de..81ce99fd9 100644
--- a/yt_dlp/extractor/ondemandkorea.py
+++ b/yt_dlp/extractor/ondemandkorea.py
@@ -1,87 +1,167 @@
+import functools
 import re
+import uuid
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    js_to_json,
+    OnDemandPagedList,
+    float_or_none,
+    int_or_none,
+    join_nonempty,
+    parse_age_limit,
+    parse_qs,
+    unified_strdate,
+    url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class OnDemandKoreaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ondemandkorea\.com/(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?ondemandkorea\.com/(?:en/)?player/vod/[a-z0-9-]+\?(?:[^#]+&)?contentId=(?P<id>\d+)'
     _GEO_COUNTRIES = ['US', 'CA']
+
     _TESTS = [{
-        'url': 'https://www.ondemandkorea.com/ask-us-anything-e351.html',
+        'url': 'https://www.ondemandkorea.com/player/vod/ask-us-anything?contentId=686471',
+        'md5': 'e2ff77255d989e3135bde0c5889fbce8',
         'info_dict': {
-            'id': 'ask-us-anything-e351',
+            'id': '686471',
             'ext': 'mp4',
-            'title': 'Ask Us Anything : Jung Sung-ho, Park Seul-gi, Kim Bo-min, Yang Seung-won - 09/24/2022',
-            'description': 'A talk show/game show with a school theme where celebrity guests appear as “transfer students.”',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'title': 'Ask Us Anything: Jung Sung-ho, Park Seul-gi, Kim Bo-min, Yang Seung-won',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)',
+            'duration': 5486.955,
+            'release_date': '20220924',
+            'series': 'Ask Us Anything',
+            'series_id': 11790,
+            'episode_number': 351,
+            'episode': 'Jung Sung-ho, Park Seul-gi, Kim Bo-min, Yang Seung-won',
         },
-        'params': {
-            'skip_download': 'm3u8 download'
-        }
     }, {
-        'url': 'https://www.ondemandkorea.com/work-later-drink-now-e1.html',
+        'url': 'https://www.ondemandkorea.com/player/vod/breakup-probation-a-week?contentId=1595796',
+        'md5': '57266c720006962be7ff415b24775caa',
         'info_dict': {
-            'id': 'work-later-drink-now-e1',
+            'id': '1595796',
             'ext': 'mp4',
-            'title': 'Work Later, Drink Now : E01',
-            'description': 'Work Later, Drink First follows three women who find solace in a glass of liquor at the end of the day. So-hee, who gets comfort from a cup of soju af',
-            'thumbnail': r're:^https?://.*\.png$',
-            'subtitles': {
-                'English': 'mincount:1',
-            },
+            'title': 'Breakup Probation, A Week: E08',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)',
+            'duration': 1586.0,
+            'release_date': '20231001',
+            'series': 'Breakup Probation, A Week',
+            'series_id': 22912,
+            'episode_number': 8,
+            'episode': 'E08',
         },
-        'params': {
-            'skip_download': 'm3u8 download'
-        }
+    }, {
+        'url': 'https://www.ondemandkorea.com/player/vod/the-outlaws?contentId=369531',
+        'md5': 'fa5523b87aa1f6d74fc622a97f2b47cd',
+        'info_dict': {
+            'id': '369531',
+            'ext': 'mp4',
+            'release_date': '20220519',
+            'duration': 7267.0,
+            'title': 'The Outlaws: Main Movie',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)',
+            'age_limit': 18,
+        },
+    }, {
+        'url': 'https://www.ondemandkorea.com/en/player/vod/capture-the-moment-how-is-that-possible?contentId=1605006',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id, fatal=False)
 
-        if not webpage:
-            # Page sometimes returns captcha page with HTTP 403
-            raise ExtractorError(
-                'Unable to access page. You may have been blocked.',
-                expected=True)
+        data = self._download_json(
+            f'https://odkmedia.io/odx/api/v3/playback/{video_id}/', video_id, fatal=False,
+            headers={'service-name': 'odk'}, query={'did': str(uuid.uuid4())}, expected_status=(403, 404))
+        if not traverse_obj(data, ('result', {dict})):
+            msg = traverse_obj(data, ('messages', '__default'), 'title', expected_type=str)
+            raise ExtractorError(msg or 'Got empty response from playback API', expected=True)
 
-        if 'msg_block_01.png' in webpage:
-            self.raise_geo_restricted(
-                msg='This content is not available in your region',
-                countries=self._GEO_COUNTRIES)
+        data = data['result']
 
-        if 'This video is only available to ODK PLUS members.' in webpage:
-            raise ExtractorError(
-                'This video is only available to ODK PLUS members.',
-                expected=True)
+        def try_geo_bypass(url):
+            return traverse_obj(url, ({parse_qs}, 'stream_url', 0, {url_or_none})) or url
 
-        if 'ODK PREMIUM Members Only' in webpage:
-            raise ExtractorError(
-                'This video is only available to ODK PREMIUM members.',
-                expected=True)
+        def try_upgrade_quality(url):
+            mod_url = re.sub(r'_720(p?)\.m3u8', r'_1080\1.m3u8', url)
+            return mod_url if mod_url != url and self._request_webpage(
+                HEADRequest(mod_url), video_id, note='Checking for higher quality format',
+                errnote='No higher quality format found', fatal=False) else url
 
-        title = self._search_regex(
-            r'class=["\']episode_title["\'][^>]*>([^<]+)',
-            webpage, 'episode_title', fatal=False) or self._og_search_title(webpage)
+        formats = []
+        for m3u8_url in traverse_obj(data, (('sources', 'manifest'), ..., 'url', {url_or_none}, {try_geo_bypass})):
+            formats.extend(self._extract_m3u8_formats(try_upgrade_quality(m3u8_url), video_id, fatal=False))
 
-        jw_config = self._parse_json(
-            self._search_regex((
-                r'(?P<options>{\s*[\'"]tracks[\'"].*?})[)\];]+$',
-                r'playlist\s*=\s*\[(?P<options>.+)];?$',
-                r'odkPlayer\.init.*?(?P<options>{[^;]+}).*?;',
-            ), webpage, 'jw config', flags=re.MULTILINE | re.DOTALL, group='options'),
-            video_id, transform_source=js_to_json)
-        info = self._parse_jwplayer_data(
-            jw_config, video_id, require_title=False, m3u8_id='hls',
-            base_url=url)
+        subtitles = {}
+        for track in traverse_obj(data, ('text_tracks', lambda _, v: url_or_none(v['url']))):
+            subtitles.setdefault(track.get('language', 'und'), []).append({
+                'url': track['url'],
+                'ext': track.get('codec'),
+                'name': track.get('label'),
+            })
 
-        info.update({
-            'title': title,
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage)
-        })
-        return info
+        def if_series(key=None):
+            return lambda obj: obj[key] if key and obj['kind'] == 'series' else None
+
+        return {
+            'id': video_id,
+            'title': join_nonempty(
+                ('episode', 'program', 'title'),
+                ('episode', 'title'), from_dict=data, delim=': '),
+            **traverse_obj(data, {
+                'thumbnail': ('episode', 'images', 'thumbnail', {url_or_none}),
+                'release_date': ('episode', 'release_date', {lambda x: x.replace('-', '')}, {unified_strdate}),
+                'duration': ('duration', {functools.partial(float_or_none, scale=1000)}),
+                'age_limit': ('age_rating', 'name', {lambda x: x.replace('R', '')}, {parse_age_limit}),
+                'series': ('episode', {if_series(key='program')}, 'title'),
+                'series_id': ('episode', {if_series(key='program')}, 'id'),
+                'episode': ('episode', {if_series(key='title')}),
+                'episode_number': ('episode', {if_series(key='number')}, {int_or_none}),
+            }, get_all=False),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+
+class OnDemandKoreaProgramIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ondemandkorea\.com/(?:en/)?player/vod/(?P<id>[a-z0-9-]+)(?:$|#)'
+    _GEO_COUNTRIES = ['US', 'CA']
+
+    _TESTS = [{
+        'url': 'https://www.ondemandkorea.com/player/vod/uskn-news',
+        'info_dict': {
+            'id': 'uskn-news',
+        },
+        'playlist_mincount': 755,
+    }, {
+        'url': 'https://www.ondemandkorea.com/en/player/vod/the-land',
+        'info_dict': {
+            'id': 'the-land',
+        },
+        'playlist_count': 52,
+    }]
+
+    _PAGE_SIZE = 100
+
+    def _fetch_page(self, display_id, page):
+        page += 1
+        page_data = self._download_json(
+            f'https://odkmedia.io/odx/api/v3/program/{display_id}/episodes/', display_id,
+            headers={'service-name': 'odk'}, query={
+                'page': page,
+                'page_size': self._PAGE_SIZE,
+            }, note=f'Downloading page {page}', expected_status=404)
+        for episode in traverse_obj(page_data, ('result', 'results', ...)):
+            yield self.url_result(
+                f'https://www.ondemandkorea.com/player/vod/{display_id}?contentId={episode["id"]}',
+                ie=OnDemandKoreaIE, video_title=episode.get('title'))
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        entries = OnDemandPagedList(functools.partial(
+            self._fetch_page, display_id), self._PAGE_SIZE)
+
+        return self.playlist_result(entries, display_id)

From 54579be4364e148277c32e20a5c3efc2c3f52f5b Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Sat, 11 Nov 2023 19:59:01 +0000
Subject: [PATCH 657/871] [ie/nhk] Improve metadata extraction (#8388)

Authored by: garret1317
---
 yt_dlp/extractor/nhk.py | 102 ++++++++++++++++++++++++++++++++--------
 1 file changed, 82 insertions(+), 20 deletions(-)

diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py
index f6b5c501b..cc3c79174 100644
--- a/yt_dlp/extractor/nhk.py
+++ b/yt_dlp/extractor/nhk.py
@@ -3,6 +3,8 @@
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    clean_html,
+    get_element_by_class,
     int_or_none,
     join_nonempty,
     parse_duration,
@@ -45,25 +47,36 @@ def _get_api_info(self, refresh=True):
         self.cache.store('nhk', 'api_info', api_info)
         return api_info
 
-    def _extract_formats_and_subtitles(self, vod_id):
+    def _extract_stream_info(self, vod_id):
         for refresh in (False, True):
             api_info = self._get_api_info(refresh)
             if not api_info:
                 continue
 
             api_url = api_info.pop('url')
-            stream_url = traverse_obj(
+            meta = traverse_obj(
                 self._download_json(
                     api_url, vod_id, 'Downloading stream url info', fatal=False, query={
                         **api_info,
                         'type': 'json',
                         'optional_id': vod_id,
                         'active_flg': 1,
-                    }),
-                ('meta', 0, 'movie_url', ('mb_auto', 'auto_sp', 'auto_pc'), {url_or_none}), get_all=False)
-            if stream_url:
-                return self._extract_m3u8_formats_and_subtitles(stream_url, vod_id)
+                    }), ('meta', 0))
+            stream_url = traverse_obj(
+                meta, ('movie_url', ('mb_auto', 'auto_sp', 'auto_pc'), {url_or_none}), get_all=False)
 
+            if stream_url:
+                formats, subtitles = self._extract_m3u8_formats_and_subtitles(stream_url, vod_id)
+                return {
+                    **traverse_obj(meta, {
+                        'duration': ('duration', {int_or_none}),
+                        'timestamp': ('publication_date', {unified_timestamp}),
+                        'release_timestamp': ('insert_date', {unified_timestamp}),
+                        'modified_timestamp': ('update_date', {unified_timestamp}),
+                    }),
+                    'formats': formats,
+                    'subtitles': subtitles,
+                }
         raise ExtractorError('Unable to extract stream url')
 
     def _extract_episode_info(self, url, episode=None):
@@ -77,11 +90,11 @@ def _extract_episode_info(self, url, episode=None):
         if fetch_episode:
             episode = self._call_api(
                 episode_id, lang, is_video, True, episode_id[:4] == '9999')[0]
-        title = episode.get('sub_title_clean') or episode['sub_title']
 
         def get_clean_field(key):
-            return episode.get(key + '_clean') or episode.get(key)
+            return clean_html(episode.get(key + '_clean') or episode.get(key))
 
+        title = get_clean_field('sub_title')
         series = get_clean_field('title')
 
         thumbnails = []
@@ -96,22 +109,30 @@ def get_clean_field(key):
                 'url': 'https://www3.nhk.or.jp' + img_path,
             })
 
+        episode_name = title
+        if series and title:
+            title = f'{series} - {title}'
+        elif series and not title:
+            title = series
+            series = None
+            episode_name = None
+        else:  # title, no series
+            episode_name = None
+
         info = {
             'id': episode_id + '-' + lang,
-            'title': '%s - %s' % (series, title) if series and title else title,
+            'title': title,
             'description': get_clean_field('description'),
             'thumbnails': thumbnails,
             'series': series,
-            'episode': title,
+            'episode': episode_name,
         }
+
         if is_video:
             vod_id = episode['vod_id']
-            formats, subs = self._extract_formats_and_subtitles(vod_id)
-
             info.update({
+                **self._extract_stream_info(vod_id),
                 'id': vod_id,
-                'formats': formats,
-                'subtitles': subs,
             })
 
         else:
@@ -148,6 +169,14 @@ class NhkVodIE(NhkBaseIE):
             'thumbnail': 'md5:51bcef4a21936e7fea1ff4e06353f463',
             'episode': 'The Tohoku Shinkansen: Full Speed Ahead',
             'series': 'Japan Railway Journal',
+            'modified_timestamp': 1694243656,
+            'timestamp': 1681428600,
+            'release_timestamp': 1693883728,
+            'duration': 1679,
+            'upload_date': '20230413',
+            'modified_date': '20230909',
+            'release_date': '20230905',
+
         },
     }, {
         # video clip
@@ -161,6 +190,13 @@ class NhkVodIE(NhkBaseIE):
             'thumbnail': 'md5:d6a4d9b6e9be90aaadda0bcce89631ed',
             'series': 'Dining with the Chef',
             'episode': 'Chef Saito\'s Family recipe: MENCHI-KATSU',
+            'duration': 148,
+            'upload_date': '20190816',
+            'release_date': '20230902',
+            'release_timestamp': 1693619292,
+            'modified_timestamp': 1694168033,
+            'modified_date': '20230908',
+            'timestamp': 1565997540,
         },
     }, {
         # radio
@@ -170,7 +206,7 @@ class NhkVodIE(NhkBaseIE):
             'ext': 'm4a',
             'title': 'Living in Japan - Tips for Travelers to Japan / Ramen Vending Machines',
             'series': 'Living in Japan',
-            'description': 'md5:850611969932874b4a3309e0cae06c2f',
+            'description': 'md5:0a0e2077d8f07a03071e990a6f51bfab',
             'thumbnail': 'md5:960622fb6e06054a4a1a0c97ea752545',
             'episode': 'Tips for Travelers to Japan / Ramen Vending Machines'
         },
@@ -212,6 +248,23 @@ class NhkVodIE(NhkBaseIE):
             'description': 'md5:9c1d6cbeadb827b955b20e99ab920ff0',
         },
         'skip': 'expires 2023-10-15',
+    }, {
+        # a one-off (single-episode series). title from the api is just '<p></p>'
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/3004952/',
+        'info_dict': {
+            'id': 'nw_vod_v_en_3004_952_20230723091000_01_1690074552',
+            'ext': 'mp4',
+            'title': 'Barakan Discovers AMAMI OSHIMA: Isson\'s Treasure Island',
+            'description': 'md5:5db620c46a0698451cc59add8816b797',
+            'thumbnail': 'md5:67d9ff28009ba379bfa85ad1aaa0e2bd',
+            'release_date': '20230905',
+            'timestamp': 1690103400,
+            'duration': 2939,
+            'release_timestamp': 1693898699,
+            'modified_timestamp': 1698057495,
+            'modified_date': '20231023',
+            'upload_date': '20230723',
+        },
     }]
 
     def _real_extract(self, url):
@@ -226,13 +279,15 @@ class NhkVodProgramIE(NhkBaseIE):
         'info_dict': {
             'id': 'sumo',
             'title': 'GRAND SUMO Highlights',
+            'description': 'md5:fc20d02dc6ce85e4b72e0273aa52fdbf',
         },
-        'playlist_mincount': 12,
+        'playlist_mincount': 0,
     }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway',
         'info_dict': {
             'id': 'japanrailway',
             'title': 'Japan Railway Journal',
+            'description': 'md5:ea39d93af7d05835baadf10d1aae0e3f',
         },
         'playlist_mincount': 12,
     }, {
@@ -241,6 +296,7 @@ class NhkVodProgramIE(NhkBaseIE):
         'info_dict': {
             'id': 'japanrailway',
             'title': 'Japan Railway Journal',
+            'description': 'md5:ea39d93af7d05835baadf10d1aae0e3f',
         },
         'playlist_mincount': 5,
     }, {
@@ -265,11 +321,11 @@ def _real_extract(self, url):
             entries.append(self._extract_episode_info(
                 urljoin(url, episode_path), episode))
 
-        program_title = None
-        if entries:
-            program_title = entries[0].get('series')
+        html = self._download_webpage(url, program_id)
+        program_title = clean_html(get_element_by_class('p-programDetail__title', html))
+        program_description = clean_html(get_element_by_class('p-programDetail__text', html))
 
-        return self.playlist_result(entries, program_id, program_title)
+        return self.playlist_result(entries, program_id, program_title, program_description)
 
 
 class NhkForSchoolBangumiIE(InfoExtractor):
@@ -421,6 +477,7 @@ class NhkRadiruIE(InfoExtractor):
         'skip': 'Episode expired on 2023-04-16',
         'info_dict': {
             'channel': 'NHK-FM',
+            'uploader': 'NHK-FM',
             'description': 'md5:94b08bdeadde81a97df4ec882acce3e9',
             'ext': 'm4a',
             'id': '0449_01_3853544',
@@ -441,6 +498,7 @@ class NhkRadiruIE(InfoExtractor):
             'title': 'ベストオブクラシック',
             'description': '世界中の上質な演奏会をじっくり堪能する本格派クラシック番組。',
             'channel': 'NHK-FM',
+            'uploader': 'NHK-FM',
             'thumbnail': 'https://www.nhk.or.jp/prog/img/458/g458.jpg',
         },
         'playlist_mincount': 3,
@@ -454,6 +512,7 @@ class NhkRadiruIE(InfoExtractor):
             'title': '有島武郎「一房のぶどう」',
             'description': '朗読：川野一宇（ラジオ深夜便アンカー）\r\n\r\n（2016年12月8日放送「ラジオ深夜便『アンカー朗読シリーズ』」より）',
             'channel': 'NHKラジオ第1、NHK-FM',
+            'uploader': 'NHKラジオ第1、NHK-FM',
             'timestamp': 1635757200,
             'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F300/img/corner/box_109_thumbnail.jpg',
             'release_date': '20161207',
@@ -469,6 +528,7 @@ class NhkRadiruIE(InfoExtractor):
             'id': 'F261_01_3855109',
             'ext': 'm4a',
             'channel': 'NHKラジオ第1',
+            'uploader': 'NHKラジオ第1',
             'timestamp': 1681635900,
             'release_date': '20230416',
             'series': 'NHKラジオニュース',
@@ -513,6 +573,7 @@ def _real_extract(self, url):
         series_meta = traverse_obj(meta, {
             'title': 'program_name',
             'channel': 'media_name',
+            'uploader': 'media_name',
             'thumbnail': (('thumbnail_c', 'thumbnail_p'), {url_or_none}),
         }, get_all=False)
 
@@ -541,6 +602,7 @@ class NhkRadioNewsPageIE(InfoExtractor):
             'thumbnail': 'https://www.nhk.or.jp/radioondemand/json/F261/img/RADIONEWS_640.jpg',
             'description': 'md5:bf2c5b397e44bc7eb26de98d8f15d79d',
             'channel': 'NHKラジオ第1',
+            'uploader': 'NHKラジオ第1',
             'title': 'NHKラジオニュース',
         }
     }]

From 312a2d1e8bc247264f9d85c5ec764e33aa0133b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aniol=20Pag=C3=A8s?=
 <47257568+aniolpages@users.noreply.github.com>
Date: Sat, 11 Nov 2023 21:00:31 +0100
Subject: [PATCH 658/871] [ie/LaXarxaMes] Add extractor (#8412)

Authored by: aniolpages
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/laxarxames.py  | 73 +++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 yt_dlp/extractor/laxarxames.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 731450725..212c6ffb0 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -953,6 +953,7 @@
     LastFMPlaylistIE,
     LastFMUserIE,
 )
+from .laxarxames import LaXarxaMesIE
 from .lbry import (
     LBRYIE,
     LBRYChannelIE,
diff --git a/yt_dlp/extractor/laxarxames.py b/yt_dlp/extractor/laxarxames.py
new file mode 100644
index 000000000..e157f7c08
--- /dev/null
+++ b/yt_dlp/extractor/laxarxames.py
@@ -0,0 +1,73 @@
+import json
+
+from .brightcove import BrightcoveNewIE
+from .common import InfoExtractor
+from ..utils import ExtractorError
+from ..utils.traversal import traverse_obj
+
+
+class LaXarxaMesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?laxarxames\.cat/(?:[^/?#]+/)*?(player|movie-details)/(?P<id>\d+)'
+    _NETRC_MACHINE = 'laxarxames'
+    _TOKEN = None
+    _TESTS = [{
+        'url': 'https://www.laxarxames.cat/player/3459421',
+        'md5': '0966f46c34275934c19af78f3df6e2bc',
+        'info_dict': {
+            'id': '6339612436112',
+            'ext': 'mp4',
+            'title': 'Resum | UA Horta — UD Viladecans',
+            'timestamp': 1697905186,
+            'thumbnail': r're:https?://.*\.jpg',
+            'description': '',
+            'upload_date': '20231021',
+            'duration': 129.44,
+            'tags': ['ott', 'esports', '23-24', ' futbol', ' futbol-partits', 'elit', 'resum'],
+            'uploader_id': '5779379807001',
+        },
+        'skip': 'Requires login',
+    }]
+
+    def _perform_login(self, username, password):
+        if self._TOKEN:
+            return
+
+        login = self._download_json(
+            'https://api.laxarxames.cat/Authorization/SignIn', None, note='Logging in', headers={
+                'X-Tenantorigin': 'https://laxarxames.cat',
+                'Content-Type': 'application/json',
+            }, data=json.dumps({
+                'Username': username,
+                'Password': password,
+                'Device': {
+                    'PlatformCode': 'WEB',
+                    'Name': 'Mac OS ()',
+                },
+            }).encode(), expected_status=401)
+
+        self._TOKEN = traverse_obj(login, ('AuthorizationToken', 'Token', {str}))
+        if not self._TOKEN:
+            raise ExtractorError('Login failed', expected=True)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        if not self._TOKEN:
+            self.raise_login_required()
+
+        media_play_info = self._download_json(
+            'https://api.laxarxames.cat/Media/GetMediaPlayInfo', video_id,
+            data=json.dumps({
+                'MediaId': int(video_id),
+                'StreamType': 'MAIN'
+            }).encode(), headers={
+                'Authorization': f'Bearer {self._TOKEN}',
+                'X-Tenantorigin': 'https://laxarxames.cat',
+                'Content-Type': 'application/json',
+            })
+
+        if not traverse_obj(media_play_info, ('ContentUrl', {str})):
+            self.raise_no_formats('No video found', expected=True)
+
+        return self.url_result(
+            f'https://players.brightcove.net/5779379807001/default_default/index.html?videoId={media_play_info["ContentUrl"]}',
+            BrightcoveNewIE, video_id, media_play_info.get('Title'))

From 15b252dfd2c6807fe57afc5a95e59abadb32ccd2 Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Sat, 11 Nov 2023 15:02:59 -0500
Subject: [PATCH 659/871] [ie/weibo] Fix extraction (#8463)

Closes #8445
Authored by: c-basalt
---
 yt_dlp/extractor/weibo.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/yt_dlp/extractor/weibo.py b/yt_dlp/extractor/weibo.py
index b0c3052b6..2fca745aa 100644
--- a/yt_dlp/extractor/weibo.py
+++ b/yt_dlp/extractor/weibo.py
@@ -1,3 +1,4 @@
+import json
 import random
 import itertools
 import urllib.parse
@@ -18,24 +19,33 @@
 
 
 class WeiboBaseIE(InfoExtractor):
-    def _update_visitor_cookies(self, video_id):
+    def _update_visitor_cookies(self, visitor_url, video_id):
+        headers = {'Referer': visitor_url}
+        chrome_ver = self._search_regex(
+            r'Chrome/(\d+)', self.get_param('http_headers')['User-Agent'], 'user agent version', default='90')
         visitor_data = self._download_json(
             'https://passport.weibo.com/visitor/genvisitor', video_id,
             note='Generating first-visit guest request',
-            transform_source=strip_jsonp,
+            headers=headers, transform_source=strip_jsonp,
             data=urlencode_postdata({
                 'cb': 'gen_callback',
-                'fp': '{"os":"2","browser":"Gecko57,0,0,0","fonts":"undefined","screenInfo":"1440*900*24","plugins":""}',
-            }))
+                'fp': json.dumps({
+                    'os': '1',
+                    'browser': f'Chrome{chrome_ver},0,0,0',
+                    'fonts': 'undefined',
+                    'screenInfo': '1920*1080*24',
+                    'plugins': ''
+                }, separators=(',', ':'))}))['data']
 
         self._download_webpage(
             'https://passport.weibo.com/visitor/visitor', video_id,
             note='Running first-visit callback to get guest cookies',
-            query={
+            headers=headers, query={
                 'a': 'incarnate',
-                't': visitor_data['data']['tid'],
-                'w': 2,
-                'c': '%03d' % visitor_data['data']['confidence'],
+                't': visitor_data['tid'],
+                'w': 3 if visitor_data.get('new_tid') else 2,
+                'c': f'{visitor_data.get("confidence", 100):03d}',
+                'gc': '',
                 'cb': 'cross_domain',
                 'from': 'weibo',
                 '_rand': random.random(),
@@ -44,7 +54,7 @@ def _update_visitor_cookies(self, video_id):
     def _weibo_download_json(self, url, video_id, *args, fatal=True, note='Downloading JSON metadata', **kwargs):
         webpage, urlh = self._download_webpage_handle(url, video_id, *args, fatal=fatal, note=note, **kwargs)
         if urllib.parse.urlparse(urlh.url).netloc == 'passport.weibo.com':
-            self._update_visitor_cookies(video_id)
+            self._update_visitor_cookies(urlh.url, video_id)
             webpage = self._download_webpage(url, video_id, *args, fatal=fatal, note=note, **kwargs)
         return self._parse_json(webpage, video_id, fatal=fatal)
 

From c76c96677ff6a056f5844a568ef05ee22c46d6f4 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 11 Nov 2023 14:03:50 -0600
Subject: [PATCH 660/871] [ie/thisoldhouse] Add login support (#8561)

Closes #8257
Authored by: bashonly
---
 yt_dlp/extractor/thisoldhouse.py | 71 +++++++++++++++++++++++++++-----
 1 file changed, 60 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/extractor/thisoldhouse.py b/yt_dlp/extractor/thisoldhouse.py
index cc7beeea5..15f8380d3 100644
--- a/yt_dlp/extractor/thisoldhouse.py
+++ b/yt_dlp/extractor/thisoldhouse.py
@@ -1,11 +1,23 @@
+import json
+
 from .common import InfoExtractor
+from .zype import ZypeIE
 from ..networking import HEADRequest
+from ..networking.exceptions import HTTPError
+from ..utils import (
+    ExtractorError,
+    filter_dict,
+    parse_qs,
+    try_call,
+    urlencode_postdata,
+)
 
 
 class ThisOldHouseIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?thisoldhouse\.com/(?:watch|how-to|tv-episode|(?:[^/]+/)?\d+)/(?P<id>[^/?#]+)'
+    _NETRC_MACHINE = 'thisoldhouse'
+    _VALID_URL = r'https?://(?:www\.)?thisoldhouse\.com/(?:watch|how-to|tv-episode|(?:[^/?#]+/)?\d+)/(?P<id>[^/?#]+)'
     _TESTS = [{
-        'url': 'https://www.thisoldhouse.com/how-to/how-to-build-storage-bench',
+        'url': 'https://www.thisoldhouse.com/furniture/21017078/how-to-build-a-storage-bench',
         'info_dict': {
             'id': '5dcdddf673c3f956ef5db202',
             'ext': 'mp4',
@@ -23,13 +35,16 @@ class ThisOldHouseIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
+        # Page no longer has video
         'url': 'https://www.thisoldhouse.com/watch/arlington-arts-crafts-arts-and-crafts-class-begins',
         'only_matching': True,
     }, {
+        # 404 Not Found
         'url': 'https://www.thisoldhouse.com/tv-episode/ask-toh-shelf-rough-electric',
         'only_matching': True,
     }, {
-        'url': 'https://www.thisoldhouse.com/furniture/21017078/how-to-build-a-storage-bench',
+        # 404 Not Found
+        'url': 'https://www.thisoldhouse.com/how-to/how-to-build-storage-bench',
         'only_matching': True,
     }, {
         'url': 'https://www.thisoldhouse.com/21113884/s41-e13-paradise-lost',
@@ -39,17 +54,51 @@ class ThisOldHouseIE(InfoExtractor):
         'url': 'https://www.thisoldhouse.com/21083431/seaside-transformation-the-westerly-project',
         'only_matching': True,
     }]
-    _ZYPE_TMPL = 'https://player.zype.com/embed/%s.html?api_key=hsOk_yMSPYNrT22e9pu8hihLXjaZf0JW5jsOWv4ZqyHJFvkJn6rtToHl09tbbsbe'
+
+    _LOGIN_URL = 'https://login.thisoldhouse.com/usernamepassword/login'
+
+    def _perform_login(self, username, password):
+        self._request_webpage(
+            HEADRequest('https://www.thisoldhouse.com/insider'), None, 'Requesting session cookies')
+        urlh = self._request_webpage(
+            'https://www.thisoldhouse.com/wp-login.php', None, 'Requesting login info',
+            errnote='Unable to login', query={'redirect_to': 'https://www.thisoldhouse.com/insider'})
+
+        try:
+            auth_form = self._download_webpage(
+                self._LOGIN_URL, None, 'Submitting credentials', headers={
+                    'Content-Type': 'application/json',
+                    'Referer': urlh.url,
+                }, data=json.dumps(filter_dict({
+                    **{('client_id' if k == 'client' else k): v[0] for k, v in parse_qs(urlh.url).items()},
+                    'tenant': 'thisoldhouse',
+                    'username': username,
+                    'password': password,
+                    'popup_options': {},
+                    'sso': True,
+                    '_csrf': try_call(lambda: self._get_cookies(self._LOGIN_URL)['_csrf'].value),
+                    '_intstate': 'deprecated',
+                }), separators=(',', ':')).encode())
+        except ExtractorError as e:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                raise ExtractorError('Invalid username or password', expected=True)
+            raise
+
+        self._request_webpage(
+            'https://login.thisoldhouse.com/login/callback', None, 'Completing login',
+            data=urlencode_postdata(self._hidden_inputs(auth_form)))
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         if 'To Unlock This content' in webpage:
-            self.raise_login_required(method='cookies')
-        video_url = self._search_regex(
+            self.raise_login_required(
+                'This video is only available for subscribers. '
+                'Note that --cookies-from-browser may not work due to this site using session cookies')
+
+        video_url, video_id = self._search_regex(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?thisoldhouse\.(?:chorus\.build|com)/videos/zype/([0-9a-f]{24})[^\'"]*)[\'"]',
-            webpage, 'video url')
-        if 'subscription_required=true' in video_url or 'c-entry-group-labels__image' in webpage:
-            return self.url_result(self._request_webpage(HEADRequest(video_url), display_id).url, 'Zype', display_id)
-        video_id = self._search_regex(r'(?:https?:)?//(?:www\.)?thisoldhouse\.(?:chorus\.build|com)/videos/zype/([0-9a-f]{24})', video_url, 'video id')
-        return self.url_result(self._ZYPE_TMPL % video_id, 'Zype', video_id)
+            webpage, 'video url', group=(1, 2))
+        video_url = self._request_webpage(HEADRequest(video_url), video_id, 'Resolving Zype URL').url
+
+        return self.url_result(video_url, ZypeIE, video_id)

From 2863fcf2b6876d0c7965ff7d6d9242eea653dc6b Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 11 Nov 2023 14:04:29 -0600
Subject: [PATCH 661/871] [ie/theatercomplextown] Add extractors (#8560)

Closes #8491
Authored by: bashonly
---
 yt_dlp/extractor/_extractors.py |   2 +
 yt_dlp/extractor/stacommu.py    | 155 ++++++++++++++++++++++++--------
 2 files changed, 121 insertions(+), 36 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 212c6ffb0..c4bf2acdf 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1911,6 +1911,8 @@
 from .stacommu import (
     StacommuLiveIE,
     StacommuVODIE,
+    TheaterComplexTownVODIE,
+    TheaterComplexTownPPVIE,
 )
 from .stanfordoc import StanfordOpenClassroomIE
 from .startv import StarTVIE
diff --git a/yt_dlp/extractor/stacommu.py b/yt_dlp/extractor/stacommu.py
index 6f58f06dc..1308c595d 100644
--- a/yt_dlp/extractor/stacommu.py
+++ b/yt_dlp/extractor/stacommu.py
@@ -38,9 +38,48 @@ def _extract_hls_key(self, data, path, decrypt):
             return None
         return traverse_obj(encryption_data, {'key': ('key', {decrypt}), 'iv': ('iv', {decrypt})})
 
+    def _extract_vod(self, url):
+        video_id = self._match_id(url)
+        video_info = self._download_metadata(
+            url, video_id, 'ja', ('dehydratedState', 'queries', 0, 'state', 'data'))
+        hls_info, decrypt = self._call_encrypted_api(
+            video_id, ':watch', 'stream information', data={'method': 1})
+
+        return {
+            'id': video_id,
+            'formats': self._get_formats(hls_info, ('protocolHls', 'url', {url_or_none}), video_id),
+            'hls_aes': self._extract_hls_key(hls_info, 'protocolHls', decrypt),
+            **traverse_obj(video_info, {
+                'title': ('displayName', {str}),
+                'description': ('description', {str}),
+                'timestamp': ('watchStartTime', {int_or_none}),
+                'thumbnail': ('keyVisualUrl', {url_or_none}),
+                'cast': ('casts', ..., 'displayName', {str}),
+                'duration': ('duration', {int}),
+            }),
+        }
+
+    def _extract_ppv(self, url):
+        video_id = self._match_id(url)
+        video_info = self._call_api(video_id, msg='video information', query={'al': 'ja'}, auth=False)
+        hls_info, decrypt = self._call_encrypted_api(
+            video_id, ':watchArchive', 'stream information', data={'method': 1})
+
+        return {
+            'id': video_id,
+            'formats': self._get_formats(hls_info, ('hls', 'urls', ..., {url_or_none}), video_id),
+            'hls_aes': self._extract_hls_key(hls_info, 'hls', decrypt),
+            **traverse_obj(video_info, {
+                'title': ('displayName', {str}),
+                'timestamp': ('startTime', {int_or_none}),
+                'thumbnail': ('keyVisualUrl', {url_or_none}),
+                'duration': ('duration', {int_or_none}),
+            }),
+        }
+
 
 class StacommuVODIE(StacommuBaseIE):
-    _VALID_URL = r'https?://www\.stacommu\.jp/videos/episodes/(?P<id>[\da-zA-Z]+)'
+    _VALID_URL = r'https?://www\.stacommu\.jp/(?:en/)?videos/episodes/(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         # not encrypted
         'url': 'https://www.stacommu.jp/videos/episodes/aXcVKjHyAENEjard61soZZ',
@@ -79,34 +118,19 @@ class StacommuVODIE(StacommuBaseIE):
         'params': {
             'skip_download': 'm3u8',
         },
+    }, {
+        'url': 'https://www.stacommu.jp/en/videos/episodes/aXcVKjHyAENEjard61soZZ',
+        'only_matching': True,
     }]
 
     _API_PATH = 'videoEpisodes'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        video_info = self._download_metadata(
-            url, video_id, 'ja', ('dehydratedState', 'queries', 0, 'state', 'data'))
-        hls_info, decrypt = self._call_encrypted_api(
-            video_id, ':watch', 'stream information', data={'method': 1})
-
-        return {
-            'id': video_id,
-            'formats': self._get_formats(hls_info, ('protocolHls', 'url', {url_or_none}), video_id),
-            'hls_aes': self._extract_hls_key(hls_info, 'protocolHls', decrypt),
-            **traverse_obj(video_info, {
-                'title': ('displayName', {str}),
-                'description': ('description', {str}),
-                'timestamp': ('watchStartTime', {int_or_none}),
-                'thumbnail': ('keyVisualUrl', {url_or_none}),
-                'cast': ('casts', ..., 'displayName', {str}),
-                'duration': ('duration', {int}),
-            }),
-        }
+        return self._extract_vod(url)
 
 
 class StacommuLiveIE(StacommuBaseIE):
-    _VALID_URL = r'https?://www\.stacommu\.jp/live/(?P<id>[\da-zA-Z]+)'
+    _VALID_URL = r'https?://www\.stacommu\.jp/(?:en/)?live/(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         'url': 'https://www.stacommu.jp/live/d2FJ3zLnndegZJCAEzGM3m',
         'info_dict': {
@@ -125,24 +149,83 @@ class StacommuLiveIE(StacommuBaseIE):
         'params': {
             'skip_download': 'm3u8',
         },
+    }, {
+        'url': 'https://www.stacommu.jp/en/live/d2FJ3zLnndegZJCAEzGM3m',
+        'only_matching': True,
     }]
 
     _API_PATH = 'events'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        video_info = self._call_api(video_id, msg='video information', query={'al': 'ja'}, auth=False)
-        hls_info, decrypt = self._call_encrypted_api(
-            video_id, ':watchArchive', 'stream information', data={'method': 1})
+        return self._extract_ppv(url)
 
-        return {
-            'id': video_id,
-            'formats': self._get_formats(hls_info, ('hls', 'urls', ..., {url_or_none}), video_id),
-            'hls_aes': self._extract_hls_key(hls_info, 'hls', decrypt),
-            **traverse_obj(video_info, {
-                'title': ('displayName', {str}),
-                'timestamp': ('startTime', {int_or_none}),
-                'thumbnail': ('keyVisualUrl', {url_or_none}),
-                'duration': ('duration', {int_or_none}),
-            }),
-        }
+
+class TheaterComplexTownBaseIE(StacommuBaseIE):
+    _NETRC_MACHINE = 'theatercomplextown'
+    _API_HOST = 'api.theater-complex.town'
+    _LOGIN_QUERY = {'key': 'AIzaSyAgNCqToaIz4a062EeIrkhI_xetVfAOrfc'}
+    _LOGIN_HEADERS = {
+        'Accept': '*/*',
+        'Content-Type': 'application/json',
+        'X-Client-Version': 'Chrome/JsCore/9.23.0/FirebaseCore-web',
+        'Referer': 'https://www.theater-complex.town/',
+        'Origin': 'https://www.theater-complex.town',
+    }
+
+
+class TheaterComplexTownVODIE(TheaterComplexTownBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:en/)?videos/episodes/(?P<id>\w+)'
+    IE_NAME = 'theatercomplextown:vod'
+    _TESTS = [{
+        'url': 'https://www.theater-complex.town/videos/episodes/hoxqidYNoAn7bP92DN6p78',
+        'info_dict': {
+            'id': 'hoxqidYNoAn7bP92DN6p78',
+            'ext': 'mp4',
+            'title': '演劇ドラフトグランプリ2023　劇団『恋のぼり』〜劇団名決定秘話ラジオ',
+            'description': 'md5:a7e2e9cf570379ea67fb630f345ff65d',
+            'cast': ['玉城 裕規', '石川 凌雅'],
+            'thumbnail': 'https://image.theater-complex.town/5URnXX6KCeDysuFrPkP38o/5URnXX6KCeDysuFrPkP38o',
+            'upload_date': '20231103',
+            'timestamp': 1699016400,
+            'duration': 868,
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }, {
+        'url': 'https://www.theater-complex.town/en/videos/episodes/6QT7XYwM9dJz5Gf9VB6K5y',
+        'only_matching': True,
+    }]
+
+    _API_PATH = 'videoEpisodes'
+
+    def _real_extract(self, url):
+        return self._extract_vod(url)
+
+
+class TheaterComplexTownPPVIE(TheaterComplexTownBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:en/)?ppv/(?P<id>\w+)'
+    IE_NAME = 'theatercomplextown:ppv'
+    _TESTS = [{
+        'url': 'https://www.theater-complex.town/ppv/wytW3X7khrjJBUpKuV3jen',
+        'info_dict': {
+            'id': 'wytW3X7khrjJBUpKuV3jen',
+            'ext': 'mp4',
+            'title': 'BREAK FREE STARS　11月5日（日）12:30千秋楽公演',
+            'thumbnail': 'https://image.theater-complex.town/5GWEB31JcTUfjtgdeV5t6o/5GWEB31JcTUfjtgdeV5t6o',
+            'upload_date': '20231105',
+            'timestamp': 1699155000,
+            'duration': 8378,
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }, {
+        'url': 'https://www.theater-complex.town/en/ppv/wytW3X7khrjJBUpKuV3jen',
+        'only_matching': True,
+    }]
+
+    _API_PATH = 'events'
+
+    def _real_extract(self, url):
+        return self._extract_ppv(url)

From f6e97090d2ed9e05441ab0f4bec3559b816d7a00 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 11 Nov 2023 14:05:07 -0600
Subject: [PATCH 662/871] [ie/twitter:broadcast] Support `--wait-for-video`
 (#8475)

Closes #8473
Authored by: bashonly
---
 yt_dlp/extractor/periscope.py | 9 ++++++---
 yt_dlp/extractor/twitter.py   | 3 +++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/periscope.py b/yt_dlp/extractor/periscope.py
index dcd021926..3d1375b64 100644
--- a/yt_dlp/extractor/periscope.py
+++ b/yt_dlp/extractor/periscope.py
@@ -4,6 +4,7 @@
     parse_iso8601,
     unescapeHTML,
 )
+from ..utils.traversal import traverse_obj
 
 
 class PeriscopeBaseIE(InfoExtractor):
@@ -20,8 +21,6 @@ def _parse_broadcast_data(self, broadcast, video_id):
         title = broadcast.get('status') or 'Periscope Broadcast'
         uploader = broadcast.get('user_display_name') or broadcast.get('username')
         title = '%s - %s' % (uploader, title) if uploader else title
-        is_live = broadcast.get('state').lower() == 'running'
-
         thumbnails = [{
             'url': broadcast[image],
         } for image in ('image_url', 'image_url_medium', 'image_url_small') if broadcast.get(image)]
@@ -31,12 +30,16 @@ def _parse_broadcast_data(self, broadcast, video_id):
             'title': title,
             'timestamp': parse_iso8601(broadcast.get('created_at')) or int_or_none(
                 broadcast.get('created_at_ms'), scale=1000),
+            'release_timestamp': int_or_none(broadcast.get('scheduled_start_ms'), scale=1000),
             'uploader': uploader,
             'uploader_id': broadcast.get('user_id') or broadcast.get('username'),
             'thumbnails': thumbnails,
             'view_count': int_or_none(broadcast.get('total_watched')),
             'tags': broadcast.get('tags'),
-            'is_live': is_live,
+            'live_status': {
+                'running': 'is_live',
+                'not_started': 'is_upcoming',
+            }.get(traverse_obj(broadcast, ('state', {str.lower}))) or 'was_live'
         }
 
     @staticmethod
diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 7bd78eb48..d7609bc81 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1619,6 +1619,9 @@ def _real_extract(self, url):
         info['title'] = broadcast.get('status') or info.get('title')
         info['uploader_id'] = broadcast.get('twitter_username') or info.get('uploader_id')
         info['uploader_url'] = format_field(broadcast, 'twitter_username', 'https://twitter.com/%s', default=None)
+        if info['live_status'] == 'is_upcoming':
+            return info
+
         media_key = broadcast['media_key']
         source = self._call_api(
             f'live_video_stream/status/{media_key}', media_key)['source']

From 6ba3085616652cbf05d1858efc321fdbfc4c6119 Mon Sep 17 00:00:00 2001
From: Esokrates <Esokrarkose@gmail.com>
Date: Sat, 11 Nov 2023 20:06:25 +0000
Subject: [PATCH 663/871] [ie/orf:podcast] Add extractor (#8486)

Closes #5265
Authored by: Esokrates
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/orf.py         | 45 +++++++++++++++++++++++++++++++--
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index c4bf2acdf..525944c61 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1420,6 +1420,7 @@
     ORFTVthekIE,
     ORFFM4StoryIE,
     ORFRadioIE,
+    ORFPodcastIE,
     ORFIPTVIE,
 )
 from .outsidetv import OutsideTVIE
diff --git a/yt_dlp/extractor/orf.py b/yt_dlp/extractor/orf.py
index cc3c003fa..9a48ae1b3 100644
--- a/yt_dlp/extractor/orf.py
+++ b/yt_dlp/extractor/orf.py
@@ -4,15 +4,16 @@
 from .common import InfoExtractor
 from ..networking import HEADRequest
 from ..utils import (
+    InAdvancePagedList,
     clean_html,
     determine_ext,
     float_or_none,
-    InAdvancePagedList,
     int_or_none,
     join_nonempty,
+    make_archive_id,
+    mimetype2ext,
     orderedSet,
     remove_end,
-    make_archive_id,
     smuggle_url,
     strip_jsonp,
     try_call,
@@ -21,6 +22,7 @@
     unsmuggle_url,
     url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class ORFTVthekIE(InfoExtractor):
@@ -334,6 +336,45 @@ def _real_extract(self, url):
             self._entries(data, station or station2), show_id, data.get('title'), clean_html(data.get('subtitle')))
 
 
+class ORFPodcastIE(InfoExtractor):
+    IE_NAME = 'orf:podcast'
+    _STATION_RE = '|'.join(map(re.escape, (
+        'bgl', 'fm4', 'ktn', 'noe', 'oe1', 'oe3',
+        'ooe', 'sbg', 'stm', 'tir', 'tv', 'vbg', 'wie')))
+    _VALID_URL = rf'https?://sound\.orf\.at/podcast/(?P<station>{_STATION_RE})/(?P<show>[\w-]+)/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://sound.orf.at/podcast/oe3/fruehstueck-bei-mir/nicolas-stockhammer-15102023',
+        'md5': '526a5700e03d271a1505386a8721ab9b',
+        'info_dict': {
+            'id': 'nicolas-stockhammer-15102023',
+            'ext': 'mp3',
+            'title': 'Nicolas Stockhammer (15.10.2023)',
+            'duration': 3396.0,
+            'series': 'Frühstück bei mir',
+        },
+        'skip': 'ORF podcasts are only available for a limited time'
+    }]
+
+    def _real_extract(self, url):
+        station, show, show_id = self._match_valid_url(url).group('station', 'show', 'id')
+        data = self._download_json(
+            f'https://audioapi.orf.at/radiothek/api/2.0/podcast/{station}/{show}/{show_id}', show_id)
+
+        return {
+            'id': show_id,
+            'ext': 'mp3',
+            'vcodec': 'none',
+            **traverse_obj(data, ('payload', {
+                'url': ('enclosures', 0, 'url'),
+                'ext': ('enclosures', 0, 'type', {mimetype2ext}),
+                'title': 'title',
+                'description': ('description', {clean_html}),
+                'duration': ('duration', {functools.partial(float_or_none, scale=1000)}),
+                'series': ('podcast', 'title'),
+            })),
+        }
+
+
 class ORFIPTVIE(InfoExtractor):
     IE_NAME = 'orf:iptv'
     IE_DESC = 'iptv.ORF.at'

From 46acc418a53470b7f32581b3309c3cb87aa8488d Mon Sep 17 00:00:00 2001
From: LoserFox <57448027+LoserFox@users.noreply.github.com>
Date: Sun, 12 Nov 2023 04:08:53 +0800
Subject: [PATCH 664/871] [ie/neteasemusic] Improve metadata extraction (#8531)

Closes #8530
Authored by: LoserFox
---
 yt_dlp/extractor/neteasemusic.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/yt_dlp/extractor/neteasemusic.py b/yt_dlp/extractor/neteasemusic.py
index 68bfcb6ba..d332b840c 100644
--- a/yt_dlp/extractor/neteasemusic.py
+++ b/yt_dlp/extractor/neteasemusic.py
@@ -142,6 +142,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'subtitles': {'lyrics': [{'ext': 'lrc'}]},
             "duration": 256,
             'thumbnail': r're:^http.*\.jpg',
+            'album': '偶像练习生 表演曲目合集',
+            'average_rating': int,
+            'album_artist': '偶像练习生',
         },
     }, {
         'note': 'No lyrics.',
@@ -155,6 +158,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'timestamp': 1202745600,
             'duration': 263,
             'thumbnail': r're:^http.*\.jpg',
+            'album': 'Piano Solos Vol. 2',
+            'album_artist': 'Dustin O\'Halloran',
+            'average_rating': int,
         },
     }, {
         'url': 'https://y.music.163.com/m/song?app_version=8.8.45&id=95670&uct2=sKnvS4+0YStsWkqsPhFijw%3D%3D&dlt=0846',
@@ -171,6 +177,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'duration': 268,
             'alt_title': '伴唱:现代人乐队 合唱:总政歌舞团',
             'thumbnail': r're:^http.*\.jpg',
+            'average_rating': int,
+            'album': '红色摇滚',
+            'album_artist': '侯牧人',
         },
     }, {
         'url': 'http://music.163.com/#/song?id=32102397',
@@ -186,6 +195,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'subtitles': {'lyrics': [{'ext': 'lrc'}]},
             'duration': 199,
             'thumbnail': r're:^http.*\.jpg',
+            'album': 'Bad Blood',
+            'average_rating': int,
+            'album_artist': 'Taylor Swift',
         },
         'skip': 'Blocked outside Mainland China',
     }, {
@@ -203,6 +215,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'duration': 229,
             'alt_title': '说出愿望吧(Genie)',
             'thumbnail': r're:^http.*\.jpg',
+            'average_rating': int,
+            'album': 'Oh!',
+            'album_artist': '少女时代',
         },
         'skip': 'Blocked outside Mainland China',
     }]
@@ -253,12 +268,15 @@ def _real_extract(self, url):
             'formats': formats,
             'alt_title': '/'.join(traverse_obj(info, (('transNames', 'alias'), ...))) or None,
             'creator': ' / '.join(traverse_obj(info, ('artists', ..., 'name'))) or None,
+            'album_artist': ' / '.join(traverse_obj(info, ('album', 'artists', ..., 'name'))) or None,
             **lyric_data,
             **traverse_obj(info, {
                 'title': ('name', {str}),
                 'timestamp': ('album', 'publishTime', {self.kilo_or_none}),
                 'thumbnail': ('album', 'picUrl', {url_or_none}),
                 'duration': ('duration', {self.kilo_or_none}),
+                'album': ('album', 'name', {str}),
+                'average_rating': ('score', {int_or_none}),
             }),
         }
 

From ef12dbdcd3e7264bd3d744c1e3107597bd23ad35 Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Sat, 11 Nov 2023 20:10:19 +0000
Subject: [PATCH 665/871] [ie/radiocomercial] Add extractors (#8508)

Authored by: SirElderling
---
 yt_dlp/extractor/_extractors.py    |   4 +
 yt_dlp/extractor/radiocomercial.py | 150 +++++++++++++++++++++++++++++
 2 files changed, 154 insertions(+)
 create mode 100644 yt_dlp/extractor/radiocomercial.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 525944c61..8b036bb69 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1583,6 +1583,10 @@
     RadioCanadaIE,
     RadioCanadaAudioVideoIE,
 )
+from .radiocomercial import (
+    RadioComercialIE,
+    RadioComercialPlaylistIE,
+)
 from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
diff --git a/yt_dlp/extractor/radiocomercial.py b/yt_dlp/extractor/radiocomercial.py
new file mode 100644
index 000000000..07891fe41
--- /dev/null
+++ b/yt_dlp/extractor/radiocomercial.py
@@ -0,0 +1,150 @@
+import itertools
+
+from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
+from ..utils import (
+    ExtractorError,
+    extract_attributes,
+    get_element_by_class,
+    get_element_html_by_class,
+    get_element_text_and_html_by_tag,
+    get_elements_html_by_class,
+    int_or_none,
+    join_nonempty,
+    try_call,
+    unified_strdate,
+    update_url,
+    urljoin
+)
+from ..utils.traversal import traverse_obj
+
+
+class RadioComercialIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/[^/?#]+/t?(?P<season>\d+)/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao/t6/taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas#page-content-wrapper',
+        'md5': '5f4fe8e485b29d2e8fd495605bc2c7e4',
+        'info_dict': {
+            'id': 'taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas',
+            'ext': 'mp3',
+            'title': 'Taylor Swift entranhando-se que nem uma espada no ventre dos fãs.',
+            'release_date': '20231025',
+            'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg',
+            'season': 6
+        }
+    }, {
+        'url': 'https://radiocomercial.pt/podcasts/convenca-me-num-minuto/t3/convenca-me-num-minuto-que-os-lobisomens-existem',
+        'md5': '47e96c273aef96a8eb160cd6cf46d782',
+        'info_dict': {
+            'id': 'convenca-me-num-minuto-que-os-lobisomens-existem',
+            'ext': 'mp3',
+            'title': 'Convença-me num minuto que os lobisomens existem',
+            'release_date': '20231026',
+            'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg',
+            'season': 3
+        }
+    }, {
+        'url': 'https://radiocomercial.pt/podcasts/inacreditavel-by-ines-castel-branco/t2/o-desastre-de-aviao',
+        'md5': '69be64255420fec23b7259955d771e54',
+        'info_dict': {
+            'id': 'o-desastre-de-aviao',
+            'ext': 'mp3',
+            'title': 'O desastre de avião',
+            'description': 'md5:8a82beeb372641614772baab7246245f',
+            'release_date': '20231101',
+            'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg',
+            'season': 2
+        },
+        'params': {
+            # inconsistant md5
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://radiocomercial.pt/podcasts/tnt-todos-no-top/2023/t-n-t-29-de-outubro',
+        'md5': '91d32d4d4b1407272068b102730fc9fa',
+        'info_dict': {
+            'id': 't-n-t-29-de-outubro',
+            'ext': 'mp3',
+            'title': 'T.N.T 29 de outubro',
+            'release_date': '20231029',
+            'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg',
+            'season': 2023
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id, season = self._match_valid_url(url).group('id', 'season')
+        webpage = self._download_webpage(url, video_id)
+        return {
+            'id': video_id,
+            'title': self._html_extract_title(webpage),
+            'description': self._og_search_description(webpage, default=None),
+            'release_date': unified_strdate(get_element_by_class(
+                'date', get_element_html_by_class('descriptions', webpage) or '')),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'season': int_or_none(season),
+            'url': extract_attributes(get_element_html_by_class('audiofile', webpage) or '').get('href'),
+        }
+
+
+class RadioComercialPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/(?P<id>[\w-]+)(?:/t?(?P<season>\d+))?/?(?:$|[?#])'
+    _TESTS = [{
+        'url': 'https://radiocomercial.pt/podcasts/convenca-me-num-minuto/t3',
+        'info_dict': {
+            'id': 'convenca-me-num-minuto_t3',
+            'title': 'Convença-me num Minuto - Temporada 3',
+        },
+        'playlist_mincount': 32
+    }, {
+        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao',
+        'info_dict': {
+            'id': 'o-homem-que-mordeu-o-cao',
+            'title': 'O Homem Que Mordeu o Cão',
+        },
+        'playlist_mincount': 19
+    }, {
+        'url': 'https://radiocomercial.pt/podcasts/as-minhas-coisas-favoritas',
+        'info_dict': {
+            'id': 'as-minhas-coisas-favoritas',
+            'title': 'As Minhas Coisas Favoritas',
+        },
+        'playlist_mincount': 131
+    }, {
+        'url': 'https://radiocomercial.pt/podcasts/tnt-todos-no-top/t2023',
+        'info_dict': {
+            'id': 'tnt-todos-no-top_t2023',
+            'title': 'TNT - Todos No Top - Temporada 2023',
+        },
+        'playlist_mincount': 39
+    }]
+
+    def _entries(self, url, playlist_id):
+        for page in itertools.count(1):
+            try:
+                webpage = self._download_webpage(
+                    f'{url}/{page}', playlist_id, f'Downloading page {page}')
+            except ExtractorError as e:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 404:
+                    break
+                raise
+
+            episodes = get_elements_html_by_class('tm-ouvir-podcast', webpage)
+            if not episodes:
+                break
+            for url_path in traverse_obj(episodes, (..., {extract_attributes}, 'href')):
+                episode_url = urljoin(url, url_path)
+                if RadioComercialIE.suitable(episode_url):
+                    yield episode_url
+
+    def _real_extract(self, url):
+        podcast, season = self._match_valid_url(url).group('id', 'season')
+        playlist_id = join_nonempty(podcast, season, delim='_t')
+        url = update_url(url, query=None, fragment=None)
+        webpage = self._download_webpage(url, playlist_id)
+
+        name = try_call(lambda: get_element_text_and_html_by_tag('h1', webpage)[0])
+        title = name if name == season else join_nonempty(name, season, delim=' - Temporada ')
+
+        return self.playlist_from_matches(
+            self._entries(url, playlist_id), playlist_id, title, ie=RadioComercialIE)

From 8afd9468b0c822843bc480d366d1c86698daabfb Mon Sep 17 00:00:00 2001
From: Frank Aurich <1100101@gmail.com>
Date: Sat, 11 Nov 2023 22:00:06 +0100
Subject: [PATCH 666/871] [ie/n-tv.de] Fix extractor (#8414)

Closes #3179
Authored by: 1100101
---
 yt_dlp/extractor/ntvde.py | 76 ++++++++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 33 deletions(-)

diff --git a/yt_dlp/extractor/ntvde.py b/yt_dlp/extractor/ntvde.py
index 6d7ea3d18..9f3a498ab 100644
--- a/yt_dlp/extractor/ntvde.py
+++ b/yt_dlp/extractor/ntvde.py
@@ -1,21 +1,21 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     js_to_json,
-    parse_duration,
+    url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class NTVDeIE(InfoExtractor):
     IE_NAME = 'n-tv.de'
-    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/videos/[^/?#]+/[^/?#]+-article(?P<id>.+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/(?:videos|magazine)/[^/?#]+/[^/?#]+-article(?P<id>[^/?#]+)\.html'
 
     _TESTS = [{
         'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
-        'md5': '6ef2514d4b1e8e03ca24b49e2f167153',
+        'md5': '6bcf2a6638cb83f45d5561659a1cb498',
         'info_dict': {
             'id': '14438086',
             'ext': 'mp4',
@@ -23,51 +23,61 @@ class NTVDeIE(InfoExtractor):
             'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
             'alt_title': 'Winterchaos auf deutschen Straßen',
             'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
-            'duration': 4020,
+            'duration': 67,
             'timestamp': 1422892797,
             'upload_date': '20150202',
         },
+    }, {
+        'url': 'https://www.n-tv.de/mediathek/magazine/auslandsreport/Juedische-Siedler-wollten-Rache-die-wollten-nur-toeten-article24523089.html',
+        'md5': 'c5c6014c014ccc3359470e1d34472bfd',
+        'info_dict': {
+            'id': '24523089',
+            'ext': 'mp4',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'title': 'Jüdische Siedler "wollten Rache, die wollten nur töten"',
+            'alt_title': 'Israelische Gewalt fern von Gaza',
+            'description': 'Vier Tage nach dem Massaker der Hamas greifen jüdische Siedler das Haus einer palästinensischen Familie im Westjordanland an. Die Überlebenden berichten, sie waren unbewaffnet, die Angreifer seien nur auf "Rache und Töten" aus gewesen. Als die Toten beerdigt werden sollen, eröffnen die Siedler erneut das Feuer.',
+            'duration': 326,
+            'timestamp': 1699688294,
+            'upload_date': '20231111',
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        info = self._parse_json(self._search_regex(
-            r'(?s)ntv\.pageInfo\.article\s*=\s*(\{.*?\});', webpage, 'info'),
-            video_id, transform_source=js_to_json)
-        timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
-        vdata = self._parse_json(self._search_regex(
-            r'(?s)\$\(\s*"\#player"\s*\)\s*\.data\(\s*"player",\s*(\{.*?\})\);',
-            webpage, 'player data'), video_id,
-            transform_source=lambda s: js_to_json(re.sub(r'advertising:\s*{[^}]+},', '', s)))
-        duration = parse_duration(vdata.get('duration'))
+        info = self._search_json(
+            r'article:', webpage, 'info', video_id, transform_source=js_to_json)
+
+        vdata = self._search_json(
+            r'\$\(\s*"#playerwrapper"\s*\)\s*\.data\(\s*"player",',
+            webpage, 'player data', video_id,
+            transform_source=lambda s: js_to_json(re.sub(r'ivw:[^},]+', '', s)))['setup']['source']
 
         formats = []
-        if vdata.get('video'):
+        if vdata.get('progressive'):
             formats.append({
-                'format_id': 'flash',
-                'url': 'rtmp://fms.n-tv.de/%s' % vdata['video'],
+                'format_id': 'http',
+                'url': vdata['progressive'],
             })
-        if vdata.get('videoMp4'):
-            formats.append({
-                'format_id': 'mobile',
-                'url': compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoMp4']),
-                'tbr': 400,  # estimation
-            })
-        if vdata.get('videoM3u8'):
-            m3u8_url = compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoM3u8'])
+        if vdata.get('hls'):
             formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
-                quality=1, m3u8_id='hls', fatal=False))
+                vdata['hls'], video_id, 'mp4', m3u8_id='hls', fatal=False))
+        if vdata.get('dash'):
+            formats.extend(self._extract_mpd_formats(vdata['dash'], video_id, fatal=False, mpd_id='dash'))
 
         return {
             'id': video_id,
-            'title': info['headline'],
-            'description': info.get('intro'),
-            'alt_title': info.get('kicker'),
-            'timestamp': timestamp,
-            'thumbnail': vdata.get('html5VideoPoster'),
-            'duration': duration,
+            **traverse_obj(info, {
+                'title': 'headline',
+                'description': 'intro',
+                'alt_title': 'kicker',
+                'timestamp': ('publishedDateAsUnixTimeStamp', {int_or_none}),
+            }),
+            **traverse_obj(vdata, {
+                'thumbnail': ('poster', {url_or_none}),
+                'duration': ('length', {int_or_none}),
+            }),
             'formats': formats,
         }

From 1d03633c5a1621b9f3a756f0a4f9dc61fab3aeaa Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sun, 12 Nov 2023 17:49:15 -0600
Subject: [PATCH 667/871] [build] Overhaul and unify release workflow

Authored by: bashonly, Grub4K

Co-authored-by: Simon Sawicki <contact@grub4k.xyz>
---
 .github/workflows/build.yml           |   4 +-
 .github/workflows/publish.yml         |  97 -------
 .github/workflows/release-master.yml  |  28 ++
 .github/workflows/release-nightly.yml |  55 ++--
 .github/workflows/release.yml         | 352 +++++++++++++++++++++-----
 devscripts/update-formulae.py         |  39 ---
 devscripts/update-version.py          |  15 +-
 devscripts/utils.py                   |   7 +-
 setup.py                              |   4 +-
 yt_dlp/update.py                      |   1 +
 yt_dlp/version.py                     |   4 +
 11 files changed, 365 insertions(+), 241 deletions(-)
 delete mode 100644 .github/workflows/publish.yml
 create mode 100644 .github/workflows/release-master.yml
 delete mode 100644 devscripts/update-formulae.py

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ac0cfdf7c..c9260eeca 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -96,9 +96,7 @@ jobs:
           auto-activate-base: false
       - name: Install Requirements
         run: |
-          sudo apt-get -y install zip pandoc man sed
-          python -m pip install -U pip setuptools wheel
-          python -m pip install -U Pyinstaller -r requirements.txt
+          sudo apt -y install zip pandoc man sed
           reqs=$(mktemp)
           cat > $reqs << EOF
           python=3.10.*
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
deleted file mode 100644
index 9ebf54e7f..000000000
--- a/.github/workflows/publish.yml
+++ /dev/null
@@ -1,97 +0,0 @@
-name: Publish
-on:
-  workflow_call:
-    inputs:
-      channel:
-        default: stable
-        required: true
-        type: string
-      version:
-        required: true
-        type: string
-      target_commitish:
-        required: true
-        type: string
-      prerelease:
-        default: false
-        required: true
-        type: boolean
-    secrets:
-      ARCHIVE_REPO_TOKEN:
-        required: false
-
-permissions:
-  contents: write
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/download-artifact@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-
-      - name: Generate release notes
-        run: |
-          printf '%s' \
-            '[![Installation](https://img.shields.io/badge/-Which%20file%20should%20I%20download%3F-white.svg?style=for-the-badge)]' \
-              '(https://github.com/yt-dlp/yt-dlp#installation "Installation instructions") ' \
-            '[![Documentation](https://img.shields.io/badge/-Docs-brightgreen.svg?style=for-the-badge&logo=GitBook&labelColor=555555)]' \
-              '(https://github.com/yt-dlp/yt-dlp/tree/2023.03.04#readme "Documentation") ' \
-            '[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)]' \
-              '(https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators "Donate") ' \
-            '[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)]' \
-              '(https://discord.gg/H5MNcFW63r "Discord") ' \
-            ${{ inputs.channel != 'nightly' && '"[![Nightly](https://img.shields.io/badge/Get%20nightly%20builds-purple.svg?style=for-the-badge)]" \
-              "(https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest \"Nightly builds\")"' || '' }} \
-            > ./RELEASE_NOTES
-          printf '\n\n' >> ./RELEASE_NOTES
-          cat >> ./RELEASE_NOTES << EOF
-          #### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files)
-          ---
-          $(python ./devscripts/make_changelog.py -vv --collapsible)
-          EOF
-          printf '%s\n\n' '**This is an automated nightly pre-release build**' >> ./NIGHTLY_NOTES
-          cat ./RELEASE_NOTES >> ./NIGHTLY_NOTES
-          printf '%s\n\n' 'Generated from: https://github.com/${{ github.repository }}/commit/${{ inputs.target_commitish }}' >> ./ARCHIVE_NOTES
-          cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES
-
-      - name: Archive nightly release
-        env:
-          GH_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
-          GH_REPO: ${{ vars.ARCHIVE_REPO }}
-        if: |
-          inputs.channel == 'nightly' && env.GH_TOKEN != '' && env.GH_REPO != ''
-        run: |
-          gh release create \
-            --notes-file ARCHIVE_NOTES \
-            --title "yt-dlp nightly ${{ inputs.version }}" \
-            ${{ inputs.version }} \
-            artifact/*
-
-      - name: Prune old nightly release
-        if: inputs.channel == 'nightly' && !vars.ARCHIVE_REPO
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          gh release delete --yes --cleanup-tag "nightly" || true
-          git tag --delete "nightly" || true
-          sleep 5  # Enough time to cover deletion race condition
-
-      - name: Publish release${{ inputs.channel == 'nightly' && ' (nightly)' || '' }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-        if: (inputs.channel == 'nightly' && !vars.ARCHIVE_REPO) || inputs.channel != 'nightly'
-        run: |
-          gh release create \
-            --notes-file ${{ inputs.channel == 'nightly' && 'NIGHTLY_NOTES' || 'RELEASE_NOTES' }} \
-            --target ${{ inputs.target_commitish }} \
-            --title "yt-dlp ${{ inputs.channel == 'nightly' && 'nightly ' || '' }}${{ inputs.version }}" \
-            ${{ inputs.prerelease && '--prerelease' || '' }} \
-            ${{ inputs.channel == 'nightly' && '"nightly"' || inputs.version }} \
-            artifact/*
diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
new file mode 100644
index 000000000..0208b3bef
--- /dev/null
+++ b/.github/workflows/release-master.yml
@@ -0,0 +1,28 @@
+name: Release (master)
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "yt_dlp/**.py"
+      - "!yt_dlp/version.py"
+      - "setup.py"
+      - "pyinst.py"
+concurrency:
+  group: release-master
+  cancel-in-progress: true
+permissions:
+  contents: read
+
+jobs:
+  release:
+    if: vars.BUILD_MASTER != ''
+    uses: ./.github/workflows/release.yml
+    with:
+      prerelease: true
+      source: master
+    permissions:
+      contents: write
+      packages: write
+      id-token: write  # mandatory for trusted publishing
+    secrets: inherit
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index 543e2e6f7..947eada27 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -1,52 +1,35 @@
 name: Release (nightly)
 on:
-  push:
-    branches:
-      - master
-    paths:
-      - "yt_dlp/**.py"
-      - "!yt_dlp/version.py"
-concurrency:
-  group: release-nightly
-  cancel-in-progress: true
+  schedule:
+    - cron: '23 23 * * *'
 permissions:
   contents: read
 
 jobs:
-  prepare:
+  check_nightly:
     if: vars.BUILD_NIGHTLY != ''
     runs-on: ubuntu-latest
     outputs:
-      version: ${{ steps.get_version.outputs.version }}
-
+      commit: ${{ steps.check_for_new_commits.outputs.commit }}
     steps:
       - uses: actions/checkout@v3
-      - name: Get version
-        id: get_version
+        with:
+          fetch-depth: 0
+      - name: Check for new commits
+        id: check_for_new_commits
         run: |
-          python devscripts/update-version.py "$(date -u +"%H%M%S")" | grep -Po "version=\d+(\.\d+){3}" >> "$GITHUB_OUTPUT"
+          relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "setup.py" "pyinst.py")
+          echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT"
 
-  build:
-    needs: prepare
-    uses: ./.github/workflows/build.yml
+  release:
+    needs: [check_nightly]
+    if: ${{ needs.check_nightly.outputs.commit }}
+    uses: ./.github/workflows/release.yml
     with:
-      version: ${{ needs.prepare.outputs.version }}
-      channel: nightly
-    permissions:
-      contents: read
-      packages: write # For package cache
-    secrets:
-      GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
-
-  publish:
-    needs: [prepare, build]
-    uses: ./.github/workflows/publish.yml
-    secrets:
-      ARCHIVE_REPO_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
+      prerelease: true
+      source: nightly
     permissions:
       contents: write
-    with:
-      channel: nightly
-      prerelease: true
-      version: ${{ needs.prepare.outputs.version }}
-      target_commitish: ${{ github.sha }}
+      packages: write
+      id-token: write  # mandatory for trusted publishing
+    secrets: inherit
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ada508be8..0e50b74e0 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,14 +1,45 @@
 name: Release
 on:
-  workflow_dispatch:
+  workflow_call:
     inputs:
-      version:
-        description: Version tag (YYYY.MM.DD[.REV])
+      prerelease:
+        required: false
+        default: true
+        type: boolean
+      source:
         required: false
         default: ''
         type: string
-      channel:
-        description: Update channel (stable/nightly/...)
+      target:
+        required: false
+        default: ''
+        type: string
+      version:
+        required: false
+        default: ''
+        type: string
+  workflow_dispatch:
+    inputs:
+      source:
+        description: |
+          SOURCE of this release's updates:
+          channel, repo, tag, or channel/repo@tag
+          (default: <current_repo>)
+        required: false
+        default: ''
+        type: string
+      target:
+        description: |
+          TARGET to publish this release to:
+          channel, tag, or channel@tag
+          (default: <source> if writable else <current_repo>[@source_tag])
+        required: false
+        default: ''
+        type: string
+      version:
+        description: |
+          VERSION: yyyy.mm.dd[.rev] or rev
+          (default: auto-generated)
         required: false
         default: ''
         type: string
@@ -26,8 +57,14 @@ jobs:
       contents: write
     runs-on: ubuntu-latest
     outputs:
-      channel: ${{ steps.set_channel.outputs.channel }}
-      version: ${{ steps.update_version.outputs.version }}
+      channel: ${{ steps.setup_variables.outputs.channel }}
+      version: ${{ steps.setup_variables.outputs.version }}
+      target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+      target_repo_token: ${{ steps.setup_variables.outputs.target_repo_token }}
+      target_tag: ${{ steps.setup_variables.outputs.target_tag }}
+      pypi_project: ${{ steps.setup_variables.outputs.pypi_project }}
+      pypi_suffix: ${{ steps.setup_variables.outputs.pypi_suffix }}
+      pypi_token: ${{ steps.setup_variables.outputs.pypi_token }}
       head_sha: ${{ steps.get_target.outputs.head_sha }}
 
     steps:
@@ -39,25 +76,132 @@ jobs:
         with:
           python-version: "3.10"
 
-      - name: Set channel
-        id: set_channel
+      - name: Process inputs
+        id: process_inputs
         run: |
-          CHANNEL="${{ github.repository == 'yt-dlp/yt-dlp' && 'stable' || github.repository }}"
-          echo "channel=${{ inputs.channel || '$CHANNEL' }}" > "$GITHUB_OUTPUT"
+          cat << EOF
+          ::group::Inputs
+          prerelease=${{ inputs.prerelease }}
+          source=${{ inputs.source }}
+          target=${{ inputs.target }}
+          version=${{ inputs.version }}
+          ::endgroup::
+          EOF
+          IFS='@' read -r source_repo source_tag <<<"${{ inputs.source }}"
+          IFS='@' read -r target_repo target_tag <<<"${{ inputs.target }}"
+          cat << EOF >> "$GITHUB_OUTPUT"
+          source_repo=${source_repo}
+          source_tag=${source_tag}
+          target_repo=${target_repo}
+          target_tag=${target_tag}
+          EOF
 
-      - name: Update version
-        id: update_version
+      - name: Setup variables
+        id: setup_variables
+        env:
+          source_repo: ${{ steps.process_inputs.outputs.source_repo }}
+          source_tag: ${{ steps.process_inputs.outputs.source_tag }}
+          target_repo: ${{ steps.process_inputs.outputs.target_repo }}
+          target_tag: ${{ steps.process_inputs.outputs.target_tag }}
         run: |
-          REVISION="${{ vars.PUSH_VERSION_COMMIT == '' && '$(date -u +"%H%M%S")' || '' }}"
-          REVISION="${{ inputs.prerelease && '$(date -u +"%H%M%S")' || '$REVISION' }}"
-          python devscripts/update-version.py ${{ inputs.version || '$REVISION' }} | \
-            grep -Po "version=\d+\.\d+\.\d+(\.\d+)?" >> "$GITHUB_OUTPUT"
+          # unholy bash monstrosity (sincere apologies)
+          fallback_token () {
+            if ${{ !secrets.ARCHIVE_REPO_TOKEN }}; then
+              echo "::error::Repository access secret ${target_repo_token^^} not found"
+              exit 1
+            fi
+            target_repo_token=ARCHIVE_REPO_TOKEN
+            return 0
+          }
+
+          source_is_channel=0
+          [[ "${source_repo}" == 'stable' ]] && source_repo='yt-dlp/yt-dlp'
+          if [[ -z "${source_repo}" ]]; then
+            source_repo='${{ github.repository }}'
+          elif [[ '${{ vars[format('{0}_archive_repo', env.source_repo)] }}' ]]; then
+            source_is_channel=1
+            source_channel='${{ vars[format('{0}_archive_repo', env.source_repo)] }}'
+          elif [[ -z "${source_tag}" && "${source_repo}" != */* ]]; then
+            source_tag="${source_repo}"
+            source_repo='${{ github.repository }}'
+          fi
+          resolved_source="${source_repo}"
+          if [[ "${source_tag}" ]]; then
+            resolved_source="${resolved_source}@${source_tag}"
+          elif [[ "${source_repo}" == 'yt-dlp/yt-dlp' ]]; then
+            resolved_source='stable'
+          fi
+
+          revision="${{ (inputs.prerelease || !vars.PUSH_VERSION_COMMIT) && '$(date -u +"%H%M%S")' || '' }}"
+          version="$(
+            python devscripts/update-version.py -c "${resolved_source}" ${{ inputs.version || '$revision' }} | \
+            grep -Po "version=\K\d+\.\d+\.\d+(\.\d+)?")"
+
+          if [[ "${target_repo}" ]]; then
+            if [[ -z "${target_tag}" ]]; then
+              if [[ '${{ vars[format('{0}_archive_repo', env.target_repo)] }}' ]]; then
+                target_tag="${source_tag:-${version}}"
+              else
+                target_tag="${target_repo}"
+                target_repo='${{ github.repository }}'
+              fi
+            fi
+            if [[ "${target_repo}" != '${{ github.repository}}' ]]; then
+              target_repo='${{ vars[format('{0}_archive_repo', env.target_repo)] }}'
+              target_repo_token='${{ env.target_repo }}_archive_repo_token'
+              ${{ !!secrets[format('{0}_archive_repo_token', env.target_repo)] }} || fallback_token
+              pypi_project='${{ vars[format('{0}_pypi_project', env.target_repo)] }}'
+              pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.target_repo)] }}'
+              ${{ !secrets[format('{0}_pypi_token', env.target_repo)] }} || pypi_token='${{ env.target_repo }}_pypi_token'
+            fi
+          else
+            target_tag="${source_tag:-${version}}"
+            if ((source_is_channel)); then
+              target_repo="${source_channel}"
+              target_repo_token='${{ env.source_repo }}_archive_repo_token'
+              ${{ !!secrets[format('{0}_archive_repo_token', env.source_repo)] }} || fallback_token
+              pypi_project='${{ vars[format('{0}_pypi_project', env.source_repo)] }}'
+              pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.source_repo)] }}'
+              ${{ !secrets[format('{0}_pypi_token', env.source_repo)] }} || pypi_token='${{ env.source_repo }}_pypi_token'
+            else
+              target_repo='${{ github.repository }}'
+            fi
+          fi
+
+          if [[ "${target_repo}" == '${{ github.repository }}' ]] && ${{ !inputs.prerelease }}; then
+            pypi_project='${{ vars.PYPI_PROJECT }}'
+          fi
+          if [[ -z "${pypi_token}" && "${pypi_project}" ]]; then
+            if ${{ !secrets.PYPI_TOKEN }}; then
+              pypi_token=OIDC
+            else
+              pypi_token=PYPI_TOKEN
+            fi
+          fi
+
+          echo "::group::Output variables"
+          cat << EOF | tee -a "$GITHUB_OUTPUT"
+          channel=${resolved_source}
+          version=${version}
+          target_repo=${target_repo}
+          target_repo_token=${target_repo_token}
+          target_tag=${target_tag}
+          pypi_project=${pypi_project}
+          pypi_suffix=${pypi_suffix}
+          pypi_token=${pypi_token}
+          EOF
+          echo "::endgroup::"
 
       - name: Update documentation
+        env:
+          version: ${{ steps.setup_variables.outputs.version }}
+          target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+        if: |
+          !inputs.prerelease && env.target_repo == github.repository
         run: |
           make doc
           sed '/### /Q' Changelog.md >> ./CHANGELOG
-          echo '### ${{ steps.update_version.outputs.version }}' >> ./CHANGELOG
+          echo '### ${{ env.version }}' >> ./CHANGELOG
           python ./devscripts/make_changelog.py -vv -c >> ./CHANGELOG
           echo >> ./CHANGELOG
           grep -Poz '(?s)### \d+\.\d+\.\d+.+' 'Changelog.md' | head -n -1 >> ./CHANGELOG
@@ -65,12 +209,16 @@ jobs:
 
       - name: Push to release
         id: push_release
-        if: ${{ !inputs.prerelease }}
+        env:
+          version: ${{ steps.setup_variables.outputs.version }}
+          target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+        if: |
+          !inputs.prerelease && env.target_repo == github.repository
         run: |
           git config --global user.name github-actions
-          git config --global user.email github-actions@example.com
+          git config --global user.email github-actions@github.com
           git add -u
-          git commit -m "Release ${{ steps.update_version.outputs.version }}" \
+          git commit -m "Release ${{ env.version }}" \
             -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
           git push origin --force ${{ github.event.ref }}:release
 
@@ -80,7 +228,10 @@ jobs:
           echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
 
       - name: Update master
-        if: vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease
+        env:
+          target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+        if: |
+          vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease && env.target_repo == github.repository
         run: git push origin ${{ github.event.ref }}
 
   build:
@@ -95,9 +246,12 @@ jobs:
     secrets:
       GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
 
-  publish_pypi_homebrew:
+  publish_pypi:
     needs: [prepare, build]
+    if: ${{ needs.prepare.outputs.pypi_project }}
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # mandatory for trusted publishing
 
     steps:
       - uses: actions/checkout@v3
@@ -107,57 +261,137 @@ jobs:
 
       - name: Install Requirements
         run: |
-          sudo apt-get -y install pandoc man
+          sudo apt -y install pandoc man
           python -m pip install -U pip setuptools wheel twine
           python -m pip install -U -r requirements.txt
 
       - name: Prepare
-        run: |
-          python devscripts/update-version.py ${{ needs.prepare.outputs.version }}
-          python devscripts/make_lazy_extractors.py
-
-      - name: Build and publish on PyPI
         env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        if: env.TWINE_PASSWORD != '' && !inputs.prerelease
+          version: ${{ needs.prepare.outputs.version }}
+          suffix: ${{ needs.prepare.outputs.pypi_suffix }}
+          channel: ${{ needs.prepare.outputs.channel }}
+          target_repo: ${{ needs.prepare.outputs.target_repo }}
+          pypi_project: ${{ needs.prepare.outputs.pypi_project }}
+        run: |
+          python devscripts/update-version.py -c "${{ env.channel }}" -r "${{ env.target_repo }}" -s "${{ env.suffix }}" "${{ env.version }}"
+          python devscripts/make_lazy_extractors.py
+          sed -i -E "s/(name=')[^']+(',  # package name)/\1${{ env.pypi_project }}\2/" setup.py
+
+      - name: Build
         run: |
           rm -rf dist/*
           make pypi-files
           python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
           python setup.py sdist bdist_wheel
+
+      - name: Publish to PyPI via token
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets[needs.prepare.outputs.pypi_token] }}
+        if: |
+          needs.prepare.outputs.pypi_token != 'OIDC' && env.TWINE_PASSWORD
+        run: |
           twine upload dist/*
 
-      - name: Checkout Homebrew repository
-        env:
-          BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' && !inputs.prerelease
-        uses: actions/checkout@v3
+      - name: Publish to PyPI via trusted publishing
+        if: |
+          needs.prepare.outputs.pypi_token == 'OIDC'
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          repository: yt-dlp/homebrew-taps
-          path: taps
-          ssh-key: ${{ secrets.BREW_TOKEN }}
-
-      - name: Update Homebrew Formulae
-        env:
-          BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' && !inputs.prerelease
-        run: |
-          python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.version }}"
-          git -C taps/ config user.name github-actions
-          git -C taps/ config user.email github-actions@example.com
-          git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.version }}'
-          git -C taps/ push
+          verbose: true
 
   publish:
     needs: [prepare, build]
-    uses: ./.github/workflows/publish.yml
     permissions:
       contents: write
-    with:
-      channel: ${{ needs.prepare.outputs.channel }}
-      prerelease: ${{ inputs.prerelease }}
-      version: ${{ needs.prepare.outputs.version }}
-      target_commitish: ${{ needs.prepare.outputs.head_sha }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/download-artifact@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Generate release notes
+        env:
+          head_sha: ${{ needs.prepare.outputs.head_sha }}
+          target_repo: ${{ needs.prepare.outputs.target_repo }}
+          target_tag: ${{ needs.prepare.outputs.target_tag }}
+        run: |
+          printf '%s' \
+            '[![Installation](https://img.shields.io/badge/-Which%20file%20should%20I%20download%3F-white.svg?style=for-the-badge)]' \
+              '(https://github.com/${{ github.repository }}#installation "Installation instructions") ' \
+            '[![Documentation](https://img.shields.io/badge/-Docs-brightgreen.svg?style=for-the-badge&logo=GitBook&labelColor=555555)]' \
+              '(https://github.com/${{ github.repository }}' \
+              '${{ env.target_repo == github.repository && format('/tree/{0}', env.target_tag) || '' }}#readme "Documentation") ' \
+            '[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)]' \
+              '(https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators "Donate") ' \
+            '[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)]' \
+              '(https://discord.gg/H5MNcFW63r "Discord") ' \
+            ${{ env.target_repo == 'yt-dlp/yt-dlp' && '\
+              "[![Nightly](https://img.shields.io/badge/Get%20nightly%20builds-purple.svg?style=for-the-badge)]" \
+              "(https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest \"Nightly builds\") " \
+              "[![Master](https://img.shields.io/badge/Get%20master%20builds-lightblue.svg?style=for-the-badge)]" \
+              "(https://github.com/yt-dlp/yt-dlp-master-builds/releases/latest \"Master builds\")"' || '' }} > ./RELEASE_NOTES
+          printf '\n\n' >> ./RELEASE_NOTES
+          cat >> ./RELEASE_NOTES << EOF
+          #### A description of the various files are in the [README](https://github.com/${{ github.repository }}#release-files)
+          ---
+          $(python ./devscripts/make_changelog.py -vv --collapsible)
+          EOF
+          printf '%s\n\n' '**This is a pre-release build**' >> ./PRERELEASE_NOTES
+          cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES
+          printf '%s\n\n' 'Generated from: https://github.com/${{ github.repository }}/commit/${{ env.head_sha }}' >> ./ARCHIVE_NOTES
+          cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES
+
+      - name: Publish to archive repo
+        env:
+          GH_TOKEN: ${{ secrets[needs.prepare.outputs.target_repo_token] }}
+          GH_REPO: ${{ needs.prepare.outputs.target_repo }}
+          version: ${{ needs.prepare.outputs.version }}
+          channel: ${{ needs.prepare.outputs.channel }}
+        if: |
+          inputs.prerelease && env.GH_TOKEN != '' && env.GH_REPO != '' && env.GH_REPO != github.repository
+        run: |
+          title="${{ startswith(env.GH_REPO, 'yt-dlp/') && 'yt-dlp ' || '' }}${{ env.channel }}"
+          gh release create \
+            --notes-file ARCHIVE_NOTES \
+            --title "${title} ${{ env.version }}" \
+            ${{ env.version }} \
+            artifact/*
+
+      - name: Prune old release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          version: ${{ needs.prepare.outputs.version }}
+          target_repo: ${{ needs.prepare.outputs.target_repo }}
+          target_tag: ${{ needs.prepare.outputs.target_tag }}
+        if: |
+          env.target_repo == github.repository && env.target_tag != env.version
+        run: |
+          gh release delete --yes --cleanup-tag "${{ env.target_tag }}" || true
+          git tag --delete "${{ env.target_tag }}" || true
+          sleep 5  # Enough time to cover deletion race condition
+
+      - name: Publish release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          version: ${{ needs.prepare.outputs.version }}
+          target_repo: ${{ needs.prepare.outputs.target_repo }}
+          target_tag: ${{ needs.prepare.outputs.target_tag }}
+          head_sha: ${{ needs.prepare.outputs.head_sha }}
+        if: |
+          env.target_repo == github.repository
+        run: |
+          title="${{ github.repository == 'yt-dlp/yt-dlp' && 'yt-dlp ' || '' }}"
+          title+="${{ env.target_tag != env.version && format('{0} ', env.target_tag) || '' }}"
+          gh release create \
+            --notes-file ${{ inputs.prerelease && 'PRERELEASE_NOTES' || 'RELEASE_NOTES' }} \
+            --target ${{ env.head_sha }} \
+            --title "${title}${{ env.version }}" \
+            ${{ inputs.prerelease && '--prerelease' || '' }} \
+            ${{ env.target_tag }} \
+            artifact/*
diff --git a/devscripts/update-formulae.py b/devscripts/update-formulae.py
deleted file mode 100644
index e79297f53..000000000
--- a/devscripts/update-formulae.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-Usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
-version can be either 0-aligned (yt-dlp version) or normalized (PyPi version)
-"""
-
-# Allow direct execution
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-
-import json
-import re
-import urllib.request
-
-from devscripts.utils import read_file, write_file
-
-filename, version = sys.argv[1:]
-
-normalized_version = '.'.join(str(int(x)) for x in version.split('.'))
-
-pypi_release = json.loads(urllib.request.urlopen(
-    'https://pypi.org/pypi/yt-dlp/%s/json' % normalized_version
-).read().decode())
-
-tarball_file = next(x for x in pypi_release['urls'] if x['filename'].endswith('.tar.gz'))
-
-sha256sum = tarball_file['digests']['sha256']
-url = tarball_file['url']
-
-formulae_text = read_file(filename)
-
-formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text, count=1)
-formulae_text = re.sub(r'url "[^"]*?"', 'url "%s"' % url, formulae_text, count=1)
-
-write_file(filename, formulae_text)
diff --git a/devscripts/update-version.py b/devscripts/update-version.py
index 0144bd284..da54a6a25 100644
--- a/devscripts/update-version.py
+++ b/devscripts/update-version.py
@@ -20,7 +20,7 @@ def get_new_version(version, revision):
         version = datetime.now(timezone.utc).strftime('%Y.%m.%d')
 
     if revision:
-        assert revision.isdigit(), 'Revision must be a number'
+        assert revision.isdecimal(), 'Revision must be a number'
     else:
         old_version = read_version().split('.')
         if version.split('.') == old_version[:3]:
@@ -46,6 +46,10 @@ def get_git_head():
 UPDATE_HINT = None
 
 CHANNEL = {channel!r}
+
+ORIGIN = {origin!r}
+
+_pkg_version = {package_version!r}
 '''
 
 if __name__ == '__main__':
@@ -53,6 +57,12 @@ def get_git_head():
     parser.add_argument(
         '-c', '--channel', default='stable',
         help='Select update channel (default: %(default)s)')
+    parser.add_argument(
+        '-r', '--origin', default='local',
+        help='Select origin/repository (default: %(default)s)')
+    parser.add_argument(
+        '-s', '--suffix', default='',
+        help='Add an alphanumeric suffix to the package version, e.g. "dev"')
     parser.add_argument(
         '-o', '--output', default='yt_dlp/version.py',
         help='The output file to write to (default: %(default)s)')
@@ -66,6 +76,7 @@ def get_git_head():
         args.version if args.version and '.' in args.version
         else get_new_version(None, args.version))
     write_file(args.output, VERSION_TEMPLATE.format(
-        version=version, git_head=git_head, channel=args.channel))
+        version=version, git_head=git_head, channel=args.channel, origin=args.origin,
+        package_version=f'{version}{args.suffix}'))
 
     print(f'version={version} ({args.channel}), head={git_head}')
diff --git a/devscripts/utils.py b/devscripts/utils.py
index f75a84da9..a952c9fae 100644
--- a/devscripts/utils.py
+++ b/devscripts/utils.py
@@ -13,10 +13,11 @@ def write_file(fname, content, mode='w'):
         return f.write(content)
 
 
-def read_version(fname='yt_dlp/version.py'):
+def read_version(fname='yt_dlp/version.py', varname='__version__'):
     """Get the version without importing the package"""
-    exec(compile(read_file(fname), fname, 'exec'))
-    return locals()['__version__']
+    items = {}
+    exec(compile(read_file(fname), fname, 'exec'), items)
+    return items[varname]
 
 
 def get_filename_args(has_infile=False, default_outfile=None):
diff --git a/setup.py b/setup.py
index 1740db27d..44055b0e9 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@
 
 from devscripts.utils import read_file, read_version
 
-VERSION = read_version()
+VERSION = read_version(varname='_pkg_version')
 
 DESCRIPTION = 'A youtube-dl fork with additional features and patches'
 
@@ -142,7 +142,7 @@ def main():
         params = build_params()
 
     setup(
-        name='yt-dlp',
+        name='yt-dlp',  # package name (do not change/remove comment)
         version=VERSION,
         maintainer='pukkandan',
         maintainer_email='pukkandan.ytdlp@gmail.com',
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index db79df127..bdaa0d9be 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -28,6 +28,7 @@
 UPDATE_SOURCES = {
     'stable': 'yt-dlp/yt-dlp',
     'nightly': 'yt-dlp/yt-dlp-nightly-builds',
+    'master': 'yt-dlp/yt-dlp-master-builds',
 }
 REPOSITORY = UPDATE_SOURCES['stable']
 
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index 9d0096316..a4b4d4101 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -9,3 +9,7 @@
 UPDATE_HINT = None
 
 CHANNEL = 'stable'
+
+ORIGIN = 'yt-dlp/yt-dlp'
+
+_pkg_version = '2023.10.13'

From 20314dd46f25e0e0a7e985a7804049aefa8b909f Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sun, 12 Nov 2023 16:12:09 -0600
Subject: [PATCH 668/871] [core] Include build origin in verbose output

Authored by: bashonly, Grub4K

Co-authored-by: Simon Sawicki <contact@grub4k.xyz>
---
 .github/workflows/build.yml   | 46 +++++++++++++++++++++++++++++------
 .github/workflows/release.yml |  4 ++-
 yt_dlp/YoutubeDL.py           |  9 ++++---
 3 files changed, 46 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c9260eeca..c5bb76d8b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -30,6 +30,10 @@ on:
       meta_files:
         default: true
         type: boolean
+      origin:
+        required: false
+        default: ''
+        type: string
     secrets:
       GPG_SIGNING_KEY:
         required: false
@@ -37,11 +41,13 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: Version tag (YYYY.MM.DD[.REV])
+        description: |
+          VERSION: yyyy.mm.dd[.rev] or rev
         required: true
         type: string
       channel:
-        description: Update channel (stable/nightly/...)
+        description: |
+          SOURCE of this build's updates: stable/nightly/master/<repo>
         required: true
         default: stable
         type: string
@@ -73,12 +79,30 @@ on:
         description: SHA2-256SUMS, SHA2-512SUMS, _update_spec
         default: true
         type: boolean
+      origin:
+        description: .
+        required: false
+        default: ''
+        type: choice
+        options:
+        - ''
 
 permissions:
   contents: read
 
 jobs:
+  process:
+    runs-on: ubuntu-latest
+    outputs:
+      origin: ${{ steps.process_origin.outputs.origin }}
+    steps:
+      - name: Process origin
+        id: process_origin
+        run: |
+          echo "origin=${{ inputs.origin || github.repository }}" >> "$GITHUB_OUTPUT"
+
   unix:
+    needs: process
     if: inputs.unix
     runs-on: ubuntu-latest
     steps:
@@ -109,7 +133,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python devscripts/make_lazy_extractors.py
       - name: Build Unix platform-independent binary
         run: |
@@ -148,6 +172,7 @@ jobs:
             yt-dlp_linux.zip
 
   linux_arm:
+    needs: process
     if: inputs.linux_arm
     permissions:
       contents: read
@@ -183,7 +208,7 @@ jobs:
           run: |
             cd repo
             python3.8 -m pip install -U Pyinstaller -r requirements.txt  # Cached version may be out of date
-            python3.8 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+            python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
             python3.8 devscripts/make_lazy_extractors.py
             python3.8 pyinst.py
 
@@ -204,6 +229,7 @@ jobs:
             repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
 
   macos:
+    needs: process
     if: inputs.macos
     runs-on: macos-11
 
@@ -219,7 +245,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -245,6 +271,7 @@ jobs:
             dist/yt-dlp_macos.zip
 
   macos_legacy:
+    needs: process
     if: inputs.macos_legacy
     runs-on: macos-latest
 
@@ -270,7 +297,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -294,6 +321,7 @@ jobs:
             dist/yt-dlp_macos_legacy
 
   windows:
+    needs: process
     if: inputs.windows
     runs-on: windows-latest
 
@@ -309,7 +337,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -341,6 +369,7 @@ jobs:
             dist/yt-dlp_win.zip
 
   windows32:
+    needs: process
     if: inputs.windows32
     runs-on: windows-latest
 
@@ -357,7 +386,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
+          python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -385,6 +414,7 @@ jobs:
   meta_files:
     if: inputs.meta_files && always() && !cancelled()
     needs:
+      - process
       - unix
       - linux_arm
       - macos
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0e50b74e0..262c93508 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -134,7 +134,8 @@ jobs:
 
           revision="${{ (inputs.prerelease || !vars.PUSH_VERSION_COMMIT) && '$(date -u +"%H%M%S")' || '' }}"
           version="$(
-            python devscripts/update-version.py -c "${resolved_source}" ${{ inputs.version || '$revision' }} | \
+            python devscripts/update-version.py \
+            -c "${resolved_source}" -r "${{ github.repository }}" ${{ inputs.version || '$revision' }} | \
             grep -Po "version=\K\d+\.\d+\.\d+(\.\d+)?")"
 
           if [[ "${target_repo}" ]]; then
@@ -240,6 +241,7 @@ jobs:
     with:
       version: ${{ needs.prepare.outputs.version }}
       channel: ${{ needs.prepare.outputs.channel }}
+      origin: ${{ needs.prepare.outputs.target_repo }}
     permissions:
       contents: read
       packages: write # For package cache
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index fb8e89443..1fb3e4ad2 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -158,7 +158,7 @@
     clean_proxies,
     std_headers,
 )
-from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
+from .version import CHANNEL, ORIGIN, RELEASE_GIT_HEAD, VARIANT, __version__
 
 if compat_os_name == 'nt':
     import ctypes
@@ -3544,7 +3544,7 @@ def sanitize_info(info_dict, remove_private_keys=False):
             'version': __version__,
             'current_git_head': current_git_head(),
             'release_git_head': RELEASE_GIT_HEAD,
-            'repository': REPOSITORY,
+            'repository': ORIGIN,
         })
 
         if remove_private_keys:
@@ -3927,8 +3927,9 @@ def get_encoding(stream):
             source += '*'
         klass = type(self)
         write_debug(join_nonempty(
-            f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
-            f'{CHANNEL}@{__version__}',
+            f'{REPOSITORY.rpartition("/")[2]} version',
+            f'{CHANNEL.rpartition("@")[2]}@{__version__}',
+            not ORIGIN.startswith('yt-dlp/') and f'from {ORIGIN}',
             f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '',
             '' if source == 'unknown' else f'({source})',
             '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}',

From 9970d74c8383432c6c8779aa47d3253dcf412b14 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sun, 12 Nov 2023 15:58:13 -0600
Subject: [PATCH 669/871] [build] Include secretstorage in Linux builds

Authored by: bashonly
---
 .github/workflows/build.yml         | 7 ++++---
 requirements.txt                    | 7 ++++---
 yt_dlp/__pyinstaller/hook-yt_dlp.py | 2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c5bb76d8b..3e9811f0f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -122,14 +122,15 @@ jobs:
         run: |
           sudo apt -y install zip pandoc man sed
           reqs=$(mktemp)
-          cat > $reqs << EOF
+          cat > "$reqs" << EOF
           python=3.10.*
           pyinstaller
           cffi
           brotli-python
+          secretstorage
           EOF
-          sed '/^brotli.*/d' requirements.txt >> $reqs
-          mamba create -n build --file $reqs
+          sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
+          mamba create -n build --file "$reqs"
 
       - name: Prepare
         run: |
diff --git a/requirements.txt b/requirements.txt
index 112c30aeb..4d2310725 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,9 @@
 mutagen
 pycryptodomex
 websockets
-brotli; platform_python_implementation=='CPython'
-brotlicffi; platform_python_implementation!='CPython'
+brotli; implementation_name=='cpython'
+brotlicffi; implementation_name!='cpython'
 certifi
 requests>=2.31.0,<3
-urllib3>=1.26.17,<3
\ No newline at end of file
+urllib3>=1.26.17,<3
+secretstorage; sys_platform=='linux' and (implementation_name!='pypy' or implementation_version>='7.3.10')
diff --git a/yt_dlp/__pyinstaller/hook-yt_dlp.py b/yt_dlp/__pyinstaller/hook-yt_dlp.py
index c7f2c0ceb..20f037d32 100644
--- a/yt_dlp/__pyinstaller/hook-yt_dlp.py
+++ b/yt_dlp/__pyinstaller/hook-yt_dlp.py
@@ -25,7 +25,7 @@ def get_hidden_imports():
     for module in ('websockets', 'requests', 'urllib3'):
         yield from collect_submodules(module)
     # These are auto-detected, but explicitly add them just in case
-    yield from ('mutagen', 'brotli', 'certifi')
+    yield from ('mutagen', 'brotli', 'certifi', 'secretstorage')
 
 
 hiddenimports = list(get_hidden_imports())

From 5438593a35b7b042fc48fe29cad0b9039f07c9bb Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sun, 12 Nov 2023 18:30:01 -0600
Subject: [PATCH 670/871] [ci] Bump `actions/checkout` to v4

Authored by: bashonly
---
 .github/workflows/build.yml           | 12 ++++++------
 .github/workflows/codeql.yml          |  2 +-
 .github/workflows/core.yml            |  2 +-
 .github/workflows/download.yml        |  4 ++--
 .github/workflows/quick-test.yml      |  4 ++--
 .github/workflows/release-nightly.yml |  2 +-
 .github/workflows/release.yml         |  6 +++---
 7 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3e9811f0f..3b513e88e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -106,7 +106,7 @@ jobs:
     if: inputs.unix
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
@@ -186,7 +186,7 @@ jobs:
           - aarch64
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           path: ./repo
       - name: Virtualized Install, Prepare & Build
@@ -235,7 +235,7 @@ jobs:
     runs-on: macos-11
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       # NB: Building universal2 does not work with python from actions/setup-python
       - name: Install Requirements
         run: |
@@ -277,7 +277,7 @@ jobs:
     runs-on: macos-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Python
         # We need the official Python, because the GA ones only support newer macOS versions
         env:
@@ -327,7 +327,7 @@ jobs:
     runs-on: windows-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with: # 3.8 is used for Win7 support
           python-version: "3.8"
@@ -375,7 +375,7 @@ jobs:
     runs-on: windows-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390
           python-version: "3.7"
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index 2821d90d0..170a6ac19 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -29,7 +29,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 049faf373..e5a976de5 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -27,7 +27,7 @@ jobs:
           python-version: pypy-3.9
           run-tests-ext: bat
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml
index c3478721c..7302a93bc 100644
--- a/.github/workflows/download.yml
+++ b/.github/workflows/download.yml
@@ -9,7 +9,7 @@ jobs:
     if: "contains(github.event.head_commit.message, 'ci run dl')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
@@ -39,7 +39,7 @@ jobs:
           python-version: pypy-3.9
           run-tests-ext: bat
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index 930e58152..e4fd89551 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -9,7 +9,7 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'ci skip all')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python 3.11
       uses: actions/setup-python@v4
       with:
@@ -25,7 +25,7 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'ci skip all')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
     - name: Install flake8
       run: pip install flake8
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index 947eada27..2e623a67c 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -12,7 +12,7 @@ jobs:
     outputs:
       commit: ${{ steps.check_for_new_commits.outputs.commit }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Check for new commits
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 262c93508..6c59626ea 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -68,7 +68,7 @@ jobs:
       head_sha: ${{ steps.get_target.outputs.head_sha }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -256,7 +256,7 @@ jobs:
       id-token: write  # mandatory for trusted publishing
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
@@ -309,7 +309,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: actions/download-artifact@v3

From 0b6ad22e6a432006a75df968f0283e6c6b3cfae6 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sun, 12 Nov 2023 16:04:04 -0600
Subject: [PATCH 671/871] [update] Overhaul self-updater

Authored by: bashonly, Grub4K

Co-authored-by: Simon Sawicki <contact@grub4k.xyz>
---
 yt_dlp/YoutubeDL.py |   5 +-
 yt_dlp/update.py    | 525 ++++++++++++++++++++++++++++----------------
 2 files changed, 343 insertions(+), 187 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 1fb3e4ad2..740826b45 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -60,7 +60,7 @@
     get_postprocessor,
 )
 from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
-from .update import REPOSITORY, _get_system_deprecation, current_git_head, detect_variant
+from .update import REPOSITORY, _get_system_deprecation, _make_label, current_git_head, detect_variant
 from .utils import (
     DEFAULT_OUTTMPL,
     IDENTITY,
@@ -3928,8 +3928,7 @@ def get_encoding(stream):
         klass = type(self)
         write_debug(join_nonempty(
             f'{REPOSITORY.rpartition("/")[2]} version',
-            f'{CHANNEL.rpartition("@")[2]}@{__version__}',
-            not ORIGIN.startswith('yt-dlp/') and f'from {ORIGIN}',
+            _make_label(ORIGIN, CHANNEL.partition('@')[2] or __version__, __version__),
             f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '',
             '' if source == 'unknown' else f'({source})',
             '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}',
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index bdaa0d9be..85c9bb962 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import atexit
 import contextlib
 import hashlib
@@ -7,6 +9,7 @@
 import re
 import subprocess
 import sys
+from dataclasses import dataclass
 from zipimport import zipimporter
 
 from .compat import functools  # isort: split
@@ -14,16 +17,23 @@
 from .networking import Request
 from .networking.exceptions import HTTPError, network_exceptions
 from .utils import (
+    NO_DEFAULT,
     Popen,
-    cached_method,
     deprecation_warning,
+    format_field,
     remove_end,
-    remove_start,
     shell_quote,
     system_identifier,
     version_tuple,
 )
-from .version import CHANNEL, UPDATE_HINT, VARIANT, __version__
+from .version import (
+    CHANNEL,
+    ORIGIN,
+    RELEASE_GIT_HEAD,
+    UPDATE_HINT,
+    VARIANT,
+    __version__,
+)
 
 UPDATE_SOURCES = {
     'stable': 'yt-dlp/yt-dlp',
@@ -31,8 +41,11 @@
     'master': 'yt-dlp/yt-dlp-master-builds',
 }
 REPOSITORY = UPDATE_SOURCES['stable']
+_INVERSE_UPDATE_SOURCES = {value: key for key, value in UPDATE_SOURCES.items()}
 
 _VERSION_RE = re.compile(r'(\d+\.)*\d+')
+_HASH_PATTERN = r'[\da-f]{40}'
+_COMMIT_RE = re.compile(rf'Generated from: https://(?:[^/?#]+/){{3}}commit/(?P<hash>{_HASH_PATTERN})')
 
 API_BASE_URL = 'https://api.github.com/repos'
 
@@ -113,6 +126,10 @@ def is_non_updateable():
         detect_variant(), _NON_UPDATEABLE_REASONS['unknown' if VARIANT else 'other'])
 
 
+def _get_binary_name():
+    return format_field(_FILE_SUFFIXES, detect_variant(), template='yt-dlp%s', ignore=None, default=None)
+
+
 def _get_system_deprecation():
     MIN_SUPPORTED, MIN_RECOMMENDED = (3, 7), (3, 8)
 
@@ -147,73 +164,117 @@ def _sha256_file(path):
     return h.hexdigest()
 
 
+def _make_label(origin, tag, version=None):
+    if '/' in origin:
+        channel = _INVERSE_UPDATE_SOURCES.get(origin, origin)
+    else:
+        channel = origin
+    label = f'{channel}@{tag}'
+    if version and version != tag:
+        label += f' build {version}'
+    if channel != origin:
+        label += f' from {origin}'
+    return label
+
+
+@dataclass
+class UpdateInfo:
+    """
+    Update target information
+
+    Can be created by `query_update()` or manually.
+
+    Attributes:
+        tag                The release tag that will be updated to. If from query_update,
+                        the value is after API resolution and update spec processing.
+                        The only property that is required.
+        version            The actual numeric version (if available) of the binary to be updated to,
+                        after API resolution and update spec processing. (default: None)
+        requested_version  Numeric version of the binary being requested (if available),
+                        after API resolution only. (default: None)
+        commit             Commit hash (if available) of the binary to be updated to,
+                        after API resolution and update spec processing. (default: None)
+                        This value will only match the RELEASE_GIT_HEAD of prerelease builds.
+        binary_name        Filename of the binary to be updated to. (default: current binary name)
+        checksum           Expected checksum (if available) of the binary to be
+                        updated to. (default: None)
+    """
+    tag: str
+    version: str | None = None
+    requested_version: str | None = None
+    commit: str | None = None
+
+    binary_name: str | None = _get_binary_name()
+    checksum: str | None = None
+
+    _has_update = True
+
+
 class Updater:
-    _exact = True
+    # XXX: use class variables to simplify testing
+    _channel = CHANNEL
+    _origin = ORIGIN
 
-    def __init__(self, ydl, target=None):
+    def __init__(self, ydl, target: str | None = None):
         self.ydl = ydl
+        # For backwards compat, target needs to be treated as if it could be None
+        self.requested_channel, sep, self.requested_tag = (target or self._channel).rpartition('@')
+        # Check if requested_tag is actually the requested repo/channel
+        if not sep and ('/' in self.requested_tag or self.requested_tag in UPDATE_SOURCES):
+            self.requested_channel = self.requested_tag
+            self.requested_tag: str = None  # type: ignore (we set it later)
+        elif not self.requested_channel:
+            # User did not specify a channel, so we are requesting the default channel
+            self.requested_channel = self._channel.partition('@')[0]
 
-        self.target_channel, sep, self.target_tag = (target or CHANNEL).rpartition('@')
-        # stable => stable@latest
-        if not sep and ('/' in self.target_tag or self.target_tag in UPDATE_SOURCES):
-            self.target_channel = self.target_tag
-            self.target_tag = None
-        elif not self.target_channel:
-            self.target_channel = CHANNEL.partition('@')[0]
-
-        if not self.target_tag:
-            self.target_tag = 'latest'
+        # --update should not be treated as an exact tag request even if CHANNEL has a @tag
+        self._exact = bool(target) and target != self._channel
+        if not self.requested_tag:
+            # User did not specify a tag, so we request 'latest' and track that no exact tag was passed
+            self.requested_tag = 'latest'
             self._exact = False
-        elif self.target_tag != 'latest':
-            self.target_tag = f'tags/{self.target_tag}'
 
-        if '/' in self.target_channel:
-            self._target_repo = self.target_channel
-            if self.target_channel not in (CHANNEL, *UPDATE_SOURCES.values()):
+        if '/' in self.requested_channel:
+            # requested_channel is actually a repository
+            self.requested_repo = self.requested_channel
+            if not self.requested_repo.startswith('yt-dlp/') and self.requested_repo != self._origin:
                 self.ydl.report_warning(
                     f'You are switching to an {self.ydl._format_err("unofficial", "red")} executable '
-                    f'from {self.ydl._format_err(self._target_repo, self.ydl.Styles.EMPHASIS)}. '
+                    f'from {self.ydl._format_err(self.requested_repo, self.ydl.Styles.EMPHASIS)}. '
                     f'Run {self.ydl._format_err("at your own risk", "light red")}')
                 self._block_restart('Automatically restarting into custom builds is disabled for security reasons')
         else:
-            self._target_repo = UPDATE_SOURCES.get(self.target_channel)
-            if not self._target_repo:
+            # Check if requested_channel resolves to a known repository or else raise
+            self.requested_repo = UPDATE_SOURCES.get(self.requested_channel)
+            if not self.requested_repo:
                 self._report_error(
-                    f'Invalid update channel {self.target_channel!r} requested. '
+                    f'Invalid update channel {self.requested_channel!r} requested. '
                     f'Valid channels are {", ".join(UPDATE_SOURCES)}', True)
 
-    def _version_compare(self, a, b, channel=CHANNEL):
-        if self._exact and channel != self.target_channel:
-            return False
+        self._identifier = f'{detect_variant()} {system_identifier()}'
 
-        if _VERSION_RE.fullmatch(f'{a}.{b}'):
-            a, b = version_tuple(a), version_tuple(b)
-            return a == b if self._exact else a >= b
-        return a == b
+    @property
+    def current_version(self):
+        """Current version"""
+        return __version__
 
-    @functools.cached_property
-    def _tag(self):
-        if self._version_compare(self.current_version, self.latest_version):
-            return self.target_tag
+    @property
+    def current_commit(self):
+        """Current commit hash"""
+        return RELEASE_GIT_HEAD
 
-        identifier = f'{detect_variant()} {self.target_channel} {system_identifier()}'
-        for line in self._download('_update_spec', 'latest').decode().splitlines():
-            if not line.startswith('lock '):
-                continue
-            _, tag, pattern = line.split(' ', 2)
-            if re.match(pattern, identifier):
-                if not self._exact:
-                    return f'tags/{tag}'
-                elif self.target_tag == 'latest' or not self._version_compare(
-                        tag, self.target_tag[5:], channel=self.target_channel):
-                    self._report_error(
-                        f'yt-dlp cannot be updated above {tag} since you are on an older Python version', True)
-                    return f'tags/{self.current_version}'
-        return self.target_tag
+    def _download_asset(self, name, tag=None):
+        if not tag:
+            tag = self.requested_tag
 
-    @cached_method
-    def _get_version_info(self, tag):
-        url = f'{API_BASE_URL}/{self._target_repo}/releases/{tag}'
+        path = 'latest/download' if tag == 'latest' else f'download/{tag}'
+        url = f'https://github.com/{self.requested_repo}/releases/{path}/{name}'
+        self.ydl.write_debug(f'Downloading {name} from {url}')
+        return self.ydl.urlopen(url).read()
+
+    def _call_api(self, tag):
+        tag = f'tags/{tag}' if tag != 'latest' else tag
+        url = f'{API_BASE_URL}/{self.requested_repo}/releases/{tag}'
         self.ydl.write_debug(f'Fetching release info: {url}')
         return json.loads(self.ydl.urlopen(Request(url, headers={
             'Accept': 'application/vnd.github+json',
@@ -221,105 +282,175 @@ def _get_version_info(self, tag):
             'X-GitHub-Api-Version': '2022-11-28',
         })).read().decode())
 
-    @property
-    def current_version(self):
-        """Current version"""
-        return __version__
+    def _get_version_info(self, tag: str) -> tuple[str | None, str | None]:
+        if _VERSION_RE.fullmatch(tag):
+            return tag, None
 
-    @staticmethod
-    def _label(channel, tag):
-        """Label for a given channel and tag"""
-        return f'{channel}@{remove_start(tag, "tags/")}'
+        api_info = self._call_api(tag)
 
-    def _get_actual_tag(self, tag):
-        if tag.startswith('tags/'):
-            return tag[5:]
-        return self._get_version_info(tag)['tag_name']
+        if tag == 'latest':
+            requested_version = api_info['tag_name']
+        else:
+            match = re.search(rf'\s+(?P<version>{_VERSION_RE.pattern})$', api_info.get('name', ''))
+            requested_version = match.group('version') if match else None
 
-    @property
-    def new_version(self):
-        """Version of the latest release we can update to"""
-        return self._get_actual_tag(self._tag)
+        if re.fullmatch(_HASH_PATTERN, api_info.get('target_commitish', '')):
+            target_commitish = api_info['target_commitish']
+        else:
+            match = _COMMIT_RE.match(api_info.get('body', ''))
+            target_commitish = match.group('hash') if match else None
 
-    @property
-    def latest_version(self):
-        """Version of the target release"""
-        return self._get_actual_tag(self.target_tag)
+        if not (requested_version or target_commitish):
+            self._report_error('One of either version or commit hash must be available on the release', expected=True)
 
-    @property
-    def has_update(self):
-        """Whether there is an update available"""
-        return not self._version_compare(self.current_version, self.new_version)
+        return requested_version, target_commitish
 
-    @functools.cached_property
-    def filename(self):
-        """Filename of the executable"""
-        return compat_realpath(_get_variant_and_executable_path()[1])
+    def _download_update_spec(self, source_tags):
+        for tag in source_tags:
+            try:
+                return self._download_asset('_update_spec', tag=tag).decode()
+            except network_exceptions as error:
+                if isinstance(error, HTTPError) and error.status == 404:
+                    continue
+                self._report_network_error(f'fetch update spec: {error}')
 
-    def _download(self, name, tag):
-        slug = 'latest/download' if tag == 'latest' else f'download/{tag[5:]}'
-        url = f'https://github.com/{self._target_repo}/releases/{slug}/{name}'
-        self.ydl.write_debug(f'Downloading {name} from {url}')
-        return self.ydl.urlopen(url).read()
-
-    @functools.cached_property
-    def release_name(self):
-        """The release filename"""
-        return f'yt-dlp{_FILE_SUFFIXES[detect_variant()]}'
-
-    @functools.cached_property
-    def release_hash(self):
-        """Hash of the latest release"""
-        hash_data = dict(ln.split()[::-1] for ln in self._download('SHA2-256SUMS', self._tag).decode().splitlines())
-        return hash_data[self.release_name]
-
-    def _report_error(self, msg, expected=False):
-        self.ydl.report_error(msg, tb=False if expected else None)
-        self.ydl._download_retcode = 100
-
-    def _report_permission_error(self, file):
-        self._report_error(f'Unable to write to {file}; Try running as administrator', True)
-
-    def _report_network_error(self, action, delim=';'):
         self._report_error(
-            f'Unable to {action}{delim} visit  '
-            f'https://github.com/{self._target_repo}/releases/{self.target_tag.replace("tags/", "tag/")}', True)
+            f'The requested tag {self.requested_tag} does not exist for {self.requested_repo}', True)
+        return None
+
+    def _process_update_spec(self, lockfile: str, resolved_tag: str):
+        lines = lockfile.splitlines()
+        is_version2 = any(line.startswith('lockV2 ') for line in lines)
+
+        for line in lines:
+            if is_version2:
+                if not line.startswith(f'lockV2 {self.requested_repo} '):
+                    continue
+                _, _, tag, pattern = line.split(' ', 3)
+            else:
+                if not line.startswith('lock '):
+                    continue
+                _, tag, pattern = line.split(' ', 2)
+
+            if re.match(pattern, self._identifier):
+                if _VERSION_RE.fullmatch(tag):
+                    if not self._exact:
+                        return tag
+                    elif self._version_compare(tag, resolved_tag):
+                        return resolved_tag
+                elif tag != resolved_tag:
+                    continue
+
+                self._report_error(
+                    f'yt-dlp cannot be updated to {resolved_tag} since you are on an older Python version', True)
+                return None
+
+        return resolved_tag
+
+    def _version_compare(self, a: str, b: str):
+        """
+        Compare two version strings
+
+        This function SHOULD NOT be called if self._exact == True
+        """
+        if _VERSION_RE.fullmatch(f'{a}.{b}'):
+            return version_tuple(a) >= version_tuple(b)
+        return a == b
+
+    def query_update(self, *, _output=False) -> UpdateInfo | None:
+        """Fetches and returns info about the available update"""
+        if not self.requested_repo:
+            self._report_error('No target repository could be determined from input')
+            return None
 
-    def check_update(self):
-        """Report whether there is an update available"""
-        if not self._target_repo:
-            return False
         try:
-            self.ydl.to_screen((
-                f'Available version: {self._label(self.target_channel, self.latest_version)}, ' if self.target_tag == 'latest' else ''
-            ) + f'Current version: {self._label(CHANNEL, self.current_version)}')
+            requested_version, target_commitish = self._get_version_info(self.requested_tag)
         except network_exceptions as e:
-            return self._report_network_error(f'obtain version info ({e})', delim='; Please try again later or')
+            self._report_network_error(f'obtain version info ({e})', delim='; Please try again later or')
+            return None
 
+        if self._exact and self._origin != self.requested_repo:
+            has_update = True
+        elif requested_version:
+            if self._exact:
+                has_update = self.current_version != requested_version
+            else:
+                has_update = not self._version_compare(self.current_version, requested_version)
+        elif target_commitish:
+            has_update = target_commitish != self.current_commit
+        else:
+            has_update = False
+
+        resolved_tag = requested_version if self.requested_tag == 'latest' else self.requested_tag
+        current_label = _make_label(self._origin, self._channel.partition("@")[2] or self.current_version, self.current_version)
+        requested_label = _make_label(self.requested_repo, resolved_tag, requested_version)
+        latest_or_requested = f'{"Latest" if self.requested_tag == "latest" else "Requested"} version: {requested_label}'
+        if not has_update:
+            if _output:
+                self.ydl.to_screen(f'{latest_or_requested}\nyt-dlp is up to date ({current_label})')
+            return None
+
+        update_spec = self._download_update_spec(('latest', None) if requested_version else (None,))
+        if not update_spec:
+            return None
+        # `result_` prefixed vars == post-_process_update_spec() values
+        result_tag = self._process_update_spec(update_spec, resolved_tag)
+        if not result_tag or result_tag == self.current_version:
+            return None
+        elif result_tag == resolved_tag:
+            result_version = requested_version
+        elif _VERSION_RE.fullmatch(result_tag):
+            result_version = result_tag
+        else:  # actual version being updated to is unknown
+            result_version = None
+
+        checksum = None
+        # Non-updateable variants can get update_info but need to skip checksum
         if not is_non_updateable():
-            self.ydl.to_screen(f'Current Build Hash: {_sha256_file(self.filename)}')
+            try:
+                hashes = self._download_asset('SHA2-256SUMS', result_tag)
+            except network_exceptions as error:
+                if not isinstance(error, HTTPError) or error.status != 404:
+                    self._report_network_error(f'fetch checksums: {error}')
+                    return None
+                self.ydl.report_warning('No hash information found for the release, skipping verification')
+            else:
+                for ln in hashes.decode().splitlines():
+                    if ln.endswith(_get_binary_name()):
+                        checksum = ln.split()[0]
+                        break
+                if not checksum:
+                    self.ydl.report_warning('The hash could not be found in the checksum file, skipping verification')
 
-        if self.has_update:
-            return True
+        if _output:
+            update_label = _make_label(self.requested_repo, result_tag, result_version)
+            self.ydl.to_screen(
+                f'Current version: {current_label}\n{latest_or_requested}'
+                + (f'\nUpgradable to: {update_label}' if update_label != requested_label else ''))
 
-        if self.target_tag == self._tag:
-            self.ydl.to_screen(f'yt-dlp is up to date ({self._label(CHANNEL, self.current_version)})')
-        elif not self._exact:
-            self.ydl.report_warning('yt-dlp cannot be updated any further since you are on an older Python version')
-        return False
+        return UpdateInfo(
+            tag=result_tag,
+            version=result_version,
+            requested_version=requested_version,
+            commit=target_commitish if result_tag == resolved_tag else None,
+            checksum=checksum)
 
-    def update(self):
+    def update(self, update_info=NO_DEFAULT):
         """Update yt-dlp executable to the latest version"""
-        if not self.check_update():
-            return
+        if update_info is NO_DEFAULT:
+            update_info = self.query_update(_output=True)
+        if not update_info:
+            return False
+
         err = is_non_updateable()
         if err:
-            return self._report_error(err, True)
-        self.ydl.to_screen(f'Updating to {self._label(self.target_channel, self.new_version)} ...')
-        if (_VERSION_RE.fullmatch(self.target_tag[5:])
-                and version_tuple(self.target_tag[5:]) < (2023, 3, 2)):
-            self.ydl.report_warning('You are downgrading to a version without --update-to')
-            self._block_restart('Cannot automatically restart to a version without --update-to')
+            self._report_error(err, True)
+            return False
+
+        self.ydl.to_screen(f'Current Build Hash: {_sha256_file(self.filename)}')
+
+        update_label = _make_label(self.requested_repo, update_info.tag, update_info.version)
+        self.ydl.to_screen(f'Updating to {update_label} ...')
 
         directory = os.path.dirname(self.filename)
         if not os.access(self.filename, os.W_OK):
@@ -338,20 +469,17 @@ def update(self):
             return self._report_error('Unable to remove the old version')
 
         try:
-            newcontent = self._download(self.release_name, self._tag)
+            newcontent = self._download_asset(update_info.binary_name, update_info.tag)
         except network_exceptions as e:
             if isinstance(e, HTTPError) and e.status == 404:
                 return self._report_error(
-                    f'The requested tag {self._label(self.target_channel, self.target_tag)} does not exist', True)
-            return self._report_network_error(f'fetch updates: {e}')
+                    f'The requested tag {self.requested_repo}@{update_info.tag} does not exist', True)
+            return self._report_network_error(f'fetch updates: {e}', tag=update_info.tag)
 
-        try:
-            expected_hash = self.release_hash
-        except Exception:
-            self.ydl.report_warning('no hash information found for the release')
-        else:
-            if hashlib.sha256(newcontent).hexdigest() != expected_hash:
-                return self._report_network_error('verify the new executable')
+        if not update_info.checksum:
+            self._block_restart('Automatically restarting into unverified builds is disabled for security reasons')
+        elif hashlib.sha256(newcontent).hexdigest() != update_info.checksum:
+            return self._report_network_error('verify the new executable', tag=update_info.tag)
 
         try:
             with open(new_filename, 'wb') as outf:
@@ -388,9 +516,14 @@ def update(self):
                 return self._report_error(
                     f'Unable to set permissions. Run: sudo chmod a+rx {compat_shlex_quote(self.filename)}')
 
-        self.ydl.to_screen(f'Updated yt-dlp to {self._label(self.target_channel, self.new_version)}')
+        self.ydl.to_screen(f'Updated yt-dlp to {update_label}')
         return True
 
+    @functools.cached_property
+    def filename(self):
+        """Filename of the executable"""
+        return compat_realpath(_get_variant_and_executable_path()[1])
+
     @functools.cached_property
     def cmd(self):
         """The command-line to run the executable, if known"""
@@ -413,6 +546,71 @@ def wrapper():
             return self.ydl._download_retcode
         self.restart = wrapper
 
+    def _report_error(self, msg, expected=False):
+        self.ydl.report_error(msg, tb=False if expected else None)
+        self.ydl._download_retcode = 100
+
+    def _report_permission_error(self, file):
+        self._report_error(f'Unable to write to {file}; try running as administrator', True)
+
+    def _report_network_error(self, action, delim=';', tag=None):
+        if not tag:
+            tag = self.requested_tag
+        self._report_error(
+            f'Unable to {action}{delim} visit  https://github.com/{self.requested_repo}/releases/'
+            + tag if tag == "latest" else f"tag/{tag}", True)
+
+    # XXX: Everything below this line in this class is deprecated / for compat only
+    @property
+    def _target_tag(self):
+        """Deprecated; requested tag with 'tags/' prepended when necessary for API calls"""
+        return f'tags/{self.requested_tag}' if self.requested_tag != 'latest' else self.requested_tag
+
+    def _check_update(self):
+        """Deprecated; report whether there is an update available"""
+        return bool(self.query_update(_output=True))
+
+    def __getattr__(self, attribute: str):
+        """Compat getter function for deprecated attributes"""
+        deprecated_props_map = {
+            'check_update': '_check_update',
+            'target_tag': '_target_tag',
+            'target_channel': 'requested_channel',
+        }
+        update_info_props_map = {
+            'has_update': '_has_update',
+            'new_version': 'version',
+            'latest_version': 'requested_version',
+            'release_name': 'binary_name',
+            'release_hash': 'checksum',
+        }
+
+        if attribute not in deprecated_props_map and attribute not in update_info_props_map:
+            raise AttributeError(f'{type(self).__name__!r} object has no attribute {attribute!r}')
+
+        msg = f'{type(self).__name__}.{attribute} is deprecated and will be removed in a future version'
+        if attribute in deprecated_props_map:
+            source_name = deprecated_props_map[attribute]
+            if not source_name.startswith('_'):
+                msg += f'. Please use {source_name!r} instead'
+            source = self
+            mapping = deprecated_props_map
+
+        else:  # attribute in update_info_props_map
+            msg += '. Please call query_update() instead'
+            source = self.query_update()
+            if source is None:
+                source = UpdateInfo('', None, None, None)
+                source._has_update = False
+            mapping = update_info_props_map
+
+        deprecation_warning(msg)
+        for target_name, source_name in mapping.items():
+            value = getattr(source, source_name)
+            setattr(self, target_name, value)
+
+        return getattr(self, attribute)
+
 
 def run_update(ydl):
     """Update the program file with the latest version from the repository
@@ -421,45 +619,4 @@ def run_update(ydl):
     return Updater(ydl).update()
 
 
-# Deprecated
-def update_self(to_screen, verbose, opener):
-    import traceback
-
-    deprecation_warning(f'"{__name__}.update_self" is deprecated and may be removed '
-                        f'in a future version. Use "{__name__}.run_update(ydl)" instead')
-
-    printfn = to_screen
-
-    class FakeYDL():
-        to_screen = printfn
-
-        def report_warning(self, msg, *args, **kwargs):
-            return printfn(f'WARNING: {msg}', *args, **kwargs)
-
-        def report_error(self, msg, tb=None):
-            printfn(f'ERROR: {msg}')
-            if not verbose:
-                return
-            if tb is None:
-                # Copied from YoutubeDL.trouble
-                if sys.exc_info()[0]:
-                    tb = ''
-                    if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
-                        tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
-                    tb += traceback.format_exc()
-                else:
-                    tb_data = traceback.format_list(traceback.extract_stack())
-                    tb = ''.join(tb_data)
-            if tb:
-                printfn(tb)
-
-        def write_debug(self, msg, *args, **kwargs):
-            printfn(f'[debug] {msg}', *args, **kwargs)
-
-        def urlopen(self, url):
-            return opener.open(url)
-
-    return run_update(FakeYDL())
-
-
 __all__ = ['Updater']

From a00af29853b8c7350ce086f4cab8c2c9cf2fcf1d Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sun, 12 Nov 2023 18:02:01 -0600
Subject: [PATCH 672/871] [cleanup] Update documentation for master and nightly
 channels

Authored by: bashonly, Grub4K

Co-authored-by: Simon Sawicki <contact@grub4k.xyz>
---
 .github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml |  2 +-
 .../2_site_support_request.yml                |  2 +-
 .../3_site_feature_request.yml                |  2 +-
 .github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml  |  2 +-
 .../ISSUE_TEMPLATE_tmpl/5_feature_request.yml |  2 +-
 .github/ISSUE_TEMPLATE_tmpl/6_question.yml    |  2 +-
 README.md                                     | 27 +++++++++++++------
 devscripts/make_issue_template.py             | 18 ++++++-------
 8 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
index a51db789f..bff28ae4e 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml
@@ -12,7 +12,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml
index 75d62e7bb..2bffe738d 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml
@@ -12,7 +12,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml
index 18b30f578..6c3127983 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml
@@ -12,7 +12,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml
index 9ab490267..5f357d96e 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml
@@ -12,7 +12,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml
index ef3bb2269..99107ff58 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml
@@ -14,7 +14,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
diff --git a/.github/ISSUE_TEMPLATE_tmpl/6_question.yml b/.github/ISSUE_TEMPLATE_tmpl/6_question.yml
index 4bef82d5a..bd742109a 100644
--- a/.github/ISSUE_TEMPLATE_tmpl/6_question.yml
+++ b/.github/ISSUE_TEMPLATE_tmpl/6_question.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
diff --git a/README.md b/README.md
index 52f8bf799..1b92c64d6 100644
--- a/README.md
+++ b/README.md
@@ -121,7 +121,7 @@ # NEW FEATURES
 
 * **Self updater**: The releases can be updated using `yt-dlp -U`, and downgraded using `--update-to` if required
 
-* **Nightly builds**: [Automated nightly builds](#update-channels) can be used with `--update-to nightly`
+* **Automated builds**: [Nightly/master builds](#update-channels) can be used with `--update-to nightly` and `--update-to master`
 
 See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/commits) for the full list of changes
 
@@ -193,9 +193,11 @@ ## UPDATE
 
 <a id="update-channels"/>
 
-There are currently two release channels for binaries, `stable` and `nightly`.
-`stable` is the default channel, and many of its changes have been tested by users of the nightly channel.
-The `nightly` channel has releases built after each push to the master branch, and will have the most recent fixes and additions, but also have more risk of regressions. They are available in [their own repo](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases).
+There are currently three release channels for binaries: `stable`, `nightly` and `master`.
+
+* `stable` is the default channel, and many of its changes have been tested by users of the `nightly` and `master` channels.
+* The `nightly` channel has releases scheduled to build every day around midnight UTC, for a snapshot of the project's new patches and changes. This is the **recommended channel for regular users** of yt-dlp. The `nightly` releases are available from [yt-dlp/yt-dlp-nightly-builds](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases) or as development releases of the `yt-dlp` PyPI package (which can be installed with pip's `--pre` flag).
+* The `master` channel features releases that are built after each push to the master branch, and these will have the very latest fixes and additions, but may also be more prone to regressions. They are available from [yt-dlp/yt-dlp-master-builds](https://github.com/yt-dlp/yt-dlp-master-builds/releases).
 
 When using `--update`/`-U`, a release binary will only update to its current channel.
 `--update-to CHANNEL` can be used to switch to a different channel when a newer version is available. `--update-to [CHANNEL@]TAG` can also be used to upgrade or downgrade to specific tags from a channel.
@@ -203,10 +205,19 @@ ## UPDATE
 You may also use `--update-to <repository>` (`<owner>/<repository>`) to update to a channel on a completely different repository. Be careful with what repository you are updating to though, there is no verification done for binaries from different repositories.
 
 Example usage:
-* `yt-dlp --update-to nightly` change to `nightly` channel and update to its latest release
-* `yt-dlp --update-to stable@2023.02.17` upgrade/downgrade to release to `stable` channel tag `2023.02.17`
-* `yt-dlp --update-to 2023.01.06` upgrade/downgrade to tag `2023.01.06` if it exists on the current channel
-* `yt-dlp --update-to example/yt-dlp@2023.03.01` upgrade/downgrade to the release from the `example/yt-dlp` repository, tag `2023.03.01`
+* `yt-dlp --update-to master` switch to the `master` channel and update to its latest release
+* `yt-dlp --update-to stable@2023.07.06` upgrade/downgrade to release to `stable` channel tag `2023.07.06`
+* `yt-dlp --update-to 2023.10.07` upgrade/downgrade to tag `2023.10.07` if it exists on the current channel
+* `yt-dlp --update-to example/yt-dlp@2023.09.24` upgrade/downgrade to the release from the `example/yt-dlp` repository, tag `2023.09.24`
+
+**Important**: Any user experiencing an issue with the `stable` release should install or update to the `nightly` release before submitting a bug report:
+```
+# To update to nightly from stable executable/binary:
+yt-dlp --update-to nightly
+
+# To install nightly with pip:
+python -m pip install -U --pre yt-dlp
+```
 
 <!-- MANPAGE: BEGIN EXCLUDED SECTION -->
 ## RELEASE FILES
diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index 39b95c8da..6c85e200f 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -12,7 +12,6 @@
 from devscripts.utils import (
     get_filename_args,
     read_file,
-    read_version,
     write_file,
 )
 
@@ -35,19 +34,18 @@
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
-        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version %(version)s [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: %(version)s, Current version: %(version)s
-        yt-dlp is up to date (%(version)s)
+        [debug] Request Handlers: urllib, requests
+        [debug] Loaded 1893 extractors
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:
@@ -66,7 +64,7 @@
 
 
 def main():
-    fields = {'version': read_version(), 'no_skip': NO_SKIP}
+    fields = {'no_skip': NO_SKIP}
     fields['verbose'] = VERBOSE_TMPL % fields
     fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose'])
 

From 87264d4fdadcddd91289b968dd0e4bf58d449267 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Sun, 12 Nov 2023 18:30:55 -0600
Subject: [PATCH 673/871] [test:update] Implement simple updater unit tests

Authored by: bashonly
---
 test/test_update.py          | 199 +++++++++++++++++++++++++++++++++++
 test/test_update.py.disabled |  30 ------
 test/versions.json           |  34 ------
 3 files changed, 199 insertions(+), 64 deletions(-)
 create mode 100644 test/test_update.py
 delete mode 100644 test/test_update.py.disabled
 delete mode 100644 test/versions.json

diff --git a/test/test_update.py b/test/test_update.py
new file mode 100644
index 000000000..134424a31
--- /dev/null
+++ b/test/test_update.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+
+# Allow direct execution
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from test.helper import FakeYDL, report_warning
+from yt_dlp.update import Updater, UpdateInfo
+
+TEST_API_DATA = {
+    'yt-dlp/yt-dlp/latest': {
+        'tag_name': '2023.12.31',
+        'target_commitish': 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
+        'name': 'yt-dlp 2023.12.31',
+        'body': 'BODY',
+    },
+    'yt-dlp/yt-dlp-nightly-builds/latest': {
+        'tag_name': '2023.12.31.123456',
+        'target_commitish': 'master',
+        'name': 'yt-dlp nightly 2023.12.31.123456',
+        'body': 'Generated from: https://github.com/yt-dlp/yt-dlp/commit/cccccccccccccccccccccccccccccccccccccccc',
+    },
+    'yt-dlp/yt-dlp-master-builds/latest': {
+        'tag_name': '2023.12.31.987654',
+        'target_commitish': 'master',
+        'name': 'yt-dlp master 2023.12.31.987654',
+        'body': 'Generated from: https://github.com/yt-dlp/yt-dlp/commit/dddddddddddddddddddddddddddddddddddddddd',
+    },
+    'yt-dlp/yt-dlp/tags/testing': {
+        'tag_name': 'testing',
+        'target_commitish': '9999999999999999999999999999999999999999',
+        'name': 'testing',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp/latest': {
+        'tag_name': '2050.12.31',
+        'target_commitish': 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
+        'name': '2050.12.31',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp/tags/pr0000': {
+        'tag_name': 'pr0000',
+        'target_commitish': 'ffffffffffffffffffffffffffffffffffffffff',
+        'name': 'pr1234 2023.11.11.000000',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp/tags/pr1234': {
+        'tag_name': 'pr1234',
+        'target_commitish': '0000000000000000000000000000000000000000',
+        'name': 'pr1234 2023.12.31.555555',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp/tags/pr9999': {
+        'tag_name': 'pr9999',
+        'target_commitish': '1111111111111111111111111111111111111111',
+        'name': 'pr9999',
+        'body': 'BODY',
+    },
+    'fork/yt-dlp-satellite/tags/pr987': {
+        'tag_name': 'pr987',
+        'target_commitish': 'master',
+        'name': 'pr987',
+        'body': 'Generated from: https://github.com/yt-dlp/yt-dlp/commit/2222222222222222222222222222222222222222',
+    },
+}
+
+TEST_LOCKFILE_V1 = '''# This file is used for regulating self-update
+lock 2022.08.18.36 .+ Python 3.6
+lock 2023.11.13 .+ Python 3.7
+'''
+
+TEST_LOCKFILE_V2 = '''# This file is used for regulating self-update
+lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3.6
+lockV2 yt-dlp/yt-dlp 2023.11.13 .+ Python 3.7
+'''
+
+TEST_LOCKFILE_V1_V2 = '''# This file is used for regulating self-update
+lock 2022.08.18.36 .+ Python 3.6
+lock 2023.11.13 .+ Python 3.7
+lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3.6
+lockV2 yt-dlp/yt-dlp 2023.11.13 .+ Python 3.7
+lockV2 fork/yt-dlp pr0000 .+ Python 3.6
+lockV2 fork/yt-dlp pr1234 .+ Python 3.7
+lockV2 fork/yt-dlp pr9999 .+ Python 3.11
+'''
+
+
+class FakeUpdater(Updater):
+    current_version = '2022.01.01'
+    current_commit = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+
+    _channel = 'stable'
+    _origin = 'yt-dlp/yt-dlp'
+
+    def _download_update_spec(self, *args, **kwargs):
+        return TEST_LOCKFILE_V1_V2
+
+    def _call_api(self, tag):
+        tag = f'tags/{tag}' if tag != 'latest' else tag
+        return TEST_API_DATA[f'{self.requested_repo}/{tag}']
+
+    def _report_error(self, msg, *args, **kwargs):
+        report_warning(msg)
+
+
+class TestUpdate(unittest.TestCase):
+    maxDiff = None
+
+    def test_update_spec(self):
+        ydl = FakeYDL()
+        updater = FakeUpdater(ydl, 'stable@latest')
+
+        def test(lockfile, identifier, input_tag, expect_tag, exact=False, repo='yt-dlp/yt-dlp'):
+            updater._identifier = identifier
+            updater._exact = exact
+            updater.requested_repo = repo
+            result = updater._process_update_spec(lockfile, input_tag)
+            self.assertEqual(
+                result, expect_tag,
+                f'{identifier!r} requesting {repo}@{input_tag} (exact={exact}) '
+                f'returned {result!r} instead of {expect_tag!r}')
+
+        test(TEST_LOCKFILE_V1, 'zip Python 3.11.0', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1, 'zip stable Python 3.11.0', '2023.11.13', '2023.11.13', exact=True)
+        test(TEST_LOCKFILE_V1, 'zip Python 3.6.0', '2023.11.13', '2022.08.18.36')
+        test(TEST_LOCKFILE_V1, 'zip stable Python 3.6.0', '2023.11.13', None, exact=True)
+        test(TEST_LOCKFILE_V1, 'zip Python 3.7.0', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1, 'zip stable Python 3.7.1', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1, 'zip Python 3.7.1', '2023.12.31', '2023.11.13')
+        test(TEST_LOCKFILE_V1, 'zip stable Python 3.7.1', '2023.12.31', '2023.11.13')
+
+        test(TEST_LOCKFILE_V2, 'zip Python 3.11.1', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V2, 'zip stable Python 3.11.1', '2023.12.31', '2023.12.31')
+        test(TEST_LOCKFILE_V2, 'zip Python 3.6.1', '2023.11.13', '2022.08.18.36')
+        test(TEST_LOCKFILE_V2, 'zip stable Python 3.7.2', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V2, 'zip Python 3.7.2', '2023.12.31', '2023.11.13')
+
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.2', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.11.2', '2023.12.31', '2023.12.31')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.2', '2023.11.13', '2022.08.18.36')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.7.3', '2023.11.13', '2023.11.13')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.3', '2023.12.31', '2023.11.13')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.3', 'pr0000', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.7.4', 'pr0000', 'pr0000', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.4', 'pr0000', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.4', 'pr1234', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.8.1', 'pr1234', 'pr1234', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.5', 'pr1234', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.3', 'pr9999', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.12.0', 'pr9999', 'pr9999', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.4', 'pr9999', None, repo='fork/yt-dlp')
+
+    def test_query_update(self):
+        ydl = FakeYDL()
+
+        def test(target, expected, current_version=None, current_commit=None, identifier=None):
+            updater = FakeUpdater(ydl, target)
+            if current_version:
+                updater.current_version = current_version
+            if current_commit:
+                updater.current_commit = current_commit
+            updater._identifier = identifier or 'zip'
+            update_info = updater.query_update(_output=True)
+            self.assertDictEqual(
+                update_info.__dict__ if update_info else {}, expected.__dict__ if expected else {})
+
+        test('yt-dlp/yt-dlp@latest', UpdateInfo(
+            '2023.12.31', version='2023.12.31', requested_version='2023.12.31', commit='b' * 40))
+        test('yt-dlp/yt-dlp-nightly-builds@latest', UpdateInfo(
+            '2023.12.31.123456', version='2023.12.31.123456', requested_version='2023.12.31.123456', commit='c' * 40))
+        test('yt-dlp/yt-dlp-master-builds@latest', UpdateInfo(
+            '2023.12.31.987654', version='2023.12.31.987654', requested_version='2023.12.31.987654', commit='d' * 40))
+        test('fork/yt-dlp@latest', UpdateInfo(
+            '2050.12.31', version='2050.12.31', requested_version='2050.12.31', commit='e' * 40))
+        test('fork/yt-dlp@pr0000', UpdateInfo(
+            'pr0000', version='2023.11.11.000000', requested_version='2023.11.11.000000', commit='f' * 40))
+        test('fork/yt-dlp@pr1234', UpdateInfo(
+            'pr1234', version='2023.12.31.555555', requested_version='2023.12.31.555555', commit='0' * 40))
+        test('fork/yt-dlp@pr9999', UpdateInfo(
+            'pr9999', version=None, requested_version=None, commit='1' * 40))
+        test('fork/yt-dlp-satellite@pr987', UpdateInfo(
+            'pr987', version=None, requested_version=None, commit='2' * 40))
+        test('yt-dlp/yt-dlp', None, current_version='2024.01.01')
+        test('stable', UpdateInfo(
+            '2023.12.31', version='2023.12.31', requested_version='2023.12.31', commit='b' * 40))
+        test('nightly', UpdateInfo(
+            '2023.12.31.123456', version='2023.12.31.123456', requested_version='2023.12.31.123456', commit='c' * 40))
+        test('master', UpdateInfo(
+            '2023.12.31.987654', version='2023.12.31.987654', requested_version='2023.12.31.987654', commit='d' * 40))
+        test('testing', None, current_commit='9' * 40)
+        test('testing', UpdateInfo('testing', commit='9' * 40))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/test_update.py.disabled b/test/test_update.py.disabled
deleted file mode 100644
index 85ac86692..000000000
--- a/test/test_update.py.disabled
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env python3
-
-# Allow direct execution
-import os
-import sys
-import unittest
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-
-import json
-
-from yt_dlp.update import rsa_verify
-
-
-class TestUpdate(unittest.TestCase):
-    def test_rsa_verify(self):
-        UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
-        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'versions.json'), 'rb') as f:
-            versions_info = f.read().decode()
-        versions_info = json.loads(versions_info)
-        signature = versions_info['signature']
-        del versions_info['signature']
-        self.assertTrue(rsa_verify(
-            json.dumps(versions_info, sort_keys=True).encode(),
-            signature, UPDATES_RSA_KEY))
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/test/versions.json b/test/versions.json
deleted file mode 100644
index 6cccc2259..000000000
--- a/test/versions.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
-    "latest": "2013.01.06", 
-    "signature": "72158cdba391628569ffdbea259afbcf279bbe3d8aeb7492690735dc1cfa6afa754f55c61196f3871d429599ab22f2667f1fec98865527b32632e7f4b3675a7ef0f0fbe084d359256ae4bba68f0d33854e531a70754712f244be71d4b92e664302aa99653ee4df19800d955b6c4149cd2b3f24288d6e4b40b16126e01f4c8ce6", 
-    "versions": {
-        "2013.01.02": {
-            "bin": [
-                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl", 
-                "f5b502f8aaa77675c4884938b1e4871ebca2611813a0c0e74f60c0fbd6dcca6b"
-            ], 
-            "exe": [
-                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl.exe", 
-                "75fa89d2ce297d102ff27675aa9d92545bbc91013f52ec52868c069f4f9f0422"
-            ], 
-            "tar": [
-                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl-2013.01.02.tar.gz", 
-                "6a66d022ac8e1c13da284036288a133ec8dba003b7bd3a5179d0c0daca8c8196"
-            ]
-        }, 
-        "2013.01.06": {
-            "bin": [
-                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl", 
-                "64b6ed8865735c6302e836d4d832577321b4519aa02640dc508580c1ee824049"
-            ], 
-            "exe": [
-                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl.exe", 
-                "58609baf91e4389d36e3ba586e21dab882daaaee537e4448b1265392ae86ff84"
-            ], 
-            "tar": [
-                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl-2013.01.06.tar.gz", 
-                "fe77ab20a95d980ed17a659aa67e371fdd4d656d19c4c7950e7b720b0c2f1a86"
-            ]
-        }
-    }
-}
\ No newline at end of file

From d4f14a72dc1dd79396e0e80980268aee902b61e4 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 14 Nov 2023 14:28:18 -0600
Subject: [PATCH 674/871] [ie] Do not test truth value of
 `xml.etree.ElementTree.Element` (#8582)

Testing the truthiness of an `xml.etree.ElementTree.Element` instance is deprecated in py3.12

Authored by: bashonly
---
 yt_dlp/extractor/cbc.py        |  7 ++++---
 yt_dlp/extractor/common.py     |  4 +++-
 yt_dlp/extractor/mtv.py        |  3 ++-
 yt_dlp/extractor/nbc.py        |  7 +++++--
 yt_dlp/extractor/slideslive.py | 12 +++++++-----
 5 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index be2d13e44..29f0e307d 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -1,8 +1,9 @@
-import re
-import json
 import base64
+import json
+import re
 import time
 import urllib.parse
+import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..compat import (
@@ -387,7 +388,7 @@ def _find_secret_formats(self, formats, video_id):
         url = re.sub(r'(Manifest\(.*?),format=[\w-]+(.*?\))', r'\1\2', base_url)
 
         secret_xml = self._download_xml(url, video_id, note='Downloading secret XML', fatal=False)
-        if not secret_xml:
+        if not isinstance(secret_xml, xml.etree.ElementTree.Element):
             return
 
         for child in secret_xml:
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index c3ceb0039..b3a45b3fb 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -2225,7 +2225,9 @@ def _extract_mpd_vod_duration(
             mpd_url, video_id,
             note='Downloading MPD VOD manifest' if note is None else note,
             errnote='Failed to download VOD manifest' if errnote is None else errnote,
-            fatal=False, data=data, headers=headers, query=query) or {}
+            fatal=False, data=data, headers=headers, query=query)
+        if not isinstance(mpd_doc, xml.etree.ElementTree.Element):
+            return None
         return int_or_none(parse_duration(mpd_doc.get('mediaPresentationDuration')))
 
     @staticmethod
diff --git a/yt_dlp/extractor/mtv.py b/yt_dlp/extractor/mtv.py
index 0d700b9a8..e192453c7 100644
--- a/yt_dlp/extractor/mtv.py
+++ b/yt_dlp/extractor/mtv.py
@@ -1,4 +1,5 @@
 import re
+import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -137,7 +138,7 @@ def _get_video_info(self, itemdoc, use_hls=True):
         mediagen_doc = self._download_xml(
             mediagen_url, video_id, 'Downloading video urls', fatal=False)
 
-        if mediagen_doc is False:
+        if not isinstance(mediagen_doc, xml.etree.ElementTree.Element):
             return None
 
         item = mediagen_doc.find('./video/item')
diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index 666550a49..2d3aa26ec 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -1,6 +1,7 @@
 import base64
 import json
 import re
+import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from .theplatform import ThePlatformIE, default_ns
@@ -803,8 +804,10 @@ def _real_extract(self, url):
             smil = self._download_xml(
                 f'https://link.theplatform.com/s/{pdk_acct}/{player_id}', video_id,
                 note='Downloading SMIL data', query=query, fatal=is_live)
-        subtitles = self._parse_smil_subtitles(smil, default_ns) if smil else {}
-        for video in smil.findall(self._xpath_ns('.//video', default_ns)) if smil else []:
+            if not isinstance(smil, xml.etree.ElementTree.Element):
+                smil = None
+        subtitles = self._parse_smil_subtitles(smil, default_ns) if smil is not None else {}
+        for video in smil.findall(self._xpath_ns('.//video', default_ns)) if smil is not None else []:
             info['duration'] = float_or_none(remove_end(video.get('dur'), 'ms'), 1000)
             video_src_url = video.get('src')
             ext = mimetype2ext(video.get('type'), default=determine_ext(video_src_url))
diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py
index 25f867a60..13f3109d7 100644
--- a/yt_dlp/extractor/slideslive.py
+++ b/yt_dlp/extractor/slideslive.py
@@ -1,5 +1,6 @@
 import re
 import urllib.parse
+import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
@@ -469,11 +470,12 @@ def _real_extract(self, url):
             slides = self._download_xml(
                 player_info['slides_xml_url'], video_id, fatal=False,
                 note='Downloading slides XML', errnote='Failed to download slides info')
-            slide_url_template = 'https://cdn.slideslive.com/data/presentations/%s/slides/big/%s%s'
-            for slide_id, slide in enumerate(slides.findall('./slide') if slides else [], 1):
-                slides_info.append((
-                    slide_id, xpath_text(slide, './slideName', 'name'), '.jpg',
-                    int_or_none(xpath_text(slide, './timeSec', 'time'))))
+            if isinstance(slides, xml.etree.ElementTree.Element):
+                slide_url_template = 'https://cdn.slideslive.com/data/presentations/%s/slides/big/%s%s'
+                for slide_id, slide in enumerate(slides.findall('./slide')):
+                    slides_info.append((
+                        slide_id, xpath_text(slide, './slideName', 'name'), '.jpg',
+                        int_or_none(xpath_text(slide, './timeSec', 'time'))))
 
         chapters, thumbnails = [], []
         if url_or_none(player_info.get('thumbnail')):

From f04b5bedad7b281bee9814686bba1762bae092eb Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Wed, 16 Aug 2023 18:42:48 -0500
Subject: [PATCH 675/871] [ie] Do not smuggle `http_headers`

See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x

Authored by: coletdjnz
---
 test/test_networking.py        |  4 ++++
 yt_dlp/extractor/cybrary.py    |  2 +-
 yt_dlp/extractor/duboku.py     |  2 +-
 yt_dlp/extractor/embedly.py    |  2 +-
 yt_dlp/extractor/generic.py    | 11 ++++++-----
 yt_dlp/extractor/slideslive.py |  2 +-
 yt_dlp/extractor/storyfire.py  |  4 +---
 yt_dlp/extractor/vimeo.py      |  6 +++---
 yt_dlp/utils/networking.py     |  1 +
 9 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index 689161fb2..4466fc048 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -1293,6 +1293,10 @@ def test_clean_header(self):
             assert 'Youtubedl-no-compression' not in rh.headers
             assert rh.headers.get('Accept-Encoding') == 'identity'
 
+        with FakeYDL({'http_headers': {'Ytdl-socks-proxy': 'socks://localhost:1080'}}) as ydl:
+            rh = self.build_handler(ydl)
+            assert 'Ytdl-socks-proxy' not in rh.headers
+
     def test_build_handler_params(self):
         with FakeYDL({
             'http_headers': {'test': 'testtest'},
diff --git a/yt_dlp/extractor/cybrary.py b/yt_dlp/extractor/cybrary.py
index aeffe93b4..c4c78ee1b 100644
--- a/yt_dlp/extractor/cybrary.py
+++ b/yt_dlp/extractor/cybrary.py
@@ -105,7 +105,7 @@ def _real_extract(self, url):
             'chapter': module.get('title'),
             'chapter_id': str_or_none(module.get('id')),
             'title': activity.get('title'),
-            'url': smuggle_url(f'https://player.vimeo.com/video/{vimeo_id}', {'http_headers': {'Referer': 'https://api.cybrary.it'}})
+            'url': smuggle_url(f'https://player.vimeo.com/video/{vimeo_id}', {'referer': 'https://api.cybrary.it'})
         }
 
 
diff --git a/yt_dlp/extractor/duboku.py b/yt_dlp/extractor/duboku.py
index fb0546cae..fc9564cef 100644
--- a/yt_dlp/extractor/duboku.py
+++ b/yt_dlp/extractor/duboku.py
@@ -138,7 +138,7 @@ def _real_extract(self, url):
             # of the video.
             return {
                 '_type': 'url_transparent',
-                'url': smuggle_url(data_url, {'http_headers': headers}),
+                'url': smuggle_url(data_url, {'referer': webpage_url}),
                 'id': video_id,
                 'title': title,
                 'series': series_title,
diff --git a/yt_dlp/extractor/embedly.py b/yt_dlp/extractor/embedly.py
index 458aaa0a0..a424b49df 100644
--- a/yt_dlp/extractor/embedly.py
+++ b/yt_dlp/extractor/embedly.py
@@ -106,4 +106,4 @@ def _real_extract(self, url):
             return self.url_result(src, YoutubeTabIE)
         return self.url_result(smuggle_url(
             urllib.parse.unquote(traverse_obj(qs, ('src', 0), ('url', 0))),
-            {'http_headers': {'Referer': url}}))
+            {'referer': url}))
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index ac7cc673f..1503e5146 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -17,6 +17,7 @@
     determine_protocol,
     dict_get,
     extract_basic_auth,
+    filter_dict,
     format_field,
     int_or_none,
     is_html,
@@ -2435,10 +2436,10 @@ def _real_extract(self, url):
         # to accept raw bytes and being able to download only a chunk.
         # It may probably better to solve this by checking Content-Type for application/octet-stream
         # after a HEAD request, but not sure if we can rely on this.
-        full_response = self._request_webpage(url, video_id, headers={
+        full_response = self._request_webpage(url, video_id, headers=filter_dict({
             'Accept-Encoding': 'identity',
-            **smuggled_data.get('http_headers', {})
-        })
+            'Referer': smuggled_data.get('referer'),
+        }))
         new_url = full_response.url
         url = urllib.parse.urlparse(url)._replace(scheme=urllib.parse.urlparse(new_url).scheme).geturl()
         if new_url != extract_basic_auth(url)[0]:
@@ -2458,7 +2459,7 @@ def _real_extract(self, url):
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
             self.report_detected('direct video link')
-            headers = smuggled_data.get('http_headers', {})
+            headers = filter_dict({'Referer': smuggled_data.get('referer')})
             format_id = str(m.group('format_id'))
             ext = determine_ext(url, default_ext=None) or urlhandle_detect_ext(full_response)
             subtitles = {}
@@ -2710,7 +2711,7 @@ def _extract_embeds(self, url, webpage, *, urlh=None, info_dict={}):
                 'url': smuggle_url(json_ld['url'], {
                     'force_videoid': video_id,
                     'to_generic': True,
-                    'http_headers': {'Referer': url},
+                    'referer': url,
                 }),
             }, json_ld)]
 
diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py
index 13f3109d7..df2af3b35 100644
--- a/yt_dlp/extractor/slideslive.py
+++ b/yt_dlp/extractor/slideslive.py
@@ -530,7 +530,7 @@ def _real_extract(self, url):
             if service_name == 'vimeo':
                 info['url'] = smuggle_url(
                     f'https://player.vimeo.com/video/{service_id}',
-                    {'http_headers': {'Referer': url}})
+                    {'referer': url})
 
         video_slides = traverse_obj(slides, ('slides', ..., 'video', 'id'))
         if not video_slides:
diff --git a/yt_dlp/extractor/storyfire.py b/yt_dlp/extractor/storyfire.py
index 035747c31..566f77782 100644
--- a/yt_dlp/extractor/storyfire.py
+++ b/yt_dlp/extractor/storyfire.py
@@ -32,9 +32,7 @@ def _parse_video(self, video):
             'description': video.get('description'),
             'url': smuggle_url(
                 'https://player.vimeo.com/video/' + vimeo_id, {
-                    'http_headers': {
-                        'Referer': 'https://storyfire.com/',
-                    }
+                    'referer': 'https://storyfire.com/',
                 }),
             'thumbnail': video.get('storyImage'),
             'view_count': int_or_none(video.get('views')),
diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py
index e72fa50fa..e5e8144bb 100644
--- a/yt_dlp/extractor/vimeo.py
+++ b/yt_dlp/extractor/vimeo.py
@@ -37,14 +37,14 @@ class VimeoBaseInfoExtractor(InfoExtractor):
 
     @staticmethod
     def _smuggle_referrer(url, referrer_url):
-        return smuggle_url(url, {'http_headers': {'Referer': referrer_url}})
+        return smuggle_url(url, {'referer': referrer_url})
 
     def _unsmuggle_headers(self, url):
         """@returns (url, smuggled_data, headers)"""
         url, data = unsmuggle_url(url, {})
         headers = self.get_param('http_headers').copy()
-        if 'http_headers' in data:
-            headers.update(data['http_headers'])
+        if 'referer' in data:
+            headers['Referer'] = data['referer']
         return url, data, headers
 
     def _perform_login(self, username, password):
diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py
index ba0493cc2..ed0250011 100644
--- a/yt_dlp/utils/networking.py
+++ b/yt_dlp/utils/networking.py
@@ -123,6 +123,7 @@ def clean_headers(headers: HTTPHeaderDict):
     if 'Youtubedl-No-Compression' in headers:  # compat
         del headers['Youtubedl-No-Compression']
         headers['Accept-Encoding'] = 'identity'
+    headers.pop('Ytdl-socks-proxy', None)
 
 
 def remove_dot_segments(path):

From b012271d01b59759e4eefeab0308698cd9e7224c Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 14 Nov 2023 22:40:38 +0100
Subject: [PATCH 676/871] [cleanup] Misc (#8510)

Authored by: bashonly, coletdjnz, dirkf, gamer191, seproDev, Grub4K
---
 .github/PULL_REQUEST_TEMPLATE.md   |  6 ------
 README.md                          |  6 +++---
 devscripts/changelog_override.json | 16 ++++++++++++++++
 yt_dlp/extractor/la7.py            |  4 ++--
 yt_dlp/extractor/redtube.py        |  2 +-
 yt_dlp/extractor/videoken.py       | 15 ++++++++-------
 yt_dlp/extractor/youtube.py        |  2 +-
 yt_dlp/networking/_requests.py     |  3 ++-
 8 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index cbed82173..c4d3e812e 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -40,10 +40,4 @@ ### What is the purpose of your *pull request*?
 - [ ] Core bug fix/improvement
 - [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes))
 
-
-<!-- Do NOT edit/remove anything below this! -->
-</details><details><summary>Copilot Summary</summary>  
-
-copilot:all
-
 </details>
diff --git a/README.md b/README.md
index 1b92c64d6..8b92f827b 100644
--- a/README.md
+++ b/README.md
@@ -163,10 +163,10 @@ ### Differences in default behavior
 For ease of use, a few more compat options are available:
 
 * `--compat-options all`: Use all compat options (Do NOT use)
-* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter`
-* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter`
+* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx`
+* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx`
 * `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date`
-* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler`. Use this to enable all future compat options
+* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx`. Use this to enable all future compat options
 
 
 # INSTALLATION
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index fe0c82c66..010820295 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -98,5 +98,21 @@
         "action": "add",
         "when": "61bdf15fc7400601c3da1aa7a43917310a5bf391",
         "short": "[priority] Security: [[CVE-2023-40581](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-40581)] [Prevent RCE when using `--exec` with `%q` on Windows](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-42h4-v29r-42qg)\n    - The shell escape function is now using `\"\"` instead of `\\\"`.\n    - `utils.Popen` has been patched to properly quote commands."
+    },
+    {
+        "action": "change",
+        "when": "8a8b54523addf46dfd50ef599761a81bc22362e6",
+        "short": "[rh:requests] Add handler for `requests` HTTP library (#3668)\n\n\tAdds support for HTTPS proxies and persistent connections (keep-alive)",
+        "authors": ["bashonly", "coletdjnz", "Grub4K"]
+    },
+    {
+        "action": "add",
+        "when": "1d03633c5a1621b9f3a756f0a4f9dc61fab3aeaa",
+        "short": "[priority] **The release channels have been adjusted!**\n\t* [`master`](https://github.com/yt-dlp/yt-dlp-master-builds) builds are made after each push, containing the latest fixes (but also possibly bugs). This was previously the `nightly` channel.\n\t* [`nightly`](https://github.com/yt-dlp/yt-dlp-nightly-builds) builds are now made once a day, if there were any changes."
+    },
+    {
+        "action": "add",
+        "when": "f04b5bedad7b281bee9814686bba1762bae092eb",
+        "short": "[priority] Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)\n\t- Disallow smuggling of arbitary `http_headers`; extractors now only use specific headers"
     }
 ]
diff --git a/yt_dlp/extractor/la7.py b/yt_dlp/extractor/la7.py
index a3cd12b00..f5fd24134 100644
--- a/yt_dlp/extractor/la7.py
+++ b/yt_dlp/extractor/la7.py
@@ -208,9 +208,9 @@ class LA7PodcastIE(LA7PodcastEpisodeIE):  # XXX: Do not subclass from concrete I
         'url': 'https://www.la7.it/propagandalive/podcast',
         'info_dict': {
             'id': 'propagandalive',
-            'title': "Propaganda Live",
+            'title': 'Propaganda Live',
         },
-        'playlist_count_min': 10,
+        'playlist_mincount': 10,
     }]
 
     def _real_extract(self, url):
diff --git a/yt_dlp/extractor/redtube.py b/yt_dlp/extractor/redtube.py
index 49076ccd8..172c31b39 100644
--- a/yt_dlp/extractor/redtube.py
+++ b/yt_dlp/extractor/redtube.py
@@ -39,7 +39,7 @@ class RedTubeIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
-            'http://www.redtube.com/%s' % video_id, video_id)
+            f'https://www.redtube.com/{video_id}', video_id)
 
         ERRORS = (
             (('video-deleted-info', '>This video has been removed'), 'has been removed'),
diff --git a/yt_dlp/extractor/videoken.py b/yt_dlp/extractor/videoken.py
index 560b41a6d..eaf0cc8ae 100644
--- a/yt_dlp/extractor/videoken.py
+++ b/yt_dlp/extractor/videoken.py
@@ -11,6 +11,7 @@
     ExtractorError,
     InAdvancePagedList,
     int_or_none,
+    remove_start,
     traverse_obj,
     update_url_query,
     url_or_none,
@@ -39,11 +40,11 @@ def _create_slideslive_url(self, video_url, video_id, referer):
         if not video_url and not video_id:
             return
         elif not video_url or 'embed/sign-in' in video_url:
-            video_url = f'https://slideslive.com/embed/{video_id.lstrip("slideslive-")}'
+            video_url = f'https://slideslive.com/embed/{remove_start(video_id, "slideslive-")}'
         if url_or_none(referer):
             return update_url_query(video_url, {
                 'embed_parent_url': referer,
-                'embed_container_origin': f'https://{urllib.parse.urlparse(referer).netloc}',
+                'embed_container_origin': f'https://{urllib.parse.urlparse(referer).hostname}',
             })
         return video_url
 
@@ -57,12 +58,12 @@ def _extract_videos(self, videos, url):
                 video_url = video_id
                 ie_key = 'Youtube'
             else:
-                video_url = traverse_obj(video, 'embed_url', 'embeddableurl')
-                if urllib.parse.urlparse(video_url).netloc == 'slideslive.com':
+                video_url = traverse_obj(video, 'embed_url', 'embeddableurl', expected_type=url_or_none)
+                if not video_url:
+                    continue
+                elif urllib.parse.urlparse(video_url).hostname == 'slideslive.com':
                     ie_key = SlidesLiveIE
                     video_url = self._create_slideslive_url(video_url, video_id, url)
-            if not video_url:
-                continue
             yield self.url_result(video_url, ie_key, video_id)
 
 
@@ -178,7 +179,7 @@ def _real_extract(self, url):
             return self.url_result(
                 self._create_slideslive_url(None, video_id, url), SlidesLiveIE, video_id)
         elif re.match(r'^[\w-]{11}$', video_id):
-            self.url_result(video_id, 'Youtube', video_id)
+            return self.url_result(video_id, 'Youtube', video_id)
         else:
             raise ExtractorError('Unable to extract without VideoKen API response')
 
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index adbac8e95..f6caf0970 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -6687,7 +6687,7 @@ class YoutubePlaylistIE(InfoExtractor):
             'uploader_url': 'https://www.youtube.com/@milan5503',
             'availability': 'public',
         },
-        'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'],
+        'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden', 'Retrying', 'Giving up'],
     }, {
         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
         'playlist_mincount': 455,
diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py
index fe3f60b0b..9fb1d75f4 100644
--- a/yt_dlp/networking/_requests.py
+++ b/yt_dlp/networking/_requests.py
@@ -255,7 +255,8 @@ def __init__(self, *args, **kwargs):
         handler.setFormatter(logging.Formatter('requests: %(message)s'))
         handler.addFilter(Urllib3LoggingFilter())
         logger.addHandler(handler)
-        logger.setLevel(logging.WARNING)
+        # TODO: Use a logger filter to suppress pool reuse warning instead
+        logger.setLevel(logging.ERROR)
 
         if self.verbose:
             # Setting this globally is not ideal, but is easier than hacking with urllib3.

From a9d3f4b20a3533d2a40104c85bc2cc6c2564c800 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Tue, 14 Nov 2023 15:58:49 -0600
Subject: [PATCH 677/871] [cleanup] Fix changelog typo

Authored by: bashonly
---
 devscripts/changelog_override.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index 010820295..ca811cb65 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -113,6 +113,6 @@
     {
         "action": "add",
         "when": "f04b5bedad7b281bee9814686bba1762bae092eb",
-        "short": "[priority] Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)\n\t- Disallow smuggling of arbitary `http_headers`; extractors now only use specific headers"
+        "short": "[priority] Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)\n\t- Disallow smuggling of arbitrary `http_headers`; extractors now only use specific headers"
     }
 ]

From 5d3a3cd4934853126a5d6b721dbec6946c9cd8ce Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Tue, 14 Nov 2023 22:09:25 +0000
Subject: [PATCH 678/871] Release 2023.11.14

Created by: Grub4K

:ci skip all :ci run dl
---
 .github/ISSUE_TEMPLATE/1_broken_site.yml      | 17 +++--
 .../ISSUE_TEMPLATE/2_site_support_request.yml | 17 +++--
 .../ISSUE_TEMPLATE/3_site_feature_request.yml | 17 +++--
 .github/ISSUE_TEMPLATE/4_bug_report.yml       | 17 +++--
 .github/ISSUE_TEMPLATE/5_feature_request.yml  | 17 +++--
 .github/ISSUE_TEMPLATE/6_question.yml         | 17 +++--
 CONTRIBUTORS                                  | 11 +++
 Changelog.md                                  | 68 +++++++++++++++++++
 README.md                                     |  3 +-
 supportedsites.md                             | 13 +++-
 yt_dlp/version.py                             |  6 +-
 11 files changed, 143 insertions(+), 60 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml
index 6c713e5a8..5df13ad9b 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.yml
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting that yt-dlp is broken on a **supported** site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -61,19 +61,18 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
-        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.13, Current version: 2023.10.13
-        yt-dlp is up to date (2023.10.13)
+        [debug] Request Handlers: urllib, requests
+        [debug] Loaded 1893 extractors
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
index e20036ce8..644c87a7e 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -73,19 +73,18 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
-        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.13, Current version: 2023.10.13
-        yt-dlp is up to date (2023.10.13)
+        [debug] Request Handlers: urllib, requests
+        [debug] Loaded 1893 extractors
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
index a9845b6b8..59d0474c2 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -69,19 +69,18 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
-        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.13, Current version: 2023.10.13
-        yt-dlp is up to date (2023.10.13)
+        [debug] Request Handlers: urllib, requests
+        [debug] Loaded 1893 extractors
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml
index d3d60a11e..e20739673 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml
@@ -18,7 +18,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -54,19 +54,18 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
-        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.13, Current version: 2023.10.13
-        yt-dlp is up to date (2023.10.13)
+        [debug] Request Handlers: urllib, requests
+        [debug] Loaded 1893 extractors
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
     validations:
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml
index 57de148d0..e06db9ccf 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml
@@ -20,7 +20,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
@@ -50,18 +50,17 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
-        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.13, Current version: 2023.10.13
-        yt-dlp is up to date (2023.10.13)
+        [debug] Request Handlers: urllib, requests
+        [debug] Loaded 1893 extractors
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml
index 7b55a7427..571223a9c 100644
--- a/.github/ISSUE_TEMPLATE/6_question.yml
+++ b/.github/ISSUE_TEMPLATE/6_question.yml
@@ -26,7 +26,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2023.10.13** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels))
           required: true
         - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
@@ -56,18 +56,17 @@ body:
       description: |
         It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'test:youtube']
-        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
-        [debug] yt-dlp version 2023.10.13 [9d339c4] (win32_exe)
+        [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe)
         [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
-        [debug] Checking exe version: ffmpeg -bsfs
-        [debug] Checking exe version: ffprobe -bsfs
         [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
         [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
-        Latest version: 2023.10.13, Current version: 2023.10.13
-        yt-dlp is up to date (2023.10.13)
+        [debug] Request Handlers: urllib, requests
+        [debug] Loaded 1893 extractors
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest
+        yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds)
+        [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 3035ee296..71752c20a 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -513,3 +513,14 @@ awalgarg
 midnightveil
 naginatana
 Riteo
+1100101
+aniolpages
+bartbroere
+CrendKing
+Esokrates
+HitomaruKonpaku
+LoserFox
+peci1
+saintliao
+shubhexists
+SirElderling
diff --git a/Changelog.md b/Changelog.md
index 6f45eab2f..a64648120 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,74 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.11.14
+
+#### Important changes
+- **The release channels have been adjusted!**
+    * [`master`](https://github.com/yt-dlp/yt-dlp-master-builds) builds are made after each push, containing the latest fixes (but also possibly bugs). This was previously the `nightly` channel.
+    * [`nightly`](https://github.com/yt-dlp/yt-dlp-nightly-builds) builds are now made once a day, if there were any changes.
+- Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)
+    - Disallow smuggling of arbitrary `http_headers`; extractors now only use specific headers
+
+#### Core changes
+- [Add `--compat-option manifest-filesize-approx`](https://github.com/yt-dlp/yt-dlp/commit/10025b715ea01489557eb2c5a3cc04d361fcdb52) ([#8356](https://github.com/yt-dlp/yt-dlp/issues/8356)) by [bashonly](https://github.com/bashonly)
+- [Fix format sorting with `--load-info-json`](https://github.com/yt-dlp/yt-dlp/commit/595ea4a99b726b8fe9463e7853b7053978d0544e) ([#8521](https://github.com/yt-dlp/yt-dlp/issues/8521)) by [bashonly](https://github.com/bashonly)
+- [Include build origin in verbose output](https://github.com/yt-dlp/yt-dlp/commit/20314dd46f25e0e0a7e985a7804049aefa8b909f) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+- [Only ensure playlist thumbnail dir if writing thumbs](https://github.com/yt-dlp/yt-dlp/commit/a40e0b37dfc8c26916b0e01aa3f29f3bc42250b6) ([#8373](https://github.com/yt-dlp/yt-dlp/issues/8373)) by [bashonly](https://github.com/bashonly)
+- **update**: [Overhaul self-updater](https://github.com/yt-dlp/yt-dlp/commit/0b6ad22e6a432006a75df968f0283e6c6b3cfae6) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+
+#### Extractor changes
+- [Do not smuggle `http_headers`](https://github.com/yt-dlp/yt-dlp/commit/f04b5bedad7b281bee9814686bba1762bae092eb) by [coletdjnz](https://github.com/coletdjnz)
+- [Do not test truth value of `xml.etree.ElementTree.Element`](https://github.com/yt-dlp/yt-dlp/commit/d4f14a72dc1dd79396e0e80980268aee902b61e4) ([#8582](https://github.com/yt-dlp/yt-dlp/issues/8582)) by [bashonly](https://github.com/bashonly)
+- **brilliantpala**: [Fix cookies support](https://github.com/yt-dlp/yt-dlp/commit/9b5bedf13a3323074daceb0ec6ebb3cc6e0b9684) ([#8352](https://github.com/yt-dlp/yt-dlp/issues/8352)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **generic**: [Improve direct video link ext detection](https://github.com/yt-dlp/yt-dlp/commit/4ce2f29a50fcfb9920e6f2ffe42192945a2bad7e) ([#8340](https://github.com/yt-dlp/yt-dlp/issues/8340)) by [bashonly](https://github.com/bashonly)
+- **laxarxames**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/312a2d1e8bc247264f9d85c5ec764e33aa0133b5) ([#8412](https://github.com/yt-dlp/yt-dlp/issues/8412)) by [aniolpages](https://github.com/aniolpages)
+- **n-tv.de**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/8afd9468b0c822843bc480d366d1c86698daabfb) ([#8414](https://github.com/yt-dlp/yt-dlp/issues/8414)) by [1100101](https://github.com/1100101)
+- **neteasemusic**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/46acc418a53470b7f32581b3309c3cb87aa8488d) ([#8531](https://github.com/yt-dlp/yt-dlp/issues/8531)) by [LoserFox](https://github.com/LoserFox)
+- **nhk**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/54579be4364e148277c32e20a5c3efc2c3f52f5b) ([#8388](https://github.com/yt-dlp/yt-dlp/issues/8388)) by [garret1317](https://github.com/garret1317)
+- **novaembed**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/3ff494f6f41c27549420fa88be27555bd449ffdc) ([#8368](https://github.com/yt-dlp/yt-dlp/issues/8368)) by [peci1](https://github.com/peci1)
+- **npo**: [Send `POST` request to streams API endpoint](https://github.com/yt-dlp/yt-dlp/commit/8e02a4dcc800f9444e9d461edc41edd7b662f435) ([#8413](https://github.com/yt-dlp/yt-dlp/issues/8413)) by [bartbroere](https://github.com/bartbroere)
+- **ondemandkorea**: [Overhaul extractor](https://github.com/yt-dlp/yt-dlp/commit/05adfd883a4f2ecae0267e670a62a2e45c351aeb) ([#8386](https://github.com/yt-dlp/yt-dlp/issues/8386)) by [seproDev](https://github.com/seproDev)
+- **orf**: podcast: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/6ba3085616652cbf05d1858efc321fdbfc4c6119) ([#8486](https://github.com/yt-dlp/yt-dlp/issues/8486)) by [Esokrates](https://github.com/Esokrates)
+- **polskieradio**: audition: [Fix playlist extraction](https://github.com/yt-dlp/yt-dlp/commit/464327acdb353ceb91d2115163a5a9621b22fe0d) ([#8459](https://github.com/yt-dlp/yt-dlp/issues/8459)) by [shubhexists](https://github.com/shubhexists)
+- **qdance**: [Update `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/177f0d963e4b9db749805c482e6f288354c8be84) ([#8426](https://github.com/yt-dlp/yt-dlp/issues/8426)) by [bashonly](https://github.com/bashonly)
+- **radiocomercial**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/ef12dbdcd3e7264bd3d744c1e3107597bd23ad35) ([#8508](https://github.com/yt-dlp/yt-dlp/issues/8508)) by [SirElderling](https://github.com/SirElderling)
+- **sbs.co.kr**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/25a4bd345a0dcfece6fef752d4537eb403da94d9) ([#8326](https://github.com/yt-dlp/yt-dlp/issues/8326)) by [seproDev](https://github.com/seproDev)
+- **theatercomplextown**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/2863fcf2b6876d0c7965ff7d6d9242eea653dc6b) ([#8560](https://github.com/yt-dlp/yt-dlp/issues/8560)) by [bashonly](https://github.com/bashonly)
+- **thisav**: [Remove](https://github.com/yt-dlp/yt-dlp/commit/cb480e390d85fb3a598c1b6d5eef3438ce729fc9) ([#8346](https://github.com/yt-dlp/yt-dlp/issues/8346)) by [bashonly](https://github.com/bashonly)
+- **thisoldhouse**: [Add login support](https://github.com/yt-dlp/yt-dlp/commit/c76c96677ff6a056f5844a568ef05ee22c46d6f4) ([#8561](https://github.com/yt-dlp/yt-dlp/issues/8561)) by [bashonly](https://github.com/bashonly)
+- **twitcasting**: [Fix livestream extraction](https://github.com/yt-dlp/yt-dlp/commit/7b8b1cf5eb8bf44ce70bc24e1f56f0dba2737e98) ([#8427](https://github.com/yt-dlp/yt-dlp/issues/8427)) by [JC-Chung](https://github.com/JC-Chung), [saintliao](https://github.com/saintliao)
+- **twitter**
+    - broadcast
+        - [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/7d337ca977d73a0a6c07ab481ed8faa8f6ff8726) ([#8383](https://github.com/yt-dlp/yt-dlp/issues/8383)) by [HitomaruKonpaku](https://github.com/HitomaruKonpaku)
+        - [Support `--wait-for-video`](https://github.com/yt-dlp/yt-dlp/commit/f6e97090d2ed9e05441ab0f4bec3559b816d7a00) ([#8475](https://github.com/yt-dlp/yt-dlp/issues/8475)) by [bashonly](https://github.com/bashonly)
+- **weibo**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/15b252dfd2c6807fe57afc5a95e59abadb32ccd2) ([#8463](https://github.com/yt-dlp/yt-dlp/issues/8463)) by [c-basalt](https://github.com/c-basalt)
+- **weverse**: [Fix login error handling](https://github.com/yt-dlp/yt-dlp/commit/4a601c9eff9fb42e24a4c8da3fa03628e035b35b) ([#8458](https://github.com/yt-dlp/yt-dlp/issues/8458)) by [seproDev](https://github.com/seproDev)
+- **youtube**: [Check newly uploaded iOS HLS formats](https://github.com/yt-dlp/yt-dlp/commit/ef79d20dc9d27ac002a7196f073b37f2f2721aed) ([#8336](https://github.com/yt-dlp/yt-dlp/issues/8336)) by [bashonly](https://github.com/bashonly)
+- **zoom**: [Extract combined view formats](https://github.com/yt-dlp/yt-dlp/commit/3906de07551fedb00b789345bf24cc27d6ddf128) ([#7847](https://github.com/yt-dlp/yt-dlp/issues/7847)) by [Mipsters](https://github.com/Mipsters)
+
+#### Downloader changes
+- **aria2c**: [Remove duplicate `--file-allocation=none`](https://github.com/yt-dlp/yt-dlp/commit/21b25281c51523620706b11bfc1c4a889858e1f2) ([#8332](https://github.com/yt-dlp/yt-dlp/issues/8332)) by [CrendKing](https://github.com/CrendKing)
+- **dash**: [Force native downloader for `--live-from-start`](https://github.com/yt-dlp/yt-dlp/commit/2622c804d1a5accc3045db398e0fc52074f4bdb3) ([#8339](https://github.com/yt-dlp/yt-dlp/issues/8339)) by [bashonly](https://github.com/bashonly)
+
+#### Networking changes
+- **Request Handler**: requests: [Add handler for `requests` HTTP library (#3668)](https://github.com/yt-dlp/yt-dlp/commit/8a8b54523addf46dfd50ef599761a81bc22362e6) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz), [Grub4K](https://github.com/Grub4K) (With fixes in [4e38e2a](https://github.com/yt-dlp/yt-dlp/commit/4e38e2ae9d7380015349e6aee59c78bb3938befd))
+
+    Adds support for HTTPS proxies and persistent connections (keep-alive)
+
+#### Misc. changes
+- **build**
+    - [Include secretstorage in Linux builds](https://github.com/yt-dlp/yt-dlp/commit/9970d74c8383432c6c8779aa47d3253dcf412b14) by [bashonly](https://github.com/bashonly)
+    - [Overhaul and unify release workflow](https://github.com/yt-dlp/yt-dlp/commit/1d03633c5a1621b9f3a756f0a4f9dc61fab3aeaa) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+- **ci**
+    - [Bump `actions/checkout` to v4](https://github.com/yt-dlp/yt-dlp/commit/5438593a35b7b042fc48fe29cad0b9039f07c9bb) by [bashonly](https://github.com/bashonly)
+    - [Run core tests with dependencies](https://github.com/yt-dlp/yt-dlp/commit/700444c23ddb65f618c2abd942acdc0c58c650b1) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz)
+- **cleanup**
+    - [Fix changelog typo](https://github.com/yt-dlp/yt-dlp/commit/a9d3f4b20a3533d2a40104c85bc2cc6c2564c800) by [bashonly](https://github.com/bashonly)
+    - [Update documentation for master and nightly channels](https://github.com/yt-dlp/yt-dlp/commit/a00af29853b8c7350ce086f4cab8c2c9cf2fcf1d) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
+    - Miscellaneous: [b012271](https://github.com/yt-dlp/yt-dlp/commit/b012271d01b59759e4eefeab0308698cd9e7224c) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz), [dirkf](https://github.com/dirkf), [gamer191](https://github.com/gamer191), [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev)
+- **test**: update: [Implement simple updater unit tests](https://github.com/yt-dlp/yt-dlp/commit/87264d4fdadcddd91289b968dd0e4bf58d449267) by [bashonly](https://github.com/bashonly)
+
 ### 2023.10.13
 
 #### Core changes
diff --git a/README.md b/README.md
index 8b92f827b..33690f470 100644
--- a/README.md
+++ b/README.md
@@ -380,7 +380,8 @@ ## General Options:
                                     CHANNEL can be a repository as well. CHANNEL
                                     and TAG default to "stable" and "latest"
                                     respectively if omitted; See "UPDATE" for
-                                    details. Supported channels: stable, nightly
+                                    details. Supported channels: stable,
+                                    nightly, master
     -i, --ignore-errors             Ignore download and postprocessing errors.
                                     The download will be considered successful
                                     even if the postprocessing fails
diff --git a/supportedsites.md b/supportedsites.md
index 0ab61d68d..169da5912 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -700,6 +700,7 @@ # Supported sites
  - **LastFM**
  - **LastFMPlaylist**
  - **LastFMUser**
+ - **LaXarxaMes**: [*laxarxames*](## "netrc machine")
  - **lbry**
  - **lbry:channel**
  - **lbry:playlist**
@@ -1026,6 +1027,7 @@ # Supported sites
  - **on24**: ON24
  - **OnDemandChinaEpisode**
  - **OnDemandKorea**
+ - **OnDemandKoreaProgram**
  - **OneFootball**
  - **OnePlacePodcast**
  - **onet.pl**
@@ -1043,6 +1045,7 @@ # Supported sites
  - **OraTV**
  - **orf:​fm4:story**: fm4.orf.at stories
  - **orf:iptv**: iptv.ORF.at
+ - **orf:podcast**
  - **orf:radio**
  - **orf:tvthek**: ORF TVthek
  - **OsnatelTV**: [*osnateltv*](## "netrc machine")
@@ -1180,6 +1183,8 @@ # Supported sites
  - **radiobremen**
  - **radiocanada**
  - **radiocanada:audiovideo**
+ - **RadioComercial**
+ - **RadioComercialPlaylist**
  - **radiofrance**
  - **RadioFranceLive**
  - **RadioFrancePodcast**
@@ -1306,6 +1311,9 @@ # Supported sites
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
+ - **sbs.co.kr**
+ - **sbs.co.kr:allvod_program**
+ - **sbs.co.kr:programs_vod**
  - **schooltv**
  - **ScienceChannel**
  - **screen.yahoo:search**: Yahoo screen search; "yvsearch:" prefix
@@ -1474,6 +1482,8 @@ # Supported sites
  - **TenPlaySeason**
  - **TF1**
  - **TFO**
+ - **theatercomplextown:ppv**: [*theatercomplextown*](## "netrc machine")
+ - **theatercomplextown:vod**: [*theatercomplextown*](## "netrc machine")
  - **TheHoleTv**
  - **TheIntercept**
  - **ThePlatform**
@@ -1482,8 +1492,7 @@ # Supported sites
  - **TheSun**
  - **TheWeatherChannel**
  - **ThisAmericanLife**
- - **ThisAV**
- - **ThisOldHouse**
+ - **ThisOldHouse**: [*thisoldhouse*](## "netrc machine")
  - **ThisVid**
  - **ThisVidMember**
  - **ThisVidPlaylist**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index a4b4d4101..6fd8e5978 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.10.13'
+__version__ = '2023.11.14'
 
-RELEASE_GIT_HEAD = 'b634ba742d8f38ce9ecfa0546485728b0c6c59d1'
+RELEASE_GIT_HEAD = 'a9d3f4b20a3533d2a40104c85bc2cc6c2564c800'
 
 VARIANT = None
 
@@ -12,4 +12,4 @@
 
 ORIGIN = 'yt-dlp/yt-dlp'
 
-_pkg_version = '2023.10.13'
+_pkg_version = '2023.11.14'

From 21dc069bea2d4d99345dd969e098f4535c751d45 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Wed, 15 Nov 2023 15:34:39 +0100
Subject: [PATCH 679/871] [ie/beatbump] Update `_VALID_URL` (#8576)

Authored by: seproDev
---
 yt_dlp/extractor/beatbump.py | 42 ++++++++++++++++++++++--------------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/yt_dlp/extractor/beatbump.py b/yt_dlp/extractor/beatbump.py
index 0f40ebe7a..f48566b2d 100644
--- a/yt_dlp/extractor/beatbump.py
+++ b/yt_dlp/extractor/beatbump.py
@@ -3,14 +3,13 @@
 
 
 class BeatBumpVideoIE(InfoExtractor):
-    _VALID_URL = r'https://beatbump\.ml/listen\?id=(?P<id>[\w-]+)'
+    _VALID_URL = r'https://beatbump\.(?:ml|io)/listen\?id=(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://beatbump.ml/listen?id=MgNrAu2pzNs',
         'md5': '5ff3fff41d3935b9810a9731e485fe66',
         'info_dict': {
             'id': 'MgNrAu2pzNs',
             'ext': 'mp4',
-            'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
             'artist': 'Stephen',
             'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
             'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
@@ -22,10 +21,9 @@ class BeatBumpVideoIE(InfoExtractor):
             'alt_title': 'Voyeur Girl',
             'view_count': int,
             'track': 'Voyeur Girl',
-            'uploader': 'Stephen - Topic',
+            'uploader': 'Stephen',
             'title': 'Voyeur Girl',
             'channel_follower_count': int,
-            'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
             'age_limit': 0,
             'availability': 'public',
             'live_status': 'not_live',
@@ -36,7 +34,12 @@ class BeatBumpVideoIE(InfoExtractor):
             'tags': 'count:11',
             'creator': 'Stephen',
             'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
-        }
+            'channel_is_verified': True,
+            'heatmap': 'count:100',
+        },
+    }, {
+        'url': 'https://beatbump.io/listen?id=LDGZAprNGWo',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -45,7 +48,7 @@ def _real_extract(self, url):
 
 
 class BeatBumpPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https://beatbump\.ml/(?:release\?id=|artist/|playlist/)(?P<id>[\w-]+)'
+    _VALID_URL = r'https://beatbump\.(?:ml|io)/(?:release\?id=|artist/|playlist/)(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://beatbump.ml/release?id=MPREb_gTAcphH99wE',
         'playlist_count': 50,
@@ -56,25 +59,28 @@ class BeatBumpPlaylistIE(InfoExtractor):
             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
             'description': '',
             'tags': [],
-            'modified_date': '20221223',
-        }
+            'modified_date': '20231110',
+        },
+        'expected_warnings': ['YouTube Music is not directly supported'],
     }, {
         'url': 'https://beatbump.ml/artist/UC_aEa8K-EOJ3D6gOs7HcyNg',
         'playlist_mincount': 1,
         'params': {'flatplaylist': True},
         'info_dict': {
             'id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
-            'uploader_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
+            'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
             'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
-            'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
+            'uploader_id': '@NoCopyrightSounds',
             'channel_follower_count': int,
-            'title': 'NoCopyrightSounds - Videos',
+            'title': 'NoCopyrightSounds',
             'uploader': 'NoCopyrightSounds',
             'description': 'md5:cd4fd53d81d363d05eee6c1b478b491a',
             'channel': 'NoCopyrightSounds',
-            'tags': 'count:12',
+            'tags': 'count:65',
             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
+            'channel_is_verified': True,
         },
+        'expected_warnings': ['YouTube Music is not directly supported'],
     }, {
         'url': 'https://beatbump.ml/playlist/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
         'playlist_mincount': 1,
@@ -84,16 +90,20 @@ class BeatBumpPlaylistIE(InfoExtractor):
             'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
             'view_count': int,
-            'channel_url': 'https://www.youtube.com/@NoCopyrightSounds',
-            'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
+            'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
+            'uploader_id': '@NoCopyrightSounds',
             'title': 'NCS : All Releases 💿',
             'uploader': 'NoCopyrightSounds',
             'availability': 'public',
             'channel': 'NoCopyrightSounds',
             'tags': [],
-            'modified_date': '20221225',
+            'modified_date': '20231112',
             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
-        }
+        },
+        'expected_warnings': ['YouTube Music is not directly supported'],
+    }, {
+        'url': 'https://beatbump.io/playlist/VLPLFCHGavqRG-q_2ZhmgU2XB2--ZY6irT1c',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0f634dba3afdc429ece8839b02f6d56c27b7973a Mon Sep 17 00:00:00 2001
From: FrankZ85 <43293037+FrankZ85@users.noreply.github.com>
Date: Wed, 15 Nov 2023 23:38:52 +0100
Subject: [PATCH 680/871] [ie/tv5mondeplus] Extract subtitles (#4209)

Closes #4205
Authored by: FrankZ85
---
 yt_dlp/extractor/tv5mondeplus.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/yt_dlp/extractor/tv5mondeplus.py b/yt_dlp/extractor/tv5mondeplus.py
index 4da1b26d1..a445fae85 100644
--- a/yt_dlp/extractor/tv5mondeplus.py
+++ b/yt_dlp/extractor/tv5mondeplus.py
@@ -84,6 +84,13 @@ class TV5MondePlusIE(InfoExtractor):
     }]
     _GEO_BYPASS = False
 
+    @staticmethod
+    def _extract_subtitles(data_captions):
+        subtitles = {}
+        for f in traverse_obj(data_captions, ('files', lambda _, v: url_or_none(v['file']))):
+            subtitles.setdefault(f.get('label') or 'fra', []).append({'url': f['file']})
+        return subtitles
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
@@ -176,6 +183,8 @@ def process_video_files(v):
             'duration': duration,
             'upload_date': upload_date,
             'formats': formats,
+            'subtitles': self._extract_subtitles(self._parse_json(
+                traverse_obj(vpl_data, ('data-captions', {str}), default='{}'), display_id, fatal=False)),
             'series': series,
             'episode': episode,
         }

From 0783fd558ed0d3a8bc754beb75a406256f8b97b2 Mon Sep 17 00:00:00 2001
From: almx <a@almx.dk>
Date: Wed, 15 Nov 2023 23:42:18 +0100
Subject: [PATCH 681/871] [ie/DRTV] Fix extractor (#8484)

Closes #8298
Authored by: almx, seproDev

Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 yt_dlp/extractor/drtv.py | 399 +++++++++++++++------------------------
 1 file changed, 155 insertions(+), 244 deletions(-)

diff --git a/yt_dlp/extractor/drtv.py b/yt_dlp/extractor/drtv.py
index 6c381aa14..6d5f3f649 100644
--- a/yt_dlp/extractor/drtv.py
+++ b/yt_dlp/extractor/drtv.py
@@ -1,21 +1,17 @@
-import binascii
-import hashlib
-import re
+import json
+import uuid
 
 from .common import InfoExtractor
-from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
-from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
-    float_or_none,
     int_or_none,
     mimetype2ext,
-    str_or_none,
-    traverse_obj,
-    unified_timestamp,
+    parse_iso8601,
+    try_call,
     update_url_query,
     url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 SERIES_API = 'https://production-cdn.dr-massive.com/api/page?device=web_browser&item_detail_expand=all&lang=da&max_list_prefetch=3&path=%s'
 
@@ -24,7 +20,7 @@ class DRTVIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:www\.)?dr\.dk/(?:tv/se|nyheder|(?P<radio>radio|lyd)(?:/ondemand)?)/(?:[^/]+/)*|
+                            (?:www\.)?dr\.dk/tv/se(?:/ondemand)?/(?:[^/?#]+/)*|
                             (?:www\.)?(?:dr\.dk|dr-massive\.com)/drtv/(?:se|episode|program)/
                         )
                         (?P<id>[\da-z_-]+)
@@ -53,22 +49,6 @@ class DRTVIE(InfoExtractor):
         },
         'expected_warnings': ['Unable to download f4m manifest'],
         'skip': 'this video has been removed',
-    }, {
-        # embed
-        'url': 'https://www.dr.dk/nyheder/indland/live-christianias-rydning-af-pusher-street-er-i-gang',
-        'info_dict': {
-            'id': 'urn:dr:mu:programcard:57c926176187a50a9c6e83c6',
-            'ext': 'mp4',
-            'title': 'christiania pusher street ryddes drdkrjpo',
-            'description': 'md5:2a71898b15057e9b97334f61d04e6eb5',
-            'timestamp': 1472800279,
-            'upload_date': '20160902',
-            'duration': 131.4,
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'expected_warnings': ['Unable to download f4m manifest'],
     }, {
         # with SignLanguage formats
         'url': 'https://www.dr.dk/tv/se/historien-om-danmark/-/historien-om-danmark-stenalder',
@@ -87,33 +67,54 @@ class DRTVIE(InfoExtractor):
             'season': 'Historien om Danmark',
             'series': 'Historien om Danmark',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'skip': 'this video has been removed',
     }, {
-        'url': 'https://www.dr.dk/lyd/p4kbh/regionale-nyheder-kh4/p4-nyheder-2019-06-26-17-30-9',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.dr.dk/drtv/se/bonderoeven_71769',
+        'url': 'https://www.dr.dk/drtv/se/frank-and-kastaniegaarden_71769',
         'info_dict': {
             'id': '00951930010',
             'ext': 'mp4',
-            'title': 'Bonderøven 2019 (1:8)',
-            'description': 'md5:b6dcfe9b6f0bea6703e9a0092739a5bd',
-            'timestamp': 1654856100,
-            'upload_date': '20220610',
-            'duration': 2576.6,
-            'season': 'Bonderøven 2019',
-            'season_id': 'urn:dr:mu:bundle:5c201667a11fa01ca4528ce5',
+            'title': 'Frank & Kastaniegaarden',
+            'description': 'md5:974e1780934cf3275ef10280204bccb0',
+            'release_timestamp': 1546545600,
+            'release_date': '20190103',
+            'duration': 2576,
+            'season': 'Frank & Kastaniegaarden',
+            'season_id': '67125',
             'release_year': 2019,
             'season_number': 2019,
             'series': 'Frank & Kastaniegaarden',
             'episode_number': 1,
-            'episode': 'Episode 1',
+            'episode': 'Frank & Kastaniegaarden',
+            'thumbnail': r're:https?://.+',
         },
         'params': {
             'skip_download': True,
         },
+    }, {
+        # Foreign and Regular subtitle track
+        'url': 'https://www.dr.dk/drtv/se/spise-med-price_-pasta-selv_397445',
+        'info_dict': {
+            'id': '00212301010',
+            'ext': 'mp4',
+            'episode_number': 1,
+            'title': 'Spise med Price: Pasta Selv',
+            'alt_title': '1. Pasta Selv',
+            'release_date': '20230807',
+            'description': 'md5:2da9060524fed707810d71080b3d0cd8',
+            'duration': 1750,
+            'season': 'Spise med Price',
+            'release_timestamp': 1691438400,
+            'season_id': '397440',
+            'episode': 'Spise med Price: Pasta Selv',
+            'thumbnail': r're:https?://.+',
+            'season_number': 15,
+            'series': 'Spise med Price',
+            'release_year': 2022,
+            'subtitles': 'mincount:2',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
     }, {
         'url': 'https://www.dr.dk/drtv/episode/bonderoeven_71769',
         'only_matching': True,
@@ -123,226 +124,127 @@ class DRTVIE(InfoExtractor):
     }, {
         'url': 'https://www.dr.dk/drtv/program/jagten_220924',
         'only_matching': True,
-    }, {
-        'url': 'https://www.dr.dk/lyd/p4aarhus/regionale-nyheder-ar4/regionale-nyheder-2022-05-05-12-30-3',
-        'info_dict': {
-            'id': 'urn:dr:mu:programcard:6265cb2571401424d0360113',
-            'title': "Regionale nyheder",
-            'ext': 'mp4',
-            'duration': 120.043,
-            'series': 'P4 Østjylland regionale nyheder',
-            'timestamp': 1651746600,
-            'season': 'Regionale nyheder',
-            'release_year': 0,
-            'season_id': 'urn:dr:mu:bundle:61c26889539f0201586b73c5',
-            'description': '',
-            'upload_date': '20220505',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'skip': 'this video has been removed',
-    }, {
-        'url': 'https://www.dr.dk/lyd/p4kbh/regionale-nyheder-kh4/regionale-nyheder-2023-03-14-10-30-9',
-        'info_dict': {
-            'ext': 'mp4',
-            'id': '14802310112',
-            'timestamp': 1678786200,
-            'duration': 120.043,
-            'season_id': 'urn:dr:mu:bundle:63a4f7c87140143504b6710f',
-            'series': 'P4 København regionale nyheder',
-            'upload_date': '20230314',
-            'release_year': 0,
-            'description': 'Hør seneste regionale nyheder fra P4 København.',
-            'season': 'Regionale nyheder',
-            'title': 'Regionale nyheder',
-        },
     }]
 
+    SUBTITLE_LANGS = {
+        'DanishLanguageSubtitles': 'da',
+        'ForeignLanguageSubtitles': 'da_foreign',
+        'CombinedLanguageSubtitles': 'da_combined',
+    }
+
+    _TOKEN = None
+
+    def _real_initialize(self):
+        if self._TOKEN:
+            return
+
+        token_response = self._download_json(
+            'https://production.dr-massive.com/api/authorization/anonymous-sso', None,
+            note='Downloading anonymous token', headers={
+                'content-type': 'application/json',
+            }, query={
+                'device': 'web_browser',
+                'ff': 'idp,ldp,rpt',
+                'lang': 'da',
+                'supportFallbackToken': 'true',
+            }, data=json.dumps({
+                'deviceId': str(uuid.uuid4()),
+                'scopes': ['Catalog'],
+                'optout': True,
+            }).encode())
+
+        self._TOKEN = traverse_obj(
+            token_response, (lambda _, x: x['type'] == 'UserAccount', 'value', {str}), get_all=False)
+        if not self._TOKEN:
+            raise ExtractorError('Unable to get anonymous token')
+
     def _real_extract(self, url):
-        raw_video_id, is_radio_url = self._match_valid_url(url).group('id', 'radio')
+        url_slug = self._match_id(url)
+        webpage = self._download_webpage(url, url_slug)
 
-        webpage = self._download_webpage(url, raw_video_id)
-
-        if '>Programmet er ikke længere tilgængeligt' in webpage:
-            raise ExtractorError(
-                'Video %s is not available' % raw_video_id, expected=True)
-
-        video_id = self._search_regex(
-            (r'data-(?:material-identifier|episode-slug)="([^"]+)"',
-             r'data-resource="[^>"]+mu/programcard/expanded/([^"]+)"'),
-            webpage, 'video id', default=None)
-
-        if not video_id:
-            video_id = self._search_regex(
-                r'(urn(?:%3A|:)dr(?:%3A|:)mu(?:%3A|:)programcard(?:%3A|:)[\da-f]+)',
-                webpage, 'urn', default=None)
-            if video_id:
-                video_id = compat_urllib_parse_unquote(video_id)
-
-        _PROGRAMCARD_BASE = 'https://www.dr.dk/mu-online/api/1.4/programcard'
-        query = {'expanded': 'true'}
-
-        if video_id:
-            programcard_url = '%s/%s' % (_PROGRAMCARD_BASE, video_id)
+        json_data = self._search_json(
+            r'window\.__data\s*=', webpage, 'data', url_slug, fatal=False) or {}
+        item = traverse_obj(
+            json_data, ('cache', 'page', ..., (None, ('entries', 0)), 'item', {dict}), get_all=False)
+        if item:
+            item_id = item.get('id')
         else:
-            programcard_url = _PROGRAMCARD_BASE
-            if is_radio_url:
-                video_id = self._search_nextjs_data(
-                    webpage, raw_video_id)['props']['pageProps']['episode']['productionNumber']
-            else:
-                json_data = self._search_json(
-                    r'window\.__data\s*=', webpage, 'data', raw_video_id)
-                video_id = traverse_obj(json_data, (
-                    'cache', 'page', ..., (None, ('entries', 0)), 'item', 'customId',
-                    {lambda x: x.split(':')[-1]}), get_all=False)
-                if not video_id:
-                    raise ExtractorError('Unable to extract video id')
-            query['productionnumber'] = video_id
+            item_id = url_slug.rsplit('_', 1)[-1]
+            item = self._download_json(
+                f'https://production-cdn.dr-massive.com/api/items/{item_id}', item_id,
+                note='Attempting to download backup item data', query={
+                    'device': 'web_browser',
+                    'expand': 'all',
+                    'ff': 'idp,ldp,rpt',
+                    'geoLocation': 'dk',
+                    'isDeviceAbroad': 'false',
+                    'lang': 'da',
+                    'segments': 'drtv,optedout',
+                    'sub': 'Anonymous',
+                })
 
-        data = self._download_json(
-            programcard_url, video_id, 'Downloading video JSON', query=query)
-
-        supplementary_data = {}
-        if re.search(r'_\d+$', raw_video_id):
-            supplementary_data = self._download_json(
-                SERIES_API % f'/episode/{raw_video_id}', raw_video_id, fatal=False) or {}
-
-        title = str_or_none(data.get('Title')) or re.sub(
-            r'\s*\|\s*(?:TV\s*\|\s*DR|DRTV)$', '',
-            self._og_search_title(webpage))
-        description = self._og_search_description(
-            webpage, default=None) or data.get('Description')
-
-        timestamp = unified_timestamp(
-            data.get('PrimaryBroadcastStartTime') or data.get('SortDateTime'))
-
-        thumbnail = None
-        duration = None
-
-        restricted_to_denmark = False
+        video_id = try_call(lambda: item['customId'].rsplit(':', 1)[-1]) or item_id
+        stream_data = self._download_json(
+            f'https://production.dr-massive.com/api/account/items/{item_id}/videos', video_id,
+            note='Downloading stream data', query={
+                'delivery': 'stream',
+                'device': 'web_browser',
+                'ff': 'idp,ldp,rpt',
+                'lang': 'da',
+                'resolution': 'HD-1080',
+                'sub': 'Anonymous',
+            }, headers={'authorization': f'Bearer {self._TOKEN}'})
 
         formats = []
         subtitles = {}
+        for stream in traverse_obj(stream_data, (lambda _, x: x['url'])):
+            format_id = stream.get('format', 'na')
+            access_service = stream.get('accessService')
+            preference = None
+            subtitle_suffix = ''
+            if access_service in ('SpokenSubtitles', 'SignLanguage', 'VisuallyInterpreted'):
+                preference = -1
+                format_id += f'-{access_service}'
+                subtitle_suffix = f'-{access_service}'
+            elif access_service == 'StandardVideo':
+                preference = 1
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                stream.get('url'), video_id, preference=preference, m3u8_id=format_id, fatal=False)
+            formats.extend(fmts)
 
-        assets = []
-        primary_asset = data.get('PrimaryAsset')
-        if isinstance(primary_asset, dict):
-            assets.append(primary_asset)
-        secondary_assets = data.get('SecondaryAssets')
-        if isinstance(secondary_assets, list):
-            for secondary_asset in secondary_assets:
-                if isinstance(secondary_asset, dict):
-                    assets.append(secondary_asset)
+            api_subtitles = traverse_obj(stream, ('subtitles', lambda _, v: url_or_none(v['link']), {dict}))
+            if not api_subtitles:
+                self._merge_subtitles(subs, target=subtitles)
 
-        def hex_to_bytes(hex):
-            return binascii.a2b_hex(hex.encode('ascii'))
+            for sub_track in api_subtitles:
+                lang = sub_track.get('language') or 'da'
+                subtitles.setdefault(self.SUBTITLE_LANGS.get(lang, lang) + subtitle_suffix, []).append({
+                    'url': sub_track['link'],
+                    'ext': mimetype2ext(sub_track.get('format')) or 'vtt'
+                })
 
-        def decrypt_uri(e):
-            n = int(e[2:10], 16)
-            a = e[10 + n:]
-            data = hex_to_bytes(e[10:10 + n])
-            key = hashlib.sha256(('%s:sRBzYNXBzkKgnjj8pGtkACch' % a).encode('utf-8')).digest()
-            iv = hex_to_bytes(a)
-            decrypted = unpad_pkcs7(aes_cbc_decrypt_bytes(data, key, iv))
-            return decrypted.decode('utf-8').split('?')[0]
-
-        for asset in assets:
-            kind = asset.get('Kind')
-            if kind == 'Image':
-                thumbnail = url_or_none(asset.get('Uri'))
-            elif kind in ('VideoResource', 'AudioResource'):
-                duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
-                restricted_to_denmark = asset.get('RestrictedToDenmark')
-                asset_target = asset.get('Target')
-                for link in asset.get('Links', []):
-                    uri = link.get('Uri')
-                    if not uri:
-                        encrypted_uri = link.get('EncryptedUri')
-                        if not encrypted_uri:
-                            continue
-                        try:
-                            uri = decrypt_uri(encrypted_uri)
-                        except Exception:
-                            self.report_warning(
-                                'Unable to decrypt EncryptedUri', video_id)
-                            continue
-                    uri = url_or_none(uri)
-                    if not uri:
-                        continue
-                    target = link.get('Target')
-                    format_id = target or ''
-                    if asset_target in ('SpokenSubtitles', 'SignLanguage', 'VisuallyInterpreted'):
-                        preference = -1
-                        format_id += '-%s' % asset_target
-                    elif asset_target == 'Default':
-                        preference = 1
-                    else:
-                        preference = None
-                    if target == 'HDS':
-                        f4m_formats = self._extract_f4m_formats(
-                            uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
-                            video_id, preference, f4m_id=format_id, fatal=False)
-                        if kind == 'AudioResource':
-                            for f in f4m_formats:
-                                f['vcodec'] = 'none'
-                        formats.extend(f4m_formats)
-                    elif target == 'HLS':
-                        fmts, subs = self._extract_m3u8_formats_and_subtitles(
-                            uri, video_id, 'mp4', entry_protocol='m3u8_native',
-                            quality=preference, m3u8_id=format_id, fatal=False)
-                        formats.extend(fmts)
-                        self._merge_subtitles(subs, target=subtitles)
-                    else:
-                        bitrate = link.get('Bitrate')
-                        if bitrate:
-                            format_id += '-%s' % bitrate
-                        formats.append({
-                            'url': uri,
-                            'format_id': format_id,
-                            'tbr': int_or_none(bitrate),
-                            'ext': link.get('FileFormat'),
-                            'vcodec': 'none' if kind == 'AudioResource' else None,
-                            'quality': preference,
-                        })
-            subtitles_list = asset.get('SubtitlesList') or asset.get('Subtitleslist')
-            if isinstance(subtitles_list, list):
-                LANGS = {
-                    'Danish': 'da',
-                }
-                for subs in subtitles_list:
-                    if not isinstance(subs, dict):
-                        continue
-                    sub_uri = url_or_none(subs.get('Uri'))
-                    if not sub_uri:
-                        continue
-                    lang = subs.get('Language') or 'da'
-                    subtitles.setdefault(LANGS.get(lang, lang), []).append({
-                        'url': sub_uri,
-                        'ext': mimetype2ext(subs.get('MimeType')) or 'vtt'
-                    })
-
-        if not formats and restricted_to_denmark:
-            self.raise_geo_restricted(
-                'Unfortunately, DR is not allowed to show this program outside Denmark.',
-                countries=self._GEO_COUNTRIES)
+        if not formats and traverse_obj(item, ('season', 'customFields', 'IsGeoRestricted')):
+            self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
 
         return {
             'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
             'formats': formats,
             'subtitles': subtitles,
-            'series': str_or_none(data.get('SeriesTitle')),
-            'season': str_or_none(data.get('SeasonTitle')),
-            'season_number': int_or_none(data.get('SeasonNumber')),
-            'season_id': str_or_none(data.get('SeasonUrn')),
-            'episode': traverse_obj(supplementary_data, ('entries', 0, 'item', 'contextualTitle')) or str_or_none(data.get('EpisodeTitle')),
-            'episode_number': traverse_obj(supplementary_data, ('entries', 0, 'item', 'episodeNumber')) or int_or_none(data.get('EpisodeNumber')),
-            'release_year': int_or_none(data.get('ProductionYear')),
+            **traverse_obj(item, {
+                'title': 'title',
+                'alt_title': 'contextualTitle',
+                'description': 'description',
+                'thumbnail': ('images', 'wallpaper'),
+                'release_timestamp': ('customFields', 'BroadcastTimeDK', {parse_iso8601}),
+                'duration': ('duration', {int_or_none}),
+                'series': ('season', 'show', 'title'),
+                'season': ('season', 'title'),
+                'season_number': ('season', 'seasonNumber', {int_or_none}),
+                'season_id': 'seasonId',
+                'episode': 'episodeName',
+                'episode_number': ('episodeNumber', {int_or_none}),
+                'release_year': ('releaseYear', {int_or_none}),
+            }),
         }
 
 
@@ -412,6 +314,8 @@ class DRTVSeasonIE(InfoExtractor):
             'display_id': 'frank-and-kastaniegaarden',
             'title': 'Frank & Kastaniegaarden',
             'series': 'Frank & Kastaniegaarden',
+            'season_number': 2008,
+            'alt_title': 'Season 2008',
         },
         'playlist_mincount': 8
     }, {
@@ -421,6 +325,8 @@ class DRTVSeasonIE(InfoExtractor):
             'display_id': 'frank-and-kastaniegaarden',
             'title': 'Frank & Kastaniegaarden',
             'series': 'Frank & Kastaniegaarden',
+            'season_number': 2009,
+            'alt_title': 'Season 2009',
         },
         'playlist_mincount': 19
     }]
@@ -434,6 +340,7 @@ def _real_extract(self, url):
             'url': f'https://www.dr.dk/drtv{episode["path"]}',
             'ie_key': DRTVIE.ie_key(),
             'title': episode.get('title'),
+            'alt_title': episode.get('contextualTitle'),
             'episode': episode.get('episodeName'),
             'description': episode.get('shortDescription'),
             'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
@@ -446,6 +353,7 @@ def _real_extract(self, url):
             'id': season_id,
             'display_id': display_id,
             'title': traverse_obj(data, ('entries', 0, 'item', 'title')),
+            'alt_title': traverse_obj(data, ('entries', 0, 'item', 'contextualTitle')),
             'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
             'entries': entries,
             'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber'))
@@ -463,6 +371,7 @@ class DRTVSeriesIE(InfoExtractor):
             'display_id': 'frank-and-kastaniegaarden',
             'title': 'Frank & Kastaniegaarden',
             'series': 'Frank & Kastaniegaarden',
+            'alt_title': '',
         },
         'playlist_mincount': 15
     }]
@@ -476,6 +385,7 @@ def _real_extract(self, url):
             'url': f'https://www.dr.dk/drtv{season.get("path")}',
             'ie_key': DRTVSeasonIE.ie_key(),
             'title': season.get('title'),
+            'alt_title': season.get('contextualTitle'),
             'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
             'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber'))
         } for season in traverse_obj(data, ('entries', 0, 'item', 'show', 'seasons', 'items'))]
@@ -485,6 +395,7 @@ def _real_extract(self, url):
             'id': series_id,
             'display_id': display_id,
             'title': traverse_obj(data, ('entries', 0, 'item', 'title')),
+            'alt_title': traverse_obj(data, ('entries', 0, 'item', 'contextualTitle')),
             'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
             'entries': entries
         }

From dcfad52812aa8ce007cefbfbe63f58b49f6b1046 Mon Sep 17 00:00:00 2001
From: Eze Livinsky <eze@livinsky.com>
Date: Wed, 15 Nov 2023 17:13:05 -0600
Subject: [PATCH 682/871] [ie/eltrecetv] Add extractor (#8216)

Authored by: elivinsky
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/eltrecetv.py   | 62 +++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 yt_dlp/extractor/eltrecetv.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 8b036bb69..d57186535 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -565,6 +565,7 @@
 )
 from .elonet import ElonetIE
 from .elpais import ElPaisIE
+from .eltrecetv import ElTreceTVIE
 from .embedly import EmbedlyIE
 from .engadget import EngadgetIE
 from .epicon import (
diff --git a/yt_dlp/extractor/eltrecetv.py b/yt_dlp/extractor/eltrecetv.py
new file mode 100644
index 000000000..f64023af7
--- /dev/null
+++ b/yt_dlp/extractor/eltrecetv.py
@@ -0,0 +1,62 @@
+from .common import InfoExtractor
+
+
+class ElTreceTVIE(InfoExtractor):
+    IE_DESC = 'El Trece TV (Argentina)'
+    _VALID_URL = r'https?://(?:www\.)?eltrecetv\.com\.ar/[\w-]+/capitulos/temporada-\d+/(?P<id>[\w-]+)'
+    _TESTS = [
+        {
+            'url': 'https://www.eltrecetv.com.ar/ahora-caigo/capitulos/temporada-2023/programa-del-061023/',
+            'md5': '71a66673dc63f9a5939d97bfe4b311ba',
+            'info_dict': {
+                'id': 'AHCA05102023145553329621094',
+                'ext': 'mp4',
+                'title': 'AHORA CAIGO - Programa 06/10/23',
+                'thumbnail': 'https://thumbs.vodgc.net/AHCA05102023145553329621094.JPG?649339',
+            }
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/poco-correctos/capitulos/temporada-2023/programa-del-250923-invitada-dalia-gutmann/',
+            'only_matching': True,
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/argentina-tierra-de-amor-y-venganza/capitulos/temporada-2023/atav-2-capitulo-121-del-250923/',
+            'only_matching': True,
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/ahora-caigo/capitulos/temporada-2023/programa-del-250923/',
+            'only_matching': True,
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/pasaplatos/capitulos/temporada-2023/pasaplatos-el-restaurante-del-250923/',
+            'only_matching': True,
+        },
+        {
+            'url': 'https://www.eltrecetv.com.ar/el-galpon/capitulos/temporada-2023/programa-del-160923-invitado-raul-lavie/',
+            'only_matching': True,
+        }
+    ]
+
+    def _real_extract(self, url):
+        slug = self._match_id(url)
+        webpage = self._download_webpage(url, slug)
+        config = self._search_json(
+            r'Fusion.globalContent\s*=', webpage, 'content', slug)['promo_items']['basic']['embed']['config']
+        video_url = config['m3u8']
+        video_id = self._search_regex(r'/(\w+)\.m3u8', video_url, 'video id', default=slug)
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', m3u8_id='hls')
+        formats.extend([{
+            'url': f['url'][:-23],
+            'format_id': f['format_id'].replace('hls', 'http'),
+            'width': f.get('width'),
+            'height': f.get('height'),
+        } for f in formats if f['url'].endswith('/tracks-v1a1/index.m3u8') and f.get('height') != 1080])
+
+        return {
+            'id': video_id,
+            'title': config.get('title'),
+            'thumbnail': config.get('thumbnail'),
+            'formats': formats,
+            'subtitles': subtitles,
+        }

From b530118e7f48232cacf8050d79a6b20bdfcf5468 Mon Sep 17 00:00:00 2001
From: Awal Garg <awalgarg@gmail.com>
Date: Thu, 16 Nov 2023 04:45:06 +0530
Subject: [PATCH 683/871] [ie/JioSaavn] Add extractors (#8307)

Authored by: awalgarg
---
 yt_dlp/extractor/_extractors.py |  4 ++
 yt_dlp/extractor/jiosaavn.py    | 79 +++++++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+)
 create mode 100644 yt_dlp/extractor/jiosaavn.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index d57186535..de5a54ec8 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -894,6 +894,10 @@
     SangiinIE,
 )
 from .jeuxvideo import JeuxVideoIE
+from .jiosaavn import (
+    JioSaavnSongIE,
+    JioSaavnAlbumIE,
+)
 from .jove import JoveIE
 from .joj import JojIE
 from .jstream import JStreamIE
diff --git a/yt_dlp/extractor/jiosaavn.py b/yt_dlp/extractor/jiosaavn.py
new file mode 100644
index 000000000..552b73f71
--- /dev/null
+++ b/yt_dlp/extractor/jiosaavn.py
@@ -0,0 +1,79 @@
+from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    url_or_none,
+    urlencode_postdata,
+    urljoin,
+)
+from ..utils.traversal import traverse_obj
+
+
+class JioSaavnBaseIE(InfoExtractor):
+    def _extract_initial_data(self, url, audio_id):
+        webpage = self._download_webpage(url, audio_id)
+        return self._search_json(
+            r'window\.__INITIAL_DATA__\s*=', webpage,
+            'init json', audio_id, transform_source=js_to_json)
+
+
+class JioSaavnSongIE(JioSaavnBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:jiosaavn\.com/song/[^/?#]+/|saavn\.com/s/song/(?:[^/?#]+/){3})(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.jiosaavn.com/song/leja-re/OQsEfQFVUXk',
+        'md5': '7b1f70de088ede3a152ea34aece4df42',
+        'info_dict': {
+            'id': 'OQsEfQFVUXk',
+            'ext': 'mp3',
+            'title': 'Leja Re',
+            'album': 'Leja Re',
+            'thumbnail': 'https://c.saavncdn.com/258/Leja-Re-Hindi-2018-20181124024539-500x500.jpg',
+        },
+    }, {
+        'url': 'https://www.saavn.com/s/song/hindi/Saathiya/O-Humdum-Suniyo-Re/KAMiazoCblU',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        song_data = self._extract_initial_data(url, audio_id)['song']['song']
+        media_data = self._download_json(
+            'https://www.jiosaavn.com/api.php', audio_id, data=urlencode_postdata({
+                '__call': 'song.generateAuthToken',
+                '_format': 'json',
+                'bitrate': '128',
+                'url': song_data['encrypted_media_url'],
+            }))
+
+        return {
+            'id': audio_id,
+            'url': media_data['auth_url'],
+            'ext': media_data.get('type'),
+            'vcodec': 'none',
+            **traverse_obj(song_data, {
+                'title': ('title', 'text'),
+                'album': ('album', 'text'),
+                'thumbnail': ('image', 0, {url_or_none}),
+            }),
+        }
+
+
+class JioSaavnAlbumIE(JioSaavnBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:jio)?saavn\.com/album/[^/?#]+/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.jiosaavn.com/album/96/buIOjYZDrNA_',
+        'info_dict': {
+            'id': 'buIOjYZDrNA_',
+            'title': '96',
+        },
+        'playlist_count': 10,
+    }]
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+        album_view = self._extract_initial_data(url, album_id)['albumView']
+
+        return self.playlist_from_matches(
+            traverse_obj(album_view, (
+                'modules', lambda _, x: x['key'] == 'list', 'data', ..., 'title', 'action', {str})),
+            album_id, traverse_obj(album_view, ('album', 'title', 'text', {str})), ie=JioSaavnSongIE,
+            getter=lambda x: urljoin('https://www.jiosaavn.com/', x))

From 5efe68b73cbf6e907c2e6a3aa338664385084184 Mon Sep 17 00:00:00 2001
From: Boris Nagaev <bnagaev@gmail.com>
Date: Wed, 15 Nov 2023 20:16:54 -0300
Subject: [PATCH 684/871] [ie/ZenYandex] Fix extraction (#8454)

Closes #8275
Authored by: starius
---
 yt_dlp/extractor/yandexvideo.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/yandexvideo.py b/yt_dlp/extractor/yandexvideo.py
index 727250ee8..4382a5684 100644
--- a/yt_dlp/extractor/yandexvideo.py
+++ b/yt_dlp/extractor/yandexvideo.py
@@ -194,7 +194,7 @@ class ZenYandexIE(InfoExtractor):
             'id': '60c7c443da18892ebfe85ed7',
             'ext': 'mp4',
             'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
-            'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89',
+            'description': 'md5:8684912f6086f298f8078d4af0e8a600',
             'thumbnail': 're:^https://avatars.dzeninfra.ru/',
             'uploader': 'AcademeG DailyStream'
         },
@@ -209,7 +209,7 @@ class ZenYandexIE(InfoExtractor):
             'id': '60c7c443da18892ebfe85ed7',
             'ext': 'mp4',
             'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
-            'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89',
+            'description': 'md5:8684912f6086f298f8078d4af0e8a600',
             'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
             'uploader': 'AcademeG DailyStream',
             'upload_date': '20191111',
@@ -258,7 +258,7 @@ def _real_extract(self, url):
             video_id = self._match_id(redirect)
             webpage = self._download_webpage(redirect, video_id, note='Redirecting')
         data_json = self._search_json(
-            r'data\s*=', webpage, 'metadata', video_id, contains_pattern=r'{["\']_*serverState_*video.+}')
+            r'("data"\s*:|data\s*=)', webpage, 'metadata', video_id, contains_pattern=r'{["\']_*serverState_*video.+}')
         serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)',
                                          webpage, 'server state').replace('State', 'Settings')
         uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
@@ -266,22 +266,25 @@ def _real_extract(self, url):
         uploader_name = extract_attributes(uploader).get('aria-label')
         video_json = try_get(data_json, lambda x: x[serverstate]['exportData']['video'], dict)
         stream_urls = try_get(video_json, lambda x: x['video']['streams'])
-        formats = []
+        formats, subtitles = [], {}
         for s_url in stream_urls:
             ext = determine_ext(s_url)
             if ext == 'mpd':
-                formats.extend(self._extract_mpd_formats(s_url, video_id, mpd_id='dash'))
+                fmts, subs = self._extract_mpd_formats_and_subtitles(s_url, video_id, mpd_id='dash')
             elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(s_url, video_id, 'mp4'))
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(s_url, video_id, 'mp4')
+            formats.extend(fmts)
+            subtitles = self._merge_subtitles(subtitles, subs)
         return {
             'id': video_id,
             'title': video_json.get('title') or self._og_search_title(webpage),
             'formats': formats,
+            'subtitles': subtitles,
             'duration': int_or_none(video_json.get('duration')),
             'view_count': int_or_none(video_json.get('views')),
             'timestamp': int_or_none(video_json.get('publicationDate')),
             'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
-            'description': self._og_search_description(webpage) or try_get(data_json, lambda x: x['og']['description']),
+            'description': video_json.get('description') or self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
         }
 
@@ -296,6 +299,7 @@ class ZenYandexChannelIE(InfoExtractor):
             'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
         },
         'playlist_mincount': 169,
+        'skip': 'The page does not exist',
     }, {
         'url': 'https://dzen.ru/tok_media',
         'info_dict': {
@@ -304,6 +308,7 @@ class ZenYandexChannelIE(InfoExtractor):
             'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
         },
         'playlist_mincount': 169,
+        'skip': 'The page does not exist',
     }, {
         'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5',
         'info_dict': {
@@ -318,21 +323,21 @@ class ZenYandexChannelIE(InfoExtractor):
         'url': 'https://zen.yandex.ru/jony_me',
         'info_dict': {
             'id': 'jony_me',
-            'description': 'md5:a2c62b4ef5cf3e3efb13d25f61f739e1',
+            'description': 'md5:ce0a5cad2752ab58701b5497835b2cc5',
             'title': 'JONY ',
         },
-        'playlist_count': 20,
+        'playlist_count': 18,
     }, {
         # Test that the playlist extractor finishes extracting when the
         # channel has more than one page of entries
         'url': 'https://zen.yandex.ru/tatyanareva',
         'info_dict': {
             'id': 'tatyanareva',
-            'description': 'md5:296b588d60841c3756c9105f237b70c6',
+            'description': 'md5:40a1e51f174369ec3ba9d657734ac31f',
             'title': 'Татьяна Рева',
             'entries': 'maxcount:200',
         },
-        'playlist_count': 46,
+        'playlist_mincount': 46,
     }, {
         'url': 'https://dzen.ru/id/606fd806cc13cb3c58c05cf5',
         'info_dict': {
@@ -375,7 +380,7 @@ def _real_extract(self, url):
             item_id = self._match_id(redirect)
             webpage = self._download_webpage(redirect, item_id, note='Redirecting')
         data = self._search_json(
-            r'var\s+data\s*=', webpage, 'channel data', item_id, contains_pattern=r'{\"__serverState__.+}')
+            r'("data"\s*:|data\s*=)', webpage, 'channel data', item_id, contains_pattern=r'{\"__serverState__.+}')
         server_state_json = traverse_obj(data, lambda k, _: k.startswith('__serverState__'), get_all=False)
         server_settings_json = traverse_obj(data, lambda k, _: k.startswith('__serverSettings__'), get_all=False)
 

From a489f071508ec5caf5f32052d142afe86c28df7a Mon Sep 17 00:00:00 2001
From: TravisDupes <32820373+TravisDupes@users.noreply.github.com>
Date: Wed, 15 Nov 2023 18:19:34 -0500
Subject: [PATCH 685/871] [ie/dailymotion] Improve `_VALID_URL` (#7692)

Closes #7601
Authored by: TravisDupes
---
 yt_dlp/extractor/dailymotion.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/dailymotion.py b/yt_dlp/extractor/dailymotion.py
index 21263d41b..708d6fed2 100644
--- a/yt_dlp/extractor/dailymotion.py
+++ b/yt_dlp/extractor/dailymotion.py
@@ -93,7 +93,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
     _VALID_URL = r'''(?ix)
                     https?://
                         (?:
-                            (?:(?:www|touch|geo)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:(?:embed|swf|\#)/)|player\.html\?)?video|swf)|
+                            (?:(?:www|touch|geo)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:(?:embed|swf|\#)/)|player(?:/\w+)?\.html\?)?video|swf)|
                             (?:www\.)?lequipe\.fr/video
                         )
                         [/=](?P<id>[^/?_&]+)(?:.+?\bplaylist=(?P<playlist_id>x[0-9a-z]+))?
@@ -107,13 +107,17 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'id': 'x5kesuj',
             'ext': 'mp4',
             'title': 'Office Christmas Party Review –  Jason Bateman, Olivia Munn, T.J. Miller',
-            'description': 'Office Christmas Party Review -  Jason Bateman, Olivia Munn, T.J. Miller',
+            'description': 'Office Christmas Party Review - Jason Bateman, Olivia Munn, T.J. Miller',
             'duration': 187,
             'timestamp': 1493651285,
             'upload_date': '20170501',
             'uploader': 'Deadline',
             'uploader_id': 'x1xm8ri',
             'age_limit': 0,
+            'view_count': int,
+            'like_count': int,
+            'tags': ['hollywood', 'celeb', 'celebrity', 'movies', 'red carpet'],
+            'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/K456B1aXqIx58LKWQ/x1080',
         },
     }, {
         'url': 'https://geo.dailymotion.com/player.html?video=x89eyek&mute=true',
@@ -132,7 +136,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'view_count': int,
             'like_count': int,
             'tags': ['en_quete_d_esprit'],
-            'thumbnail': 'https://s2.dmcdn.net/v/Tncwi1YGKdvFbDuDY/x1080',
+            'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/Tncwi1YNg_RUl7ueu/x1080',
         }
     }, {
         'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
@@ -201,6 +205,12 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
     }, {
         'url': 'https://www.dailymotion.com/video/x3z49k?playlist=xv4bw',
         'only_matching': True,
+    }, {
+        'url': 'https://geo.dailymotion.com/player/x86gw.html?video=k46oCapRs4iikoz9DWy',
+        'only_matching': True,
+    }, {
+        'url': 'https://geo.dailymotion.com/player/xakln.html?video=x8mjju4&customConfig%5BcustomParams%5D=%2Ffr-fr%2Ftennis%2Fwimbledon-mens-singles%2Farticles-video',
+        'only_matching': True,
     }]
     _GEO_BYPASS = False
     _COMMON_MEDIA_FIELDS = '''description

From e569c2d1f4b665795a2b64f0aaf7f76930664233 Mon Sep 17 00:00:00 2001
From: aarubui <aarubui@users.noreply.github.com>
Date: Thu, 16 Nov 2023 10:21:33 +1100
Subject: [PATCH 686/871] [ie/njpwworld] Remove (#8570)

Authored by: aarubui
---
 supportedsites.md               |  1 -
 yt_dlp/extractor/_extractors.py |  1 -
 yt_dlp/extractor/njpwworld.py   | 82 ---------------------------------
 yt_dlp/extractor/unsupported.py |  8 ++++
 4 files changed, 8 insertions(+), 84 deletions(-)
 delete mode 100644 yt_dlp/extractor/njpwworld.py

diff --git a/supportedsites.md b/supportedsites.md
index 169da5912..0f1e89f03 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -976,7 +976,6 @@ # Supported sites
  - **Nitter**
  - **njoy**: N-JOY
  - **njoy:embed**
- - **NJPWWorld**: [*njpwworld*](## "netrc machine") 新日本プロレスワールド
  - **NobelPrize**
  - **NoicePodcast**
  - **NonkTube**
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index de5a54ec8..38eda2941 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1325,7 +1325,6 @@
 from .ninenow import NineNowIE
 from .nintendo import NintendoIE
 from .nitter import NitterIE
-from .njpwworld import NJPWWorldIE
 from .nobelprize import NobelPrizeIE
 from .noice import NoicePodcastIE
 from .nonktube import NonkTubeIE
diff --git a/yt_dlp/extractor/njpwworld.py b/yt_dlp/extractor/njpwworld.py
deleted file mode 100644
index 607838133..000000000
--- a/yt_dlp/extractor/njpwworld.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    get_element_by_class,
-    urlencode_postdata,
-)
-
-
-class NJPWWorldIE(InfoExtractor):
-    _VALID_URL = r'https?://(front\.)?njpwworld\.com/p/(?P<id>[a-z0-9_]+)'
-    IE_DESC = '新日本プロレスワールド'
-    _NETRC_MACHINE = 'njpwworld'
-
-    _TESTS = [{
-        'url': 'http://njpwworld.com/p/s_series_00155_1_9/',
-        'info_dict': {
-            'id': 's_series_00155_1_9',
-            'ext': 'mp4',
-            'title': '闘強導夢2000 2000年1月4日 東京ドーム 第9試合 ランディ・サベージ VS リック・スタイナー',
-            'tags': list,
-        },
-        'params': {
-            'skip_download': True,  # AES-encrypted m3u8
-        },
-        'skip': 'Requires login',
-    }, {
-        'url': 'https://front.njpwworld.com/p/s_series_00563_16_bs',
-        'info_dict': {
-            'id': 's_series_00563_16_bs',
-            'ext': 'mp4',
-            'title': 'WORLD TAG LEAGUE 2020 & BEST OF THE SUPER Jr.27 2020年12月6日 福岡・福岡国際センター バックステージコメント（字幕あり）',
-            'tags': ["福岡・福岡国際センター", "バックステージコメント", "2020", "20年代"],
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    _LOGIN_URL = 'https://front.njpwworld.com/auth/login'
-
-    def _perform_login(self, username, password):
-        # Setup session (will set necessary cookies)
-        self._request_webpage(
-            'https://njpwworld.com/', None, note='Setting up session')
-
-        webpage, urlh = self._download_webpage_handle(
-            self._LOGIN_URL, None,
-            note='Logging in', errnote='Unable to login',
-            data=urlencode_postdata({'login_id': username, 'pw': password}),
-            headers={'Referer': 'https://front.njpwworld.com/auth'})
-        # /auth/login will return 302 for successful logins
-        if urlh.url == self._LOGIN_URL:
-            self.report_warning('unable to login')
-            return False
-
-        return True
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        formats = []
-        for kind, vid in re.findall(r'if\s+\(\s*imageQualityType\s*==\s*\'([^\']+)\'\s*\)\s*{\s*video_id\s*=\s*"(\d+)"', webpage):
-            player_path = '/intent?id=%s&type=url' % vid
-            player_url = compat_urlparse.urljoin(url, player_path)
-            formats += self._extract_m3u8_formats(
-                player_url, video_id, 'mp4', 'm3u8_native', m3u8_id=kind, fatal=False, quality=int(kind == 'high'))
-
-        tag_block = get_element_by_class('tag-block', webpage)
-        tags = re.findall(
-            r'<a[^>]+class="tag-[^"]+"[^>]*>([^<]+)</a>', tag_block
-        ) if tag_block else None
-
-        return {
-            'id': video_id,
-            'title': get_element_by_class('article-title', webpage) or self._og_search_title(webpage),
-            'formats': formats,
-            'tags': tags,
-        }
diff --git a/yt_dlp/extractor/unsupported.py b/yt_dlp/extractor/unsupported.py
index d610baecb..a3f9911e2 100644
--- a/yt_dlp/extractor/unsupported.py
+++ b/yt_dlp/extractor/unsupported.py
@@ -48,6 +48,7 @@ class KnownDRMIE(UnsupportedInfoExtractor):
         r'joyn\.de',
         r'amazon\.(?:\w{2}\.)?\w+/gp/video',
         r'music\.amazon\.(?:\w{2}\.)?\w+',
+        r'(?:watch|front)\.njpwworld\.com',
     )
 
     _TESTS = [{
@@ -141,6 +142,13 @@ class KnownDRMIE(UnsupportedInfoExtractor):
         # https://github.com/yt-dlp/yt-dlp/issues/5767
         'url': 'https://www.hulu.com/movie/anthem-6b25fac9-da2b-45a3-8e09-e4156b0471cc',
         'only_matching': True,
+    }, {
+        # https://github.com/yt-dlp/yt-dlp/pull/8570
+        'url': 'https://watch.njpwworld.com/player/36447/series?assetType=series',
+        'only_matching': True,
+    }, {
+        'url': 'https://front.njpwworld.com/p/s_series_00563_16_bs',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 2325d03aa7bb80f56ba52cd6992258e44727b424 Mon Sep 17 00:00:00 2001
From: JC-Chung <52159296+JC-Chung@users.noreply.github.com>
Date: Thu, 16 Nov 2023 07:23:18 +0800
Subject: [PATCH 687/871] [ie/twitcasting] Fix livestream detection (#8574)

Authored by: JC-Chung
---
 yt_dlp/extractor/twitcasting.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py
index 32a38c86e..85c7b20de 100644
--- a/yt_dlp/extractor/twitcasting.py
+++ b/yt_dlp/extractor/twitcasting.py
@@ -11,6 +11,7 @@
     float_or_none,
     get_element_by_class,
     get_element_by_id,
+    int_or_none,
     parse_duration,
     qualities,
     str_to_int,
@@ -241,6 +242,8 @@ class TwitCastingLiveIE(InfoExtractor):
         'expected_exception': 'UserNotLive',
     }]
 
+    _PROTECTED_LIVE_RE = r'(?s)(<span\s*class="tw-movie-thumbnail2-badge"\s*data-status="live">\s*LIVE)'
+
     def _real_extract(self, url):
         uploader_id = self._match_id(url)
         self.to_screen(
@@ -248,24 +251,27 @@ def _real_extract(self, url):
             'Pass "https://twitcasting.tv/{0}/show" to download the history'.format(uploader_id))
 
         webpage = self._download_webpage(url, uploader_id)
-        current_live = self._search_regex(
-            (r'data-type="movie" data-id="(\d+)">',
-             r'tw-sound-flag-open-link" data-id="(\d+)" style=',),
-            webpage, 'current live ID', default=None)
-        if not current_live:
+        is_live = self._search_regex(  # first pattern is for public live
+            (r'(data-is-onlive="true")', self._PROTECTED_LIVE_RE), webpage, 'is live?', default=None)
+        current_live = int_or_none(self._search_regex(
+            (r'data-type="movie" data-id="(\d+)">',  # not available?
+             r'tw-sound-flag-open-link" data-id="(\d+)" style=',  # not available?
+             r'data-movie-id="(\d+)"'),  # if not currently live, value may be 0
+            webpage, 'current live ID', default=None))
+        if is_live and not current_live:
             # fetch unfiltered /show to find running livestreams; we can't get ID of the password-protected livestream above
             webpage = self._download_webpage(
                 f'https://twitcasting.tv/{uploader_id}/show/', uploader_id,
                 note='Downloading live history')
-            is_live = self._search_regex(r'(?s)(<span\s*class="tw-movie-thumbnail-badge"\s*data-status="live">\s*LIVE)', webpage, 'is live?', default=None)
+            is_live = self._search_regex(self._PROTECTED_LIVE_RE, webpage, 'is live?', default=None)
             if is_live:
                 # get the first live; running live is always at the first
                 current_live = self._search_regex(
-                    r'(?s)<a\s+class="tw-movie-thumbnail"\s*href="/[^/]+/movie/(?P<video_id>\d+)"\s*>.+?</a>',
+                    r'(?s)<a\s+class="tw-movie-thumbnail2"\s*href="/[^/]+/movie/(?P<video_id>\d+)"\s*>.+?</a>',
                     webpage, 'current live ID 2', default=None, group='video_id')
         if not current_live:
             raise UserNotLive(video_id=uploader_id)
-        return self.url_result('https://twitcasting.tv/%s/movie/%s' % (uploader_id, current_live))
+        return self.url_result(f'https://twitcasting.tv/{uploader_id}/movie/{current_live}', TwitCastingIE)
 
 
 class TwitCastingUserIE(InfoExtractor):

From 15cb3528cbda7b6198f49a6b5953c226d701696b Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Wed, 15 Nov 2023 17:24:55 -0600
Subject: [PATCH 688/871] [ie/abc.net.au:iview:showseries] Fix extraction
 (#8586)

Closes #8554, Closes #8572
Authored by: bashonly
---
 yt_dlp/extractor/abc.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/abc.py b/yt_dlp/extractor/abc.py
index 9d527246a..a7b614ca1 100644
--- a/yt_dlp/extractor/abc.py
+++ b/yt_dlp/extractor/abc.py
@@ -16,6 +16,7 @@
     try_get,
     unescapeHTML,
     update_url_query,
+    url_or_none,
 )
 
 
@@ -379,6 +380,18 @@ class ABCIViewShowSeriesIE(InfoExtractor):
             'noplaylist': True,
             'skip_download': 'm3u8',
         },
+    }, {
+        # 'videoEpisodes' is a dict with `items` key
+        'url': 'https://iview.abc.net.au/show/7-30-mark-humphries-satire',
+        'info_dict': {
+            'id': '178458-0',
+            'title': 'Episodes',
+            'description': 'Satirist Mark Humphries brings his unique perspective on current political events for 7.30.',
+            'series': '7.30 Mark Humphries Satire',
+            'season': 'Episodes',
+            'thumbnail': r're:^https?://cdn\.iview\.abc\.net\.au/thumbs/.*\.jpg$'
+        },
+        'playlist_count': 15,
     }]
 
     def _real_extract(self, url):
@@ -398,12 +411,14 @@ def _real_extract(self, url):
         series = video_data['selectedSeries']
         return {
             '_type': 'playlist',
-            'entries': [self.url_result(episode['shareUrl'])
-                        for episode in series['_embedded']['videoEpisodes']],
+            'entries': [self.url_result(episode_url, ABCIViewIE)
+                        for episode_url in traverse_obj(series, (
+                            '_embedded', 'videoEpisodes', (None, 'items'), ..., 'shareUrl', {url_or_none}))],
             'id': series.get('id'),
             'title': dict_get(series, ('title', 'displaySubtitle')),
             'description': series.get('description'),
             'series': dict_get(series, ('showTitle', 'displayTitle')),
             'season': dict_get(series, ('title', 'displaySubtitle')),
-            'thumbnail': series.get('thumbnail'),
+            'thumbnail': traverse_obj(
+                series, 'thumbnail', ('images', lambda _, v: v['name'] == 'seriesThumbnail', 'url'), get_all=False),
         }

From 24f827875c6ba513f12ed09a3aef2bbed223760d Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Wed, 15 Nov 2023 17:31:32 -0600
Subject: [PATCH 689/871] [build] Make `secretstorage` an optional dependency
 (#8585)

Authored by: bashonly
---
 .github/workflows/build.yml   | 4 ++--
 .github/workflows/release.yml | 4 ++--
 requirements.txt              | 1 -
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3b513e88e..a52973ea2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -204,11 +204,11 @@ jobs:
             apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
             python3.8 -m pip install -U pip setuptools wheel
             # Cannot access requirements.txt from the repo directory at this stage
-            python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi
+            python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage
 
           run: |
             cd repo
-            python3.8 -m pip install -U Pyinstaller -r requirements.txt  # Cached version may be out of date
+            python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt  # Cached version may be out of date
             python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
             python3.8 devscripts/make_lazy_extractors.py
             python3.8 pyinst.py
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6c59626ea..84e892ffe 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -216,8 +216,8 @@ jobs:
         if: |
           !inputs.prerelease && env.target_repo == github.repository
         run: |
-          git config --global user.name github-actions
-          git config --global user.email github-actions@github.com
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git add -u
           git commit -m "Release ${{ env.version }}" \
             -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
diff --git a/requirements.txt b/requirements.txt
index 4d2310725..5b6270a7d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,3 @@ brotlicffi; implementation_name!='cpython'
 certifi
 requests>=2.31.0,<3
 urllib3>=1.26.17,<3
-secretstorage; sys_platform=='linux' and (implementation_name!='pypy' or implementation_version>='7.3.10')

From fe6c82ccff6338d97711204162731a8a6741c9b4 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 16 Nov 2023 00:01:38 +0000
Subject: [PATCH 690/871] Release 2023.11.16

Created by: bashonly

:ci skip all :ci run dl
---
 CONTRIBUTORS      |  4 ++++
 Changelog.md      | 17 +++++++++++++++++
 supportedsites.md |  3 +++
 yt_dlp/version.py |  6 +++---
 4 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 71752c20a..8b6b3671e 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -524,3 +524,7 @@ peci1
 saintliao
 shubhexists
 SirElderling
+almx
+elivinsky
+starius
+TravisDupes
diff --git a/Changelog.md b/Changelog.md
index a64648120..6115446cb 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,23 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.11.16
+
+#### Extractor changes
+- **abc.net.au**: iview, showseries: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/15cb3528cbda7b6198f49a6b5953c226d701696b) ([#8586](https://github.com/yt-dlp/yt-dlp/issues/8586)) by [bashonly](https://github.com/bashonly)
+- **beatbump**: [Update `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/21dc069bea2d4d99345dd969e098f4535c751d45) ([#8576](https://github.com/yt-dlp/yt-dlp/issues/8576)) by [seproDev](https://github.com/seproDev)
+- **dailymotion**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/a489f071508ec5caf5f32052d142afe86c28df7a) ([#7692](https://github.com/yt-dlp/yt-dlp/issues/7692)) by [TravisDupes](https://github.com/TravisDupes)
+- **drtv**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/0783fd558ed0d3a8bc754beb75a406256f8b97b2) ([#8484](https://github.com/yt-dlp/yt-dlp/issues/8484)) by [almx](https://github.com/almx), [seproDev](https://github.com/seproDev)
+- **eltrecetv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/dcfad52812aa8ce007cefbfbe63f58b49f6b1046) ([#8216](https://github.com/yt-dlp/yt-dlp/issues/8216)) by [elivinsky](https://github.com/elivinsky)
+- **jiosaavn**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/b530118e7f48232cacf8050d79a6b20bdfcf5468) ([#8307](https://github.com/yt-dlp/yt-dlp/issues/8307)) by [awalgarg](https://github.com/awalgarg)
+- **njpwworld**: [Remove](https://github.com/yt-dlp/yt-dlp/commit/e569c2d1f4b665795a2b64f0aaf7f76930664233) ([#8570](https://github.com/yt-dlp/yt-dlp/issues/8570)) by [aarubui](https://github.com/aarubui)
+- **tv5mondeplus**: [Extract subtitles](https://github.com/yt-dlp/yt-dlp/commit/0f634dba3afdc429ece8839b02f6d56c27b7973a) ([#4209](https://github.com/yt-dlp/yt-dlp/issues/4209)) by [FrankZ85](https://github.com/FrankZ85)
+- **twitcasting**: [Fix livestream detection](https://github.com/yt-dlp/yt-dlp/commit/2325d03aa7bb80f56ba52cd6992258e44727b424) ([#8574](https://github.com/yt-dlp/yt-dlp/issues/8574)) by [JC-Chung](https://github.com/JC-Chung)
+- **zenyandex**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/5efe68b73cbf6e907c2e6a3aa338664385084184) ([#8454](https://github.com/yt-dlp/yt-dlp/issues/8454)) by [starius](https://github.com/starius)
+
+#### Misc. changes
+- **build**: [Make `secretstorage` an optional dependency](https://github.com/yt-dlp/yt-dlp/commit/24f827875c6ba513f12ed09a3aef2bbed223760d) ([#8585](https://github.com/yt-dlp/yt-dlp/issues/8585)) by [bashonly](https://github.com/bashonly)
+
 ### 2023.11.14
 
 #### Important changes
diff --git a/supportedsites.md b/supportedsites.md
index 0f1e89f03..0e971c135 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -414,6 +414,7 @@ # Supported sites
  - **EllenTubeVideo**
  - **Elonet**
  - **ElPais**: El País
+ - **ElTreceTV**: El Trece TV (Argentina)
  - **Embedly**
  - **EMPFlix**
  - **Engadget**
@@ -654,6 +655,8 @@ # Supported sites
  - **Jamendo**
  - **JamendoAlbum**
  - **JeuxVideo**
+ - **JioSaavnAlbum**
+ - **JioSaavnSong**
  - **Joj**
  - **Jove**
  - **JStream**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index 6fd8e5978..fd923fe45 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.11.14'
+__version__ = '2023.11.16'
 
-RELEASE_GIT_HEAD = 'a9d3f4b20a3533d2a40104c85bc2cc6c2564c800'
+RELEASE_GIT_HEAD = '24f827875c6ba513f12ed09a3aef2bbed223760d'
 
 VARIANT = None
 
@@ -12,4 +12,4 @@
 
 ORIGIN = 'yt-dlp/yt-dlp'
 
-_pkg_version = '2023.11.14'
+_pkg_version = '2023.11.16'

From f4b95acafcd69a50040730dfdf732e797278fdcc Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 16 Nov 2023 12:39:00 -0600
Subject: [PATCH 691/871] Remove Python 3.7 support (#8361)

Closes #7803
Authored by: bashonly
---
 .github/workflows/build.yml      |  15 ++++-
 .github/workflows/core.yml       |   6 +-
 .github/workflows/download.yml   |   8 +--
 .github/workflows/quick-test.yml |   2 +-
 CONTRIBUTING.md                  |   2 +-
 README.md                        |   6 +-
 setup.cfg                        |   4 +-
 setup.py                         |   4 +-
 test/test_update.py              | 108 ++++++++++++++++++-------------
 yt_dlp/__init__.py               |  11 ++--
 yt_dlp/compat/functools.py       |  14 ----
 yt_dlp/update.py                 |  14 +---
 12 files changed, 100 insertions(+), 94 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a52973ea2..d944659b8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -377,8 +377,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
-        with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390
-          python-version: "3.7"
+        with:
+          python-version: "3.8"
           architecture: "x86"
       - name: Install Requirements
         run: |
@@ -436,7 +436,16 @@ jobs:
         run: |
           cat >> _update_spec << EOF
           # This file is used for regulating self-update
-          lock 2022.08.18.36 .+ Python 3.6
+          lock 2022.08.18.36 .+ Python 3\.6
+          lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7
+          lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
+          lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6
+          lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7
+          lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
+          lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7
+          lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server)
+          lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7
+          lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server)
           EOF
 
       - name: Sign checksum files
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index e5a976de5..3c10fc17e 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -13,12 +13,12 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         # CPython 3.11 is in quick-test
-        python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.7, pypy-3.8, pypy-3.10]
+        python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.8, pypy-3.10]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
         - os: windows-latest
-          python-version: '3.7'
+          python-version: '3.8'
           run-tests-ext: bat
         - os: windows-latest
           python-version: '3.12'
@@ -32,7 +32,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install test requirements
       run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: False
diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml
index 7302a93bc..73b2f9ca3 100644
--- a/.github/workflows/download.yml
+++ b/.github/workflows/download.yml
@@ -15,7 +15,7 @@ jobs:
       with:
         python-version: 3.9
     - name: Install test requirements
-      run: pip install pytest
+      run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: true
       run: ./devscripts/run_tests.sh download
@@ -28,7 +28,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.7', '3.10', '3.12', pypy-3.7, pypy-3.8, pypy-3.10]
+        python-version: ['3.10', '3.11', '3.12', pypy-3.8, pypy-3.10]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
@@ -44,8 +44,8 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install pytest
-      run: pip install pytest
+    - name: Install test requirements
+      run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: true
       run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} download
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index e4fd89551..edbdaffd7 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -15,7 +15,7 @@ jobs:
       with:
         python-version: '3.11'
     - name: Install test requirements
-      run: pip install pytest pycryptodomex
+      run: pip install pytest -r requirements.txt
     - name: Run tests
       run: |
         python3 -m yt_dlp -v || true
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 90e7faf7c..c472f3251 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -222,7 +222,7 @@ ## Adding support for a new site
 
         $ flake8 yt_dlp/extractor/yourextractor.py
 
-1. Make sure your code works under all [Python](https://www.python.org/) versions supported by yt-dlp, namely CPython and PyPy for Python 3.7 and above. Backward compatibility is not required for even older versions of Python.
+1. Make sure your code works under all [Python](https://www.python.org/) versions supported by yt-dlp, namely CPython and PyPy for Python 3.8 and above. Backward compatibility is not required for even older versions of Python.
 1. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files, [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
 
         $ git add yt_dlp/extractor/_extractors.py
diff --git a/README.md b/README.md
index 33690f470..6fe7fab6a 100644
--- a/README.md
+++ b/README.md
@@ -131,7 +131,7 @@ ### Differences in default behavior
 
 Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc:
 
-* yt-dlp supports only [Python 3.7+](## "Windows 7"), and *may* remove support for more versions as they [become EOL](https://devguide.python.org/versions/#python-release-cycle); while [youtube-dl still supports Python 2.6+ and 3.2+](https://github.com/ytdl-org/youtube-dl/issues/30568#issue-1118238743)
+* yt-dlp supports only [Python 3.8+](## "Windows 7"), and *may* remove support for more versions as they [become EOL](https://devguide.python.org/versions/#python-release-cycle); while [youtube-dl still supports Python 2.6+ and 3.2+](https://github.com/ytdl-org/youtube-dl/issues/30568#issue-1118238743)
 * The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
 * `avconv` is not supported as an alternative to `ffmpeg`
 * yt-dlp stores config files in slightly different locations to youtube-dl. See [CONFIGURATION](#configuration) for a list of correct locations
@@ -266,7 +266,7 @@ #### Misc
 **Note**: The manpages, shell completion (autocomplete) files etc. are available inside the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
 
 ## DEPENDENCIES
-Python versions 3.7+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly.
+Python versions 3.8+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly.
 
 <!-- Python 3.5+ uses VC++14 and it is already embedded in the binary created
 <!x-- https://www.microsoft.com/en-us/download/details.aspx?id=26999 --x>
@@ -334,7 +334,7 @@ ### Standalone PyInstaller Builds
 **Important**: Running `pyinstaller` directly **without** using `pyinst.py` is **not** officially supported. This may or may not work correctly.
 
 ### Platform-independent Binary (UNIX)
-You will need the build tools `python` (3.7+), `zip`, `make` (GNU), `pandoc`\* and `pytest`\*.
+You will need the build tools `python` (3.8+), `zip`, `make` (GNU), `pandoc`\* and `pytest`\*.
 
 After installing these, simply run `make`.
 
diff --git a/setup.cfg b/setup.cfg
index 6deaa7971..a799f7293 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -26,7 +26,7 @@ markers =
 
 [tox:tox]
 skipsdist = true
-envlist = py{36,37,38,39,310,311},pypy{36,37,38,39}
+envlist = py{38,39,310,311,312},pypy{38,39,310}
 skip_missing_interpreters = true
 
 [testenv]  # tox
@@ -39,7 +39,7 @@ setenv =
 
 
 [isort]
-py_version = 37
+py_version = 38
 multi_line_output = VERTICAL_HANGING_INDENT
 line_length = 80
 reverse_relative = true
diff --git a/setup.py b/setup.py
index 44055b0e9..3d9a69d10 100644
--- a/setup.py
+++ b/setup.py
@@ -152,7 +152,7 @@ def main():
         url='https://github.com/yt-dlp/yt-dlp',
         packages=packages(),
         install_requires=REQUIREMENTS,
-        python_requires='>=3.7',
+        python_requires='>=3.8',
         project_urls={
             'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
             'Source': 'https://github.com/yt-dlp/yt-dlp',
@@ -164,11 +164,11 @@ def main():
             'Development Status :: 5 - Production/Stable',
             'Environment :: Console',
             'Programming Language :: Python',
-            'Programming Language :: Python :: 3.7',
             'Programming Language :: Python :: 3.8',
             'Programming Language :: Python :: 3.9',
             'Programming Language :: Python :: 3.10',
             'Programming Language :: Python :: 3.11',
+            'Programming Language :: Python :: 3.12',
             'Programming Language :: Python :: Implementation',
             'Programming Language :: Python :: Implementation :: CPython',
             'Programming Language :: Python :: Implementation :: PyPy',
diff --git a/test/test_update.py b/test/test_update.py
index 134424a31..2a5647e44 100644
--- a/test/test_update.py
+++ b/test/test_update.py
@@ -68,25 +68,34 @@
     },
 }
 
-TEST_LOCKFILE_V1 = '''# This file is used for regulating self-update
-lock 2022.08.18.36 .+ Python 3.6
-lock 2023.11.13 .+ Python 3.7
+TEST_LOCKFILE_COMMENT = '# This file is used for regulating self-update'
+
+TEST_LOCKFILE_V1 = r'''%s
+lock 2022.08.18.36 .+ Python 3\.6
+lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7
+lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
+''' % TEST_LOCKFILE_COMMENT
+
+TEST_LOCKFILE_V2_TMPL = r'''%s
+lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6
+lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7
+lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
+lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7
+lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server)
+lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7
+lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server)
 '''
 
-TEST_LOCKFILE_V2 = '''# This file is used for regulating self-update
-lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3.6
-lockV2 yt-dlp/yt-dlp 2023.11.13 .+ Python 3.7
-'''
+TEST_LOCKFILE_V2 = TEST_LOCKFILE_V2_TMPL % TEST_LOCKFILE_COMMENT
 
-TEST_LOCKFILE_V1_V2 = '''# This file is used for regulating self-update
-lock 2022.08.18.36 .+ Python 3.6
-lock 2023.11.13 .+ Python 3.7
-lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3.6
-lockV2 yt-dlp/yt-dlp 2023.11.13 .+ Python 3.7
+TEST_LOCKFILE_ACTUAL = TEST_LOCKFILE_V2_TMPL % TEST_LOCKFILE_V1.rstrip('\n')
+
+TEST_LOCKFILE_FORK = r'''%s# Test if a fork blocks updates to non-numeric tags
 lockV2 fork/yt-dlp pr0000 .+ Python 3.6
-lockV2 fork/yt-dlp pr1234 .+ Python 3.7
+lockV2 fork/yt-dlp pr1234 (?!win_x86_exe).+ Python 3\.7
+lockV2 fork/yt-dlp pr1234 win_x86_exe .+ Windows-(?:Vista|2008Server)
 lockV2 fork/yt-dlp pr9999 .+ Python 3.11
-'''
+''' % TEST_LOCKFILE_ACTUAL
 
 
 class FakeUpdater(Updater):
@@ -97,7 +106,7 @@ class FakeUpdater(Updater):
     _origin = 'yt-dlp/yt-dlp'
 
     def _download_update_spec(self, *args, **kwargs):
-        return TEST_LOCKFILE_V1_V2
+        return TEST_LOCKFILE_ACTUAL
 
     def _call_api(self, tag):
         tag = f'tags/{tag}' if tag != 'latest' else tag
@@ -112,7 +121,7 @@ class TestUpdate(unittest.TestCase):
 
     def test_update_spec(self):
         ydl = FakeYDL()
-        updater = FakeUpdater(ydl, 'stable@latest')
+        updater = FakeUpdater(ydl, 'stable')
 
         def test(lockfile, identifier, input_tag, expect_tag, exact=False, repo='yt-dlp/yt-dlp'):
             updater._identifier = identifier
@@ -124,35 +133,46 @@ def test(lockfile, identifier, input_tag, expect_tag, exact=False, repo='yt-dlp/
                 f'{identifier!r} requesting {repo}@{input_tag} (exact={exact}) '
                 f'returned {result!r} instead of {expect_tag!r}')
 
-        test(TEST_LOCKFILE_V1, 'zip Python 3.11.0', '2023.11.13', '2023.11.13')
-        test(TEST_LOCKFILE_V1, 'zip stable Python 3.11.0', '2023.11.13', '2023.11.13', exact=True)
-        test(TEST_LOCKFILE_V1, 'zip Python 3.6.0', '2023.11.13', '2022.08.18.36')
-        test(TEST_LOCKFILE_V1, 'zip stable Python 3.6.0', '2023.11.13', None, exact=True)
-        test(TEST_LOCKFILE_V1, 'zip Python 3.7.0', '2023.11.13', '2023.11.13')
-        test(TEST_LOCKFILE_V1, 'zip stable Python 3.7.1', '2023.11.13', '2023.11.13')
-        test(TEST_LOCKFILE_V1, 'zip Python 3.7.1', '2023.12.31', '2023.11.13')
-        test(TEST_LOCKFILE_V1, 'zip stable Python 3.7.1', '2023.12.31', '2023.11.13')
+        for lockfile in (TEST_LOCKFILE_V1, TEST_LOCKFILE_V2, TEST_LOCKFILE_ACTUAL, TEST_LOCKFILE_FORK):
+            # Normal operation
+            test(lockfile, 'zip Python 3.12.0', '2023.12.31', '2023.12.31')
+            test(lockfile, 'zip stable Python 3.12.0', '2023.12.31', '2023.12.31', exact=True)
+            # Python 3.6 --update should update only to its lock
+            test(lockfile, 'zip Python 3.6.0', '2023.11.16', '2022.08.18.36')
+            # --update-to an exact version later than the lock should return None
+            test(lockfile, 'zip stable Python 3.6.0', '2023.11.16', None, exact=True)
+            # Python 3.7 should be able to update to its lock
+            test(lockfile, 'zip Python 3.7.0', '2023.11.16', '2023.11.16')
+            test(lockfile, 'zip stable Python 3.7.1', '2023.11.16', '2023.11.16', exact=True)
+            # Non-win_x86_exe builds on py3.7 must be locked
+            test(lockfile, 'zip Python 3.7.1', '2023.12.31', '2023.11.16')
+            test(lockfile, 'zip stable Python 3.7.1', '2023.12.31', None, exact=True)
+            test(  # Windows Vista w/ win_x86_exe must be locked
+                lockfile, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-Vista-6.0.6003-SP2',
+                '2023.12.31', '2023.11.16')
+            test(  # Windows 2008Server w/ win_x86_exe must be locked
+                lockfile, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-2008Server',
+                '2023.12.31', None, exact=True)
+            test(  # Windows 7 w/ win_x86_exe py3.7 build should be able to update beyond lock
+                lockfile, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-7-6.1.7601-SP1',
+                '2023.12.31', '2023.12.31')
+            test(  # Windows 8.1 w/ '2008Server' in platform string should be able to update beyond lock
+                lockfile, 'win_x86_exe Python 3.7.9 (CPython x86 32bit) - Windows-post2008Server-6.2.9200',
+                '2023.12.31', '2023.12.31', exact=True)
 
-        test(TEST_LOCKFILE_V2, 'zip Python 3.11.1', '2023.11.13', '2023.11.13')
-        test(TEST_LOCKFILE_V2, 'zip stable Python 3.11.1', '2023.12.31', '2023.12.31')
-        test(TEST_LOCKFILE_V2, 'zip Python 3.6.1', '2023.11.13', '2022.08.18.36')
-        test(TEST_LOCKFILE_V2, 'zip stable Python 3.7.2', '2023.11.13', '2023.11.13')
-        test(TEST_LOCKFILE_V2, 'zip Python 3.7.2', '2023.12.31', '2023.11.13')
-
-        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.2', '2023.11.13', '2023.11.13')
-        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.11.2', '2023.12.31', '2023.12.31')
-        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.2', '2023.11.13', '2022.08.18.36')
-        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.7.3', '2023.11.13', '2023.11.13')
-        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.3', '2023.12.31', '2023.11.13')
-        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.3', 'pr0000', None, repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.7.4', 'pr0000', 'pr0000', repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.6.4', 'pr0000', None, repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.4', 'pr1234', None, repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.8.1', 'pr1234', 'pr1234', repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.7.5', 'pr1234', None, repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.3', 'pr9999', None, repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_V1_V2, 'zip stable Python 3.12.0', 'pr9999', 'pr9999', repo='fork/yt-dlp')
-        test(TEST_LOCKFILE_V1_V2, 'zip Python 3.11.4', 'pr9999', None, repo='fork/yt-dlp')
+        # Forks can block updates to non-numeric tags rather than lock
+        test(TEST_LOCKFILE_FORK, 'zip Python 3.6.3', 'pr0000', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_FORK, 'zip stable Python 3.7.4', 'pr0000', 'pr0000', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_FORK, 'zip stable Python 3.7.4', 'pr1234', None, repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_FORK, 'zip Python 3.8.1', 'pr1234', 'pr1234', repo='fork/yt-dlp', exact=True)
+        test(
+            TEST_LOCKFILE_FORK, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-Vista-6.0.6003-SP2',
+            'pr1234', None, repo='fork/yt-dlp')
+        test(
+            TEST_LOCKFILE_FORK, 'win_x86_exe stable Python 3.7.9 (CPython x86 32bit) - Windows-7-6.1.7601-SP1',
+            '2023.12.31', '2023.12.31', repo='fork/yt-dlp')
+        test(TEST_LOCKFILE_FORK, 'zip Python 3.11.2', 'pr9999', None, repo='fork/yt-dlp', exact=True)
+        test(TEST_LOCKFILE_FORK, 'zip stable Python 3.12.0', 'pr9999', 'pr9999', repo='fork/yt-dlp')
 
     def test_query_update(self):
         ydl = FakeYDL()
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 991dbcda7..446f0c47b 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -1,8 +1,8 @@
-try:
-    import contextvars  # noqa: F401
-except Exception:
-    raise Exception(
-        f'You are using an unsupported version of Python. Only Python versions 3.7 and above are supported by yt-dlp')  # noqa: F541
+import sys
+
+if sys.version_info < (3, 8):
+    raise ImportError(
+        f'You are using an unsupported version of Python. Only Python versions 3.8 and above are supported by yt-dlp')  # noqa: F541
 
 __license__ = 'Public Domain'
 
@@ -12,7 +12,6 @@
 import optparse
 import os
 import re
-import sys
 import traceback
 
 from .compat import compat_shlex_quote
diff --git a/yt_dlp/compat/functools.py b/yt_dlp/compat/functools.py
index ec003ea90..36c983642 100644
--- a/yt_dlp/compat/functools.py
+++ b/yt_dlp/compat/functools.py
@@ -10,17 +10,3 @@
     cache  # >= 3.9
 except NameError:
     cache = lru_cache(maxsize=None)
-
-try:
-    cached_property  # >= 3.8
-except NameError:
-    class cached_property:
-        def __init__(self, func):
-            update_wrapper(self, func)
-            self.func = func
-
-        def __get__(self, instance, _):
-            if instance is None:
-                return self
-            setattr(instance, self.func.__name__, self.func(instance))
-            return getattr(instance, self.func.__name__)
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index 85c9bb962..f99583b08 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -131,7 +131,7 @@ def _get_binary_name():
 
 
 def _get_system_deprecation():
-    MIN_SUPPORTED, MIN_RECOMMENDED = (3, 7), (3, 8)
+    MIN_SUPPORTED, MIN_RECOMMENDED = (3, 8), (3, 8)
 
     if sys.version_info > MIN_RECOMMENDED:
         return None
@@ -140,16 +140,8 @@ def _get_system_deprecation():
     if sys.version_info < MIN_SUPPORTED:
         msg = f'Python version {major}.{minor} is no longer supported'
     else:
-        msg = f'Support for Python version {major}.{minor} has been deprecated. '
-        # Temporary until `win_x86_exe` uses 3.8, which will deprecate Vista and Server 2008
-        if detect_variant() == 'win_x86_exe':
-            platform_name = platform.platform()
-            if any(platform_name.startswith(f'Windows-{name}') for name in ('Vista', '2008Server')):
-                msg = 'Support for Windows Vista/Server 2008 has been deprecated. '
-            else:
-                return None
-        msg += ('See  https://github.com/yt-dlp/yt-dlp/issues/7803  for details.'
-                '\nYou may stop receiving updates on this version at any time')
+        msg = (f'Support for Python version {major}.{minor} has been deprecated. '
+               '\nYou may stop receiving updates on this version at any time')
 
     major, minor = MIN_RECOMMENDED
     return f'{msg}! Please update to Python {major}.{minor} or above'

From f96ab86cd837b1b5823baa87d144e15322ee9298 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Thu, 16 Nov 2023 21:46:13 +0100
Subject: [PATCH 692/871] [ie/drtv] Set default ext for m3u8 formats (#8590)

Closes #8589
Authored by: seproDev
---
 yt_dlp/extractor/drtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/drtv.py b/yt_dlp/extractor/drtv.py
index 6d5f3f649..2a6e337bf 100644
--- a/yt_dlp/extractor/drtv.py
+++ b/yt_dlp/extractor/drtv.py
@@ -209,7 +209,7 @@ def _real_extract(self, url):
             elif access_service == 'StandardVideo':
                 preference = 1
             fmts, subs = self._extract_m3u8_formats_and_subtitles(
-                stream.get('url'), video_id, preference=preference, m3u8_id=format_id, fatal=False)
+                stream.get('url'), video_id, ext='mp4', preference=preference, m3u8_id=format_id, fatal=False)
             formats.extend(fmts)
 
             api_subtitles = traverse_obj(stream, ('subtitles', lambda _, v: url_or_none(v['link']), {dict}))

From 64de1a4c25bada90374b88d7353754fe8fbfcc51 Mon Sep 17 00:00:00 2001
From: Ha Tien Loi <hatienloi261299@gmail.com>
Date: Fri, 17 Nov 2023 05:08:00 +0700
Subject: [PATCH 693/871] [ie/zingmp3] Add support for radio and podcasts
 (#7189)

Authored by: hatienl0i261299
---
 yt_dlp/extractor/_extractors.py |   3 +
 yt_dlp/extractor/zingmp3.py     | 254 ++++++++++++++++++++++++++++----
 2 files changed, 232 insertions(+), 25 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 38eda2941..63d1f2c1c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2588,6 +2588,9 @@
     ZingMp3ChartMusicVideoIE,
     ZingMp3UserIE,
     ZingMp3HubIE,
+    ZingMp3LiveRadioIE,
+    ZingMp3PodcastEpisodeIE,
+    ZingMp3PodcastIE,
 )
 from .zoom import ZoomIE
 from .zype import ZypeIE
diff --git a/yt_dlp/extractor/zingmp3.py b/yt_dlp/extractor/zingmp3.py
index 007658c65..f664d88d8 100644
--- a/yt_dlp/extractor/zingmp3.py
+++ b/yt_dlp/extractor/zingmp3.py
@@ -5,7 +5,15 @@
 import urllib.parse
 
 from .common import InfoExtractor
-from ..utils import int_or_none, traverse_obj, try_call, urljoin
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    join_nonempty,
+    try_call,
+    urljoin,
+    url_or_none
+)
+from ..utils.traversal import traverse_obj
 
 
 class ZingMp3BaseIE(InfoExtractor):
@@ -20,9 +28,17 @@ class ZingMp3BaseIE(InfoExtractor):
         'video-clip': '/api/v2/page/get/video',
         'lyric': '/api/v2/lyric/get/lyric',
         'song-streaming': '/api/v2/song/get/streaming',
+        'liveradio': '/api/v2/livestream/get/info',
+        'eps': '/api/v2/page/get/podcast-episode',
+        'episode-streaming': '/api/v2/podcast/episode/get/streaming',
         # Playlist
         'playlist': '/api/v2/page/get/playlist',
         'album': '/api/v2/page/get/playlist',
+        'pgr': '/api/v2/page/get/podcast-program',
+        'pgr-list': '/api/v2/podcast/episode/get/list',
+        'cgr': '/api/v2/page/get/podcast-category',
+        'cgr-list': '/api/v2/podcast/program/get/list-by-cate',
+        'cgrs': '/api/v2/page/get/podcast-categories',
         # Chart
         'zing-chart': '/api/v2/page/get/chart-home',
         'zing-chart-tuan': '/api/v2/page/get/week-chart',
@@ -33,6 +49,10 @@ class ZingMp3BaseIE(InfoExtractor):
         'user-list-song': '/api/v2/song/get/list',
         'user-list-video': '/api/v2/video/get/list',
         'hub': '/api/v2/page/get/hub-detail',
+        'new-release': '/api/v2/chart/get/new-release',
+        'top100': '/api/v2/page/get/top-100',
+        'podcast-new': '/api/v2/podcast/program/get/list-by-type',
+        'top-podcast': '/api/v2/podcast/program/get/top-episode',
     }
 
     def _api_url(self, url_type, params):
@@ -78,7 +98,7 @@ def _paged_list(self, _id, url_type):
 
 
 class ZingMp3IE(ZingMp3BaseIE):
-    _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'bai-hat|video-clip|embed'
+    _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'bai-hat|video-clip|embed|eps'
     IE_NAME = 'zingmp3'
     IE_DESC = 'zingmp3.vn'
     _TESTS = [{
@@ -102,7 +122,7 @@ class ZingMp3IE(ZingMp3BaseIE):
         },
     }, {
         'url': 'https://zingmp3.vn/video-clip/Suong-Hoa-Dua-Loi-K-ICM-RYO/ZO8ZF7C7.html',
-        'md5': '3c2081e79471a2f4a3edd90b70b185ea',
+        'md5': '92c6e7a019f06b4682a6c35ae5785fab',
         'info_dict': {
             'id': 'ZO8ZF7C7',
             'title': 'Sương Hoa Đưa Lối',
@@ -128,6 +148,20 @@ class ZingMp3IE(ZingMp3BaseIE):
             'album': 'Người Yêu Tôi Lạnh Lùng Sắt Đá (Single)',
             'album_artist': 'Mr. Siro',
         },
+    }, {
+        'url': 'https://zingmp3.vn/eps/Cham-x-Ban-Noi-Goi-La-Nha/ZZD9ACWI.html',
+        'md5': 'd52f9f63e2631e004e4f15188eedcf80',
+        'info_dict': {
+            'id': 'ZZD9ACWI',
+            'title': 'Chạm x Bạn - Nơi Gọi Là Nhà',
+            'ext': 'mp3',
+            'duration': 3716,
+            'thumbnail': r're:^https?://.+\.jpg',
+            'track': 'Chạm x Bạn - Nơi Gọi Là Nhà',
+            'artist': 'On Air',
+            'album': 'Top Podcast',
+            'album_artist': 'On Air',
+        },
     }, {
         'url': 'https://zingmp3.vn/embed/song/ZWZEI76B?start=false',
         'only_matching': True,
@@ -147,6 +181,8 @@ def _real_extract(self, url):
                 'http://api.mp3.zing.vn/api/mobile/video/getvideoinfo', item_id,
                 query={'requestdata': json.dumps({'id': item_id})},
                 note='Downloading mp4 JSON metadata').get('source')
+        elif url_type == 'eps':
+            source = self._call_api('episode-streaming', {'id': item_id})
         else:
             source = self._call_api('song-streaming', {'id': item_id})
 
@@ -189,9 +225,10 @@ def _real_extract(self, url):
             'thumbnail': traverse_obj(item, 'thumbnail', 'thumbnailM'),
             'duration': int_or_none(item.get('duration')),
             'track': traverse_obj(item, 'title', 'alias'),
-            'artist': traverse_obj(item, 'artistsNames', 'artists_names'),
-            'album': traverse_obj(item, ('album', ('name', 'title')), get_all=False),
-            'album_artist': traverse_obj(item, ('album', ('artistsNames', 'artists_names')), get_all=False),
+            'artist': traverse_obj(item, 'artistsNames', 'artists_names', ('artists', 0, 'name')),
+            'album': traverse_obj(item, ('album', ('name', 'title')), ('genres', 0, 'name'), get_all=False),
+            'album_artist': traverse_obj(item, ('album', ('artistsNames', 'artists_names')),
+                                         ('artists', 0, 'name'), get_all=False),
             'formats': formats,
             'subtitles': {'origin': [{'url': lyric}]} if lyric else None,
         }
@@ -200,12 +237,12 @@ def _real_extract(self, url):
 class ZingMp3AlbumIE(ZingMp3BaseIE):
     _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'album|playlist'
     _TESTS = [{
-        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'url': 'https://zingmp3.vn/album/Ca-Phe-Quan-Quen-Hoang-Dung-My-Anh-Da-LAB-Thinh-Suy/ZOC7WUZC.html',
         'info_dict': {
-            'id': 'ZWZBWDAF',
-            'title': 'Lâu Đài Tình Ái',
+            'id': 'ZOC7WUZC',
+            'title': 'Cà Phê Quán Quen',
         },
-        'playlist_mincount': 9,
+        'playlist_mincount': 10,
     }, {
         'url': 'https://zingmp3.vn/album/Nhung-Bai-Hat-Hay-Nhat-Cua-Mr-Siro-Mr-Siro/ZWZAEZZD.html',
         'info_dict': {
@@ -231,7 +268,7 @@ def _real_extract(self, url):
 
 
 class ZingMp3ChartHomeIE(ZingMp3BaseIE):
-    _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<id>(?:zing-chart|moi-phat-hanh))/?(?:[#?]|$)'
+    _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<id>(?:zing-chart|moi-phat-hanh|top100|podcast-discover))/?(?:[#?]|$)'
     _TESTS = [{
         'url': 'https://zingmp3.vn/zing-chart',
         'info_dict': {
@@ -244,13 +281,34 @@ class ZingMp3ChartHomeIE(ZingMp3BaseIE):
             'id': 'moi-phat-hanh',
         },
         'playlist_mincount': 100,
+    }, {
+        'url': 'https://zingmp3.vn/top100',
+        'info_dict': {
+            'id': 'top100',
+        },
+        'playlist_mincount': 50,
+    }, {
+        'url': 'https://zingmp3.vn/podcast-discover',
+        'info_dict': {
+            'id': 'podcast-discover',
+        },
+        'playlist_mincount': 4,
     }]
     IE_NAME = 'zingmp3:chart-home'
 
     def _real_extract(self, url):
         url_type = self._match_id(url)
-        data = self._call_api(url_type, {'id': url_type})
-        items = traverse_obj(data, ('RTChart', 'items') if url_type == 'zing-chart' else 'items')
+        params = {'id': url_type}
+        if url_type == 'podcast-discover':
+            params['type'] = 'discover'
+        data = self._call_api(url_type, params)
+        items = []
+        if url_type == 'top100':
+            items.extend(traverse_obj(data, (..., 'items', ..., {dict})))
+        elif url_type == 'zing-chart':
+            items.extend(traverse_obj(data, ('RTChart', 'items', ..., {dict})))
+        else:
+            items.extend(traverse_obj(data, ('items', ..., {dict})))
         return self.playlist_result(self._parse_items(items), url_type)
 
 
@@ -334,7 +392,7 @@ def _real_extract(self, url):
 
 
 class ZingMp3UserIE(ZingMp3BaseIE):
-    _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<user>[^/]+)/(?P<type>bai-hat|single|album|video)/?(?:[?#]|$)'
+    _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<user>[^/]+)/(?P<type>bai-hat|single|album|video|song)/?(?:[?#]|$)'
     IE_NAME = 'zingmp3:user'
     _TESTS = [{
         'url': 'https://zingmp3.vn/Mr-Siro/bai-hat',
@@ -368,6 +426,18 @@ class ZingMp3UserIE(ZingMp3BaseIE):
             'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
         },
         'playlist_mincount': 15,
+    }, {
+        'url': 'https://zingmp3.vn/new-release/song',
+        'info_dict': {
+            'id': 'new-release-song',
+        },
+        'playlist_mincount': 50,
+    }, {
+        'url': 'https://zingmp3.vn/new-release/album',
+        'info_dict': {
+            'id': 'new-release-album',
+        },
+        'playlist_mincount': 20,
     }]
 
     def _fetch_page(self, user_id, url_type, page):
@@ -380,20 +450,28 @@ def _fetch_page(self, user_id, url_type, page):
         })
 
     def _real_extract(self, url):
-        user_alias, url_type = self._match_valid_url(url).group('user', 'type')
+        alias, url_type = self._match_valid_url(url).group('user', 'type')
         if not url_type:
             url_type = 'bai-hat'
 
-        user_info = self._call_api('info-artist', {}, user_alias, query={'alias': user_alias})
-        if url_type in ('bai-hat', 'video'):
-            entries = self._paged_list(user_info['id'], url_type)
+        user_info = self._call_api('info-artist', {}, alias, query={'alias': alias})
+
+        # Handle for new-release
+        if alias == 'new-release' and url_type in ('song', 'album'):
+            _id = f'{alias}-{url_type}'
+            return self.playlist_result(self._parse_items(
+                self._call_api('new-release', params={'type': url_type}, display_id=_id)), _id)
         else:
-            entries = self._parse_items(traverse_obj(user_info, (
-                'sections',
-                lambda _, v: v['sectionId'] == 'aAlbum' if url_type == 'album' else v['sectionId'] == 'aSingle',
-                'items', ...)))
-        return self.playlist_result(
-            entries, user_info['id'], f'{user_info.get("name")} - {url_type}', user_info.get('biography'))
+            # Handle for user/artist
+            if url_type in ('bai-hat', 'video'):
+                entries = self._paged_list(user_info['id'], url_type)
+            else:
+                section_id = 'aAlbum' if url_type == 'album' else 'aSingle'
+                entries = self._parse_items(traverse_obj(user_info, (
+                    'sections', lambda _, v: v['sectionId'] == section_id, 'items', ...)))
+            return self.playlist_result(
+                entries, user_info['id'], join_nonempty(user_info.get('name'), url_type, delim=' - '),
+                user_info.get('biography'))
 
 
 class ZingMp3HubIE(ZingMp3BaseIE):
@@ -403,7 +481,7 @@ class ZingMp3HubIE(ZingMp3BaseIE):
         'url': 'https://zingmp3.vn/hub/Nhac-Moi/IWZ9Z0CA.html',
         'info_dict': {
             'id': 'IWZ9Z0CA',
-            'title': 'Nhạc Mới',
+            'title': 'BXH Nhạc Mới',
             'description': 'md5:1cc31b68a6f746427b07b2756c22a558',
         },
         'playlist_mincount': 20,
@@ -424,3 +502,129 @@ def _real_extract(self, url):
             'sections', lambda _, v: v['sectionId'] == 'hub', 'items', ...)))
         return self.playlist_result(
             entries, song_id, hub_detail.get('title'), hub_detail.get('description'))
+
+
+class ZingMp3LiveRadioIE(ZingMp3BaseIE):
+    IE_NAME = 'zingmp3:liveradio'
+    _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<type>(?:liveradio))/(?P<id>\w+)(?:\.html|\?)'
+    _TESTS = [{
+        'url': 'https://zingmp3.vn/liveradio/IWZ979UB.html',
+        'info_dict': {
+            'id': 'IWZ979UB',
+            'title': r're:^V\-POP',
+            'description': 'md5:aa857f8a91dc9ce69e862a809e4bdc10',
+            'protocol': 'm3u8_native',
+            'ext': 'mp4',
+            'view_count': int,
+            'thumbnail': r're:^https?://.*\.jpg',
+            'like_count': int,
+            'live_status': 'is_live',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://zingmp3.vn/liveradio/IWZ97CWB.html',
+        'info_dict': {
+            'id': 'IWZ97CWB',
+            'title': r're:^Live\s247',
+            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+            'protocol': 'm3u8_native',
+            'ext': 'm4a',
+            'view_count': int,
+            'thumbnail': r're:^https?://.*\.jpg',
+            'like_count': int,
+            'live_status': 'is_live',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        url_type, live_radio_id = self._match_valid_url(url).group('type', 'id')
+        info = self._call_api(url_type, {'id': live_radio_id})
+        manifest_url = info.get('streaming')
+        if not manifest_url:
+            raise ExtractorError('This radio is offline.', expected=True)
+        fmts, subtitles = self._extract_m3u8_formats_and_subtitles(manifest_url, live_radio_id, fatal=False)
+        return {
+            'id': live_radio_id,
+            'is_live': True,
+            'formats': fmts,
+            'subtitles': subtitles,
+            **traverse_obj(info, {
+                'title': 'title',
+                'thumbnail': (('thumbnail', 'thumbnailM', 'thumbnailV', 'thumbnailH'), {url_or_none}),
+                'view_count': ('activeUsers', {int_or_none}),
+                'like_count': ('totalReaction', {int_or_none}),
+                'description': 'description',
+            }, get_all=False),
+        }
+
+
+class ZingMp3PodcastEpisodeIE(ZingMp3BaseIE):
+    IE_NAME = 'zingmp3:podcast-episode'
+    _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'pgr|cgr'
+    _TESTS = [{
+        'url': 'https://zingmp3.vn/pgr/Nhac-Moi-Moi-Ngay/68Z9W66B.html',
+        'info_dict': {
+            'id': '68Z9W66B',
+            'title': 'Nhạc Mới Mỗi Ngày',
+            'description': 'md5:2875dfa951f8e5356742f1610cf20691'
+        },
+        'playlist_mincount': 20,
+    }, {
+        'url': 'https://zingmp3.vn/cgr/Am-nhac/IWZ980AO.html',
+        'info_dict': {
+            'id': 'IWZ980AO',
+            'title': 'Âm nhạc'
+        },
+        'playlist_mincount': 2,
+    }]
+
+    def _fetch_page(self, eps_id, url_type, page):
+        return self._call_api(url_type, {
+            'id': eps_id,
+            'page': page,
+            'count': self._PER_PAGE
+        })
+
+    def _real_extract(self, url):
+        podcast_id, url_type = self._match_valid_url(url).group('id', 'type')
+        podcast_info = self._call_api(url_type, {'id': podcast_id})
+        entries = self._paged_list(podcast_id, 'pgr-list' if url_type == 'pgr' else 'cgr-list')
+        return self.playlist_result(
+            entries, podcast_id, podcast_info.get('title'), podcast_info.get('description'))
+
+
+class ZingMp3PodcastIE(ZingMp3BaseIE):
+    IE_NAME = 'zingmp3:podcast'
+    _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<id>(?:cgr|top-podcast|podcast-new))/?(?:[#?]|$)'
+    _TESTS = [{
+        'url': 'https://zingmp3.vn/cgr',
+        'info_dict': {
+            'id': 'cgr',
+        },
+        'playlist_mincount': 5,
+    }, {
+        'url': 'https://zingmp3.vn/top-podcast',
+        'info_dict': {
+            'id': 'top-podcast',
+        },
+        'playlist_mincount': 7,
+    }, {
+        'url': 'https://zingmp3.vn/podcast-new',
+        'info_dict': {
+            'id': 'podcast-new',
+        },
+        'playlist_mincount': 4,
+    }]
+
+    def _real_extract(self, url):
+        url_type = self._match_id(url)
+        params = {'id': url_type}
+        if url_type == 'podcast-new':
+            params['type'] = 'new'
+        items = self._call_api('cgrs' if url_type == 'cgr' else url_type, params)['items']
+        return self.playlist_result(self._parse_items(items), url_type)

From 3f90813f0617e0d21302398010de7496c9ae36aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Droz?= <raphael.droz@gmail.com>
Date: Thu, 16 Nov 2023 19:24:12 -0300
Subject: [PATCH 694/871] [ie/altcensored] Add extractor (#8291)

Authored by: drzraf
---
 yt_dlp/extractor/_extractors.py |  4 ++
 yt_dlp/extractor/altcensored.py | 96 +++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 yt_dlp/extractor/altcensored.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 63d1f2c1c..ce3015b57 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -82,6 +82,10 @@
 from .aitube import AitubeKZVideoIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
+from .altcensored import (
+    AltCensoredIE,
+    AltCensoredChannelIE,
+)
 from .amara import AmaraIE
 from .alura import (
     AluraIE,
diff --git a/yt_dlp/extractor/altcensored.py b/yt_dlp/extractor/altcensored.py
new file mode 100644
index 000000000..0e1627bfd
--- /dev/null
+++ b/yt_dlp/extractor/altcensored.py
@@ -0,0 +1,96 @@
+import re
+
+from .archiveorg import ArchiveOrgIE
+from .common import InfoExtractor
+from ..utils import (
+    InAdvancePagedList,
+    int_or_none,
+    orderedSet,
+    str_to_int,
+    urljoin,
+)
+
+
+class AltCensoredIE(InfoExtractor):
+    IE_NAME = 'altcensored'
+    _VALID_URL = r'https?://(?:www\.)?altcensored\.com/(?:watch\?v=|embed/)(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.altcensored.com/watch?v=k0srjLSkga8',
+        'info_dict': {
+            'id': 'youtube-k0srjLSkga8',
+            'ext': 'webm',
+            'title': "QUELLES SONT LES CONSÉQUENCES DE L'HYPERSEXUALISATION DE LA SOCIÉTÉ ?",
+            'display_id': 'k0srjLSkga8.webm',
+            'release_date': '20180403',
+            'creator': 'Virginie Vota',
+            'release_year': 2018,
+            'upload_date': '20230318',
+            'uploader': 'admin@altcensored.com',
+            'description': 'md5:0b38a8fc04103579d5c1db10a247dc30',
+            'timestamp': 1679161343,
+            'track': 'k0srjLSkga8',
+            'duration': 926.09,
+            'thumbnail': 'https://archive.org/download/youtube-k0srjLSkga8/youtube-k0srjLSkga8.thumbs/k0srjLSkga8_000925.jpg',
+            'view_count': int,
+            'categories': ['News & Politics'],
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        return {
+            '_type': 'url_transparent',
+            'url': f'https://archive.org/details/youtube-{video_id}',
+            'ie_key': ArchiveOrgIE.ie_key(),
+            'view_count': str_to_int(self._html_search_regex(
+                r'YouTube Views:(?:\s|&nbsp;)*([\d,]+)', webpage, 'view count', default=None)),
+            'categories': self._html_search_regex(
+                r'<a href="/category/\d+">\s*\n?\s*([^<]+)</a>',
+                webpage, 'category', default='').split() or None,
+        }
+
+
+class AltCensoredChannelIE(InfoExtractor):
+    IE_NAME = 'altcensored:channel'
+    _VALID_URL = r'https?://(?:www\.)?altcensored\.com/channel/(?!page|table)(?P<id>[^/?#]+)'
+    _PAGE_SIZE = 24
+    _TESTS = [{
+        'url': 'https://www.altcensored.com/channel/UCFPTO55xxHqFqkzRZHu4kcw',
+        'info_dict': {
+            'title': 'Virginie Vota',
+            'id': 'UCFPTO55xxHqFqkzRZHu4kcw',
+        },
+        'playlist_count': 91
+    }, {
+        'url': 'https://altcensored.com/channel/UC9CcJ96HKMWn0LZlcxlpFTw',
+        'info_dict': {
+            'title': 'yukikaze775',
+            'id': 'UC9CcJ96HKMWn0LZlcxlpFTw',
+        },
+        'playlist_count': 4
+    }]
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, channel_id, 'Download channel webpage', 'Unable to get channel webpage')
+        title = self._html_search_meta('altcen_title', webpage, 'title', fatal=False)
+        page_count = int_or_none(self._html_search_regex(
+            r'<a[^>]+href="/channel/\w+/page/(\d+)">(?:\1)</a>',
+            webpage, 'page count', default='1'))
+
+        def page_func(page_num):
+            page_num += 1
+            webpage = self._download_webpage(
+                f'https://altcensored.com/channel/{channel_id}/page/{page_num}',
+                channel_id, note=f'Downloading page {page_num}')
+
+            items = re.findall(r'<a[^>]+href="(/watch\?v=[^"]+)', webpage)
+            return [self.url_result(urljoin('https://www.altcensored.com', path), AltCensoredIE)
+                    for path in orderedSet(items)]
+
+        return self.playlist_result(
+            InAdvancePagedList(page_func, page_count, self._PAGE_SIZE),
+            playlist_id=channel_id, playlist_title=title)

From 66a0127d45033c698bdbedf162cddc55d9e7b906 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Fri, 17 Nov 2023 00:46:29 +0200
Subject: [PATCH 695/871] [ie/duoplay] Add extractor (#8542)

Authored by: glensc
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/duoplay.py     | 119 ++++++++++++++++++++++++++++++++
 2 files changed, 120 insertions(+)
 create mode 100644 yt_dlp/extractor/duoplay.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ce3015b57..4b58acffc 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -545,6 +545,7 @@
     DropoutSeasonIE,
     DropoutIE
 )
+from .duoplay import DuoplayIE
 from .dw import (
     DWIE,
     DWArticleIE,
diff --git a/yt_dlp/extractor/duoplay.py b/yt_dlp/extractor/duoplay.py
new file mode 100644
index 000000000..e57fa7924
--- /dev/null
+++ b/yt_dlp/extractor/duoplay.py
@@ -0,0 +1,119 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    extract_attributes,
+    get_element_text_and_html_by_tag,
+    int_or_none,
+    join_nonempty,
+    str_or_none,
+    try_call,
+    unified_timestamp,
+)
+from ..utils.traversal import traverse_obj
+
+
+class DuoplayIE(InfoExtractor):
+    _VALID_URL = r'https://duoplay\.ee/(?P<id>\d+)/[\w-]+/?(?:\?(?:[^#]+&)?ep=(?P<ep>\d+))?'
+    _TESTS = [{
+        'note': 'Siberi võmm S02E12',
+        'url': 'https://duoplay.ee/4312/siberi-vomm?ep=24',
+        'md5': '1ff59d535310ac9c5cf5f287d8f91b2d',
+        'info_dict': {
+            'id': '4312_24',
+            'ext': 'mp4',
+            'title': 'Operatsioon "Öö"',
+            'thumbnail': r're:https://.+\.jpg(?:\?c=\d+)?$',
+            'description': 'md5:8ef98f38569d6b8b78f3d350ccc6ade8',
+            'upload_date': '20170523',
+            'timestamp': 1495567800,
+            'series': 'Siberi võmm',
+            'series_id': '4312',
+            'season': 'Season 2',
+            'season_number': 2,
+            'episode': 'Operatsioon "Öö"',
+            'episode_number': 12,
+            'episode_id': 24,
+        },
+    }, {
+        'note': 'Empty title',
+        'url': 'https://duoplay.ee/17/uhikarotid?ep=14',
+        'md5': '6aca68be71112314738dd17cced7f8bf',
+        'info_dict': {
+            'id': '17_14',
+            'ext': 'mp4',
+            'title': 'Ühikarotid',
+            'thumbnail': r're:https://.+\.jpg(?:\?c=\d+)?$',
+            'description': 'md5:4719b418e058c209def41d48b601276e',
+            'upload_date': '20100916',
+            'timestamp': 1284661800,
+            'series': 'Ühikarotid',
+            'series_id': '17',
+            'season': 'Season 2',
+            'season_number': 2,
+            'episode_id': 14,
+            'release_year': 2010,
+        },
+    }, {
+        'note': 'Movie',
+        'url': 'https://duoplay.ee/4325/naljamangud',
+        'md5': '2b0bcac4159a08b1844c2bfde06b1199',
+        'info_dict': {
+            'id': '4325',
+            'ext': 'mp4',
+            'title': 'Näljamängud',
+            'thumbnail': r're:https://.+\.jpg(?:\?c=\d+)?$',
+            'description': 'md5:fb35f5eb2ff46cdb82e4d5fbe7b49a13',
+            'cast': ['Jennifer Lawrence', 'Josh Hutcherson', 'Liam Hemsworth'],
+            'upload_date': '20231109',
+            'timestamp': 1699552800,
+            'release_year': 2012,
+        },
+    }, {
+        'note': 'Movie without expiry',
+        'url': 'https://duoplay.ee/5501/pilvede-all.-neljas-ode',
+        'md5': '7abf63d773a49ef7c39f2c127842b8fd',
+        'info_dict': {
+            'id': '5501',
+            'ext': 'mp4',
+            'title': 'Pilvede all. Neljas õde',
+            'thumbnail': r're:https://.+\.jpg(?:\?c=\d+)?$',
+            'description': 'md5:d86a70f8f31e82c369d4d4f4c79b1279',
+            'cast': 'count:9',
+            'upload_date': '20221214',
+            'timestamp': 1671054000,
+            'release_year': 2018,
+        },
+    }]
+
+    def _real_extract(self, url):
+        telecast_id, episode = self._match_valid_url(url).group('id', 'ep')
+        video_id = join_nonempty(telecast_id, episode, delim='_')
+        webpage = self._download_webpage(url, video_id)
+        video_player = try_call(lambda: extract_attributes(
+            get_element_text_and_html_by_tag('video-player', webpage)[1]))
+        if not video_player or not video_player.get('manifest-url'):
+            raise ExtractorError('No video found', expected=True)
+
+        episode_attr = self._parse_json(video_player.get(':episode') or '', video_id, fatal=False) or {}
+
+        return {
+            'id': video_id,
+            'formats': self._extract_m3u8_formats(video_player['manifest-url'], video_id, 'mp4'),
+            **traverse_obj(episode_attr, {
+                'title': 'title',
+                'description': 'synopsis',
+                'thumbnail': ('images', 'original'),
+                'timestamp': ('airtime', {lambda x: unified_timestamp(x + ' +0200')}),
+                'cast': ('cast', {lambda x: x.split(', ')}),
+                'release_year': ('year', {int_or_none}),
+            }),
+            **(traverse_obj(episode_attr, {
+                'title': (None, ('subtitle', ('episode_nr', {lambda x: f'Episode {x}' if x else None}))),
+                'series': 'title',
+                'series_id': ('telecast_id', {str_or_none}),
+                'season_number': ('season_id', {int_or_none}),
+                'episode': 'subtitle',
+                'episode_number': ('episode_nr', {int_or_none}),
+                'episode_id': ('episode_id', {int_or_none}),
+            }, get_all=False) if episode_attr.get('category') != 'movies' else {}),
+        }

From 1d24da6c899ef280d8b0a48a5e280ecd5d39cdf4 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 18 Nov 2023 21:04:42 +0100
Subject: [PATCH 696/871] [ie/nintendo] Fix Nintendo Direct extraction (#8609)

Authored by: Grub4K
---
 yt_dlp/extractor/nintendo.py | 152 ++++++++++++++++++++++++++---------
 1 file changed, 113 insertions(+), 39 deletions(-)

diff --git a/yt_dlp/extractor/nintendo.py b/yt_dlp/extractor/nintendo.py
index ed839af25..853a169bb 100644
--- a/yt_dlp/extractor/nintendo.py
+++ b/yt_dlp/extractor/nintendo.py
@@ -1,57 +1,131 @@
-import re
+import json
+import urllib.parse
 
 from .common import InfoExtractor
-from .ooyala import OoyalaIE
+from ..utils import (
+    ExtractorError,
+    make_archive_id,
+    unified_timestamp,
+    urljoin,
+)
+from ..utils.traversal import traverse_obj
 
 
 class NintendoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nintendo\.com/(?:games/detail|nintendo-direct)/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?nintendo\.com/(?:(?P<locale>\w{2}(?:-\w{2})?)/)?nintendo-direct/(?P<slug>[^/?#]+)'
     _TESTS = [{
-        'url': 'https://www.nintendo.com/games/detail/duck-hunt-wii-u/',
-        'info_dict': {
-            'id': 'MzMmticjp0VPzO3CCj4rmFOuohEuEWoW',
-            'ext': 'flv',
-            'title': 'Duck Hunt Wii U VC NES - Trailer',
-            'duration': 60.326,
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'add_ie': ['Ooyala'],
-    }, {
-        'url': 'http://www.nintendo.com/games/detail/tokyo-mirage-sessions-fe-wii-u',
-        'info_dict': {
-            'id': 'tokyo-mirage-sessions-fe-wii-u',
-            'title': 'Tokyo Mirage Sessions ♯FE',
-        },
-        'playlist_count': 4,
-    }, {
         'url': 'https://www.nintendo.com/nintendo-direct/09-04-2019/',
         'info_dict': {
-            'id': 'J2bXdmaTE6fe3dWJTPcc7m23FNbc_A1V',
             'ext': 'mp4',
-            'title': 'Switch_ROS_ND0904-H264.mov',
-            'duration': 2324.758,
+            'id': '2oPmiviVePUA1IqAZzjuVh',
+            'display_id': '09-04-2019',
+            'title': 'Nintendo Direct 9.4.2019',
+            'timestamp': 1567580400,
+            'description': 'md5:8aac2780361d8cb772b6d1de66d7d6f4',
+            'upload_date': '20190904',
+            'age_limit': 17,
+            '_old_archive_ids': ['nintendo J2bXdmaTE6fe3dWJTPcc7m23FNbc_A1V'],
         },
-        'params': {
-            'skip_download': True,
+    }, {
+        'url': 'https://www.nintendo.com/en-ca/nintendo-direct/08-31-2023/',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': '2TB2w2rJhNYF84qQ9E57hU',
+            'display_id': '08-31-2023',
+            'title': 'Super Mario Bros. Wonder Direct 8.31.2023',
+            'timestamp': 1693465200,
+            'description': 'md5:3067c5b824bcfdae9090a7f38ab2d200',
+            'tags': ['Mild Fantasy Violence', 'In-Game Purchases'],
+            'upload_date': '20230831',
+            'age_limit': 6,
+        },
+    }, {
+        'url': 'https://www.nintendo.com/us/nintendo-direct/50-fact-extravaganza/',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': 'j0BBGzfw0pQ',
+            'channel_follower_count': int,
+            'view_count': int,
+            'description': 'Learn new details about Super Smash Bros. for Wii U, which launches on November 21.',
+            'duration': 2123,
+            'availability': 'public',
+            'thumbnail': 'https://i.ytimg.com/vi_webp/j0BBGzfw0pQ/maxresdefault.webp',
+            'timestamp': 1414047600,
+            'channel_id': 'UCGIY_O-8vW4rfX98KlMkvRg',
+            'chapters': 'count:53',
+            'heatmap': 'count:100',
+            'upload_date': '20141023',
+            'uploader_id': '@NintendoAmerica',
+            'playable_in_embed': True,
+            'categories': ['Gaming'],
+            'display_id': '50-fact-extravaganza',
+            'channel': 'Nintendo of America',
+            'tags': ['Comic Mischief', 'Cartoon Violence', 'Mild Suggestive Themes'],
+            'like_count': int,
+            'channel_url': 'https://www.youtube.com/channel/UCGIY_O-8vW4rfX98KlMkvRg',
+            'age_limit': 10,
+            'uploader_url': 'https://www.youtube.com/@NintendoAmerica',
+            'comment_count': int,
+            'live_status': 'not_live',
+            'uploader': 'Nintendo of America',
+            'title': '50-FACT Extravaganza',
         },
-        'add_ie': ['Ooyala'],
     }]
 
+    def _create_asset_url(self, path):
+        return urljoin('https://assets.nintendo.com/', urllib.parse.quote(path))
+
     def _real_extract(self, url):
-        page_id = self._match_id(url)
+        locale, slug = self._match_valid_url(url).group('locale', 'slug')
 
-        webpage = self._download_webpage(url, page_id)
+        language, _, country = (locale or 'US').rpartition('-')
+        parsed_locale = f'{language.lower() or "en"}_{country.upper()}'
+        self.write_debug(f'Using locale {parsed_locale} (from {locale})', only_once=True)
 
-        entries = [
-            OoyalaIE._build_url_result(m.group('code'))
-            for m in re.finditer(
-                r'data-(?:video-id|directVideoId)=(["\'])(?P<code>(?:(?!\1).)+)\1', webpage)]
+        response = self._download_json('https://graph.nintendo.com/', slug, query={
+            'operationName': 'NintendoDirect',
+            'variables': json.dumps({
+                'locale': parsed_locale,
+                'slug': slug,
+            }, separators=(',', ':')),
+            'extensions': json.dumps({
+                'persistedQuery': {
+                    'version': 1,
+                    'sha256Hash': '969b16fe9f08b686fa37bc44d1fd913b6188e65794bb5e341c54fa683a8004cb'
+                },
+            }, separators=(',', ':')),
+        })
+        # API returns `{"data": {"direct": null}}` if no matching id
+        direct_info = traverse_obj(response, ('data', 'direct', {dict}))
+        if not direct_info:
+            raise ExtractorError(f'No Nintendo Direct with id {slug} exists', expected=True)
 
-        title = self._html_search_regex(
-            r'(?s)<(?:span|div)[^>]+class="(?:title|wrapper)"[^>]*>.*?<h1>(.+?)</h1>',
-            webpage, 'title', fatal=False)
+        errors = ', '.join(traverse_obj(response, ('errors', ..., 'message')))
+        if errors:
+            raise ExtractorError(f'GraphQL API error: {errors or "Unknown error"}')
 
-        return self.playlist_result(
-            entries, page_id, title)
+        result = traverse_obj(direct_info, {
+            'id': ('id', {str}),
+            'title': ('name', {str}),
+            'timestamp': ('startDate', {unified_timestamp}),
+            'description': ('description', 'text', {str}),
+            'age_limit': ('contentRating', 'order', {int}),
+            'tags': ('contentDescriptors', ..., 'label', {str}),
+            'thumbnail': ('thumbnail', {self._create_asset_url}),
+        })
+        result['display_id'] = slug
+
+        asset_id = traverse_obj(direct_info, ('video', 'publicId', {str}))
+        if not asset_id:
+            youtube_id = traverse_obj(direct_info, ('liveStream', {str}))
+            if not youtube_id:
+                self.raise_no_formats('Could not find any video formats', video_id=slug)
+
+            return self.url_result(youtube_id, **result, url_transparent=True)
+
+        if asset_id.startswith('Legacy Videos/'):
+            result['_old_archive_ids'] = [make_archive_id(self, asset_id[14:])]
+        result['formats'] = self._extract_m3u8_formats(
+            self._create_asset_url(f'/video/upload/sp_full_hd/v1/{asset_id}.m3u8'), slug)
+
+        return result

From 34df1c1f60fa652c0a6a5c712b06c10e45daf6b7 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sat, 18 Nov 2023 21:28:25 +0100
Subject: [PATCH 697/871] [ie/vidly] Add extractor (#8612)

Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/vidly.py       | 83 +++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+)
 create mode 100644 yt_dlp/extractor/vidly.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 4b58acffc..a9e33d32a 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2301,6 +2301,7 @@
     VidioLiveIE
 )
 from .vidlii import VidLiiIE
+from .vidly import VidlyIE
 from .viewlift import (
     ViewLiftIE,
     ViewLiftEmbedIE,
diff --git a/yt_dlp/extractor/vidly.py b/yt_dlp/extractor/vidly.py
new file mode 100644
index 000000000..49a196041
--- /dev/null
+++ b/yt_dlp/extractor/vidly.py
@@ -0,0 +1,83 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    mimetype2ext,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class VidlyIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:vid\.ly/|(?:s\.)?vid\.ly/embeded\.html\?(?:[^#]+&)?link=)(?P<id>\w+)'
+    _EMBED_REGEX = [r'<script[^>]+\bsrc=[\'"](?P<url>(?:https?:)?//vid\.ly/\w+/embed[^\'"]+)',
+                    r'<iframe[^>]+\bsrc=[\'"](?P<url>(?:https?:)?//(?:s\.)?vid\.ly/embeded\.html\?(?:[^#\'"]+&)?link=\w+[^\'"]+)']
+    _TESTS = [{
+        # JWPlayer 7, Embeds forbidden
+        'url': 'https://vid.ly/2i3o9j/embed',
+        'info_dict': {
+            'id': '2i3o9j',
+            'ext': 'mp4',
+            'title': '2i3o9j',
+            'thumbnail': r're:https://\w+\.cloudfront\.net/',
+        },
+    }, {
+        # JWPlayer 6
+        'url': 'http://s.vid.ly/embeded.html?link=jw_test&new=1&autoplay=true&controls=true',
+        'info_dict': {
+            'id': 'jw_test',
+            'ext': 'mp4',
+            'title': '2x8m8t',
+            'thumbnail': r're:https://\w+\.cloudfront\.net/',
+        },
+    }, {
+        # Vidlyplayer
+        'url': 'https://vid.ly/7x0e6l',
+        'info_dict': {
+            'id': '7x0e6l',
+            'ext': 'mp4',
+            'title': '7x0e6l',
+        },
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://www.petfinder.com/dog/gus-57378930/tn/ooltewah/furever-furkids-rescue-tn592/',
+        'info_dict': {
+            'id': 'w8p5b0',
+            'ext': 'mp4',
+            'title': 'w8p5b0',
+            'thumbnail': r're:https://\w+\.cloudfront\.net/',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        embed_script = self._download_webpage(
+            f'https://vid.ly/{video_id}/embed', video_id, headers={'Referer': 'https://vid.ly/'})
+        player = self._search_json(r'initCallback\(', embed_script, 'player', video_id)
+
+        player_type = player.get('player') or ''
+        if player_type.startswith('jwplayer'):
+            return self._parse_jwplayer_data(player['config'], video_id)
+        elif not player_type.startswith('vidly'):
+            raise ExtractorError(f'Unknown player type {player_type!r}')
+
+        formats = []
+        ext = mimetype2ext(traverse_obj(player, ('config', 'type')))
+        for source, fid in [('source', 'sd'), ('source_hd', 'hd')]:
+            if traverse_obj(player, ('config', source, {url_or_none})):
+                formats.append({
+                    'url': player['config'][source],
+                    'format_id': f'http-{fid}',
+                    'ext': ext,
+                })
+        # Has higher quality formats
+        formats.extend(self._extract_m3u8_formats(
+            f'https://d3fenhwk93s16g.cloudfront.net/{video_id}/hls.m3u8', video_id,
+            fatal=False, note='Requesting higher quality m3u8 formats',
+            errnote='No higher quality m3u8 formats found') or [])
+
+        return {
+            'id': video_id,
+            'title': video_id,
+            'formats': formats,
+        }

From 6fe82491ed622b948c512cf4aab46ac3a234ae0a Mon Sep 17 00:00:00 2001
From: Berkay <101016182+sonmezberkay@users.noreply.github.com>
Date: Sat, 18 Nov 2023 22:46:22 +0100
Subject: [PATCH 698/871] [ie/twitter:broadcast] Extract
 `concurrent_view_count` (#8600)

Authored by: sonmezberkay
---
 yt_dlp/extractor/periscope.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/yt_dlp/extractor/periscope.py b/yt_dlp/extractor/periscope.py
index 3d1375b64..d2351df1a 100644
--- a/yt_dlp/extractor/periscope.py
+++ b/yt_dlp/extractor/periscope.py
@@ -35,6 +35,7 @@ def _parse_broadcast_data(self, broadcast, video_id):
             'uploader_id': broadcast.get('user_id') or broadcast.get('username'),
             'thumbnails': thumbnails,
             'view_count': int_or_none(broadcast.get('total_watched')),
+            'concurrent_view_count': int_or_none(broadcast.get('total_watching')),
             'tags': broadcast.get('tags'),
             'live_status': {
                 'running': 'is_live',

From f223b1b0789f65e06619dcc9fc9e74f50d259379 Mon Sep 17 00:00:00 2001
From: qbnu <93988953+qbnu@users.noreply.github.com>
Date: Sat, 18 Nov 2023 21:49:23 +0000
Subject: [PATCH 699/871] [ie/vocaroo] Do not use deprecated `getheader`
 (#8606)

Authored by: qbnu
---
 yt_dlp/extractor/vocaroo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/vocaroo.py b/yt_dlp/extractor/vocaroo.py
index d98fbfd2d..e30c9597f 100644
--- a/yt_dlp/extractor/vocaroo.py
+++ b/yt_dlp/extractor/vocaroo.py
@@ -57,7 +57,7 @@ def _real_extract(self, url):
             'title': '',
             'url': url,
             'ext': 'mp3',
-            'timestamp': float_or_none(resp.getheader('x-bz-upload-timestamp'), scale=1000),
+            'timestamp': float_or_none(resp.headers.get('x-bz-upload-timestamp'), scale=1000),
             'vcodec': 'none',
             'http_headers': http_headers,
         }

From ddb2d7588bea48bae965dbfabe6df6550c9d3d43 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sat, 18 Nov 2023 22:51:18 +0100
Subject: [PATCH 700/871] [ie] Extract from `media` elements in SMIL manifests
 (#8504)

Authored by: seproDev
---
 yt_dlp/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index b3a45b3fb..507ef34cf 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -2341,7 +2341,9 @@ def _parse_smil_formats_and_subtitles(
         imgs_count = 0
 
         srcs = set()
-        media = smil.findall(self._xpath_ns('.//video', namespace)) + smil.findall(self._xpath_ns('.//audio', namespace))
+        media = itertools.chain.from_iterable(
+            smil.findall(self._xpath_ns(arg, namespace))
+            for arg in ['.//video', './/audio', './/media'])
         for medium in media:
             src = medium.get('src')
             if not src or src in srcs:

From 1fa3f24d4b5d22176b11d78420f1f4b64a5af0a8 Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Sat, 18 Nov 2023 21:54:00 +0000
Subject: [PATCH 701/871] [ie/theguardian] Add extractors (#8535)

Closes #8520
Authored by: SirElderling
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/theguardian.py | 135 ++++++++++++++++++++++++++++++++
 2 files changed, 139 insertions(+)
 create mode 100644 yt_dlp/extractor/theguardian.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index a9e33d32a..b8e6a1619 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2029,6 +2029,10 @@
 from .testurl import TestURLIE
 from .tf1 import TF1IE
 from .tfo import TFOIE
+from .theguardian import (
+    TheGuardianPodcastIE,
+    TheGuardianPodcastPlaylistIE,
+)
 from .theholetv import TheHoleTvIE
 from .theintercept import TheInterceptIE
 from .theplatform import (
diff --git a/yt_dlp/extractor/theguardian.py b/yt_dlp/extractor/theguardian.py
new file mode 100644
index 000000000..a231eccf4
--- /dev/null
+++ b/yt_dlp/extractor/theguardian.py
@@ -0,0 +1,135 @@
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    extract_attributes,
+    get_element_by_class,
+    get_element_html_by_class,
+    get_elements_html_by_class,
+    parse_qs,
+    traverse_obj,
+    unified_strdate,
+    urljoin
+)
+
+
+class TheGuardianPodcastIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?theguardian\.com/\w+/audio/\d{4}/\w{3}/\d{1,2}/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.theguardian.com/news/audio/2023/nov/03/we-are-just-getting-started-the-plastic-eating-bacteria-that-could-change-the-world-podcast',
+        'md5': 'd1771744681789b4cd7da2a08e487702',
+        'info_dict': {
+            'id': 'we-are-just-getting-started-the-plastic-eating-bacteria-that-could-change-the-world-podcast',
+            'ext': 'mp3',
+            'title': '‘We are just getting started’: the plastic-eating bacteria that could change the world – podcast',
+            'description': 'md5:cfd3df2791d394d2ab62cd571d5207ee',
+            'creator': 'Stephen Buranyi',
+            'thumbnail': 'md5:73c12558fcb3b0e2a59422bfb33b3f79',
+            'release_date': '20231103'
+        }
+    }, {
+        'url': 'https://www.theguardian.com/news/audio/2023/oct/30/the-trials-of-robert-habeck-is-the-worlds-most-powerful-green-politician-doomed-to-fail-podcast',
+        'md5': 'd1771744681789b4cd7da2a08e487702',
+        'info_dict': {
+            'id': 'the-trials-of-robert-habeck-is-the-worlds-most-powerful-green-politician-doomed-to-fail-podcast',
+            'ext': 'mp3',
+            'title': 'The trials of Robert Habeck: is the world’s most powerful green politician doomed to fail? – podcast',
+            'description': 'md5:1b5cf6582d1771c6b7077784b5456994',
+            'creator': 'Philip Oltermann',
+            'thumbnail': 'md5:6e5c5ec43843e956e20be793722e9080',
+            'release_date': '20231030'
+        }
+    }, {
+        'url': 'https://www.theguardian.com/football/audio/2023/nov/06/arsenal-feel-hard-done-by-and-luton-hold-liverpool-football-weekly',
+        'md5': 'a2fcff6f8e060a95b1483295273dc35e',
+        'info_dict': {
+            'id': 'arsenal-feel-hard-done-by-and-luton-hold-liverpool-football-weekly',
+            'ext': 'mp3',
+            'title': 'Arsenal feel hard done by and Luton hold Liverpool – Football Weekly',
+            'description': 'md5:286a9fbddaeb7c83cc65d1c4a5330b2a',
+            'creator': 'Max Rushden',
+            'thumbnail': 'md5:93eb7d6440f1bb94eb3a6cad63f48afd',
+            'release_date': '20231106'
+        }
+    }, {
+        'url': 'https://www.theguardian.com/politics/audio/2023/nov/02/the-covid-inquiry-politics-weekly-uk-podcast',
+        'md5': '06a0f7e9701a80c8064a5d35690481ec',
+        'info_dict': {
+            'id': 'the-covid-inquiry-politics-weekly-uk-podcast',
+            'ext': 'mp3',
+            'title': 'The Covid inquiry | Politics Weekly UK - podcast',
+            'description': 'md5:207c98859c14903582b17d25b014046e',
+            'creator': 'Gaby Hinsliff',
+            'thumbnail': 'md5:28932a7b5a25b057be330d2ed70ea7f3',
+            'release_date': '20231102'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage) or get_element_by_class('content__headline', webpage),
+            'description': self._og_search_description(webpage),
+            'creator': self._html_search_meta('author', webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'release_date': unified_strdate(self._html_search_meta('article:published_time', webpage)),
+            'url': extract_attributes(get_element_html_by_class(
+                'podcast__player', webpage) or '').get('data-source'),
+        }
+
+
+class TheGuardianPodcastPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?theguardian\.com/\w+/series/(?P<id>[\w-]+)(?:\?page=\d+)?'
+    _TESTS = [{
+        'url': 'https://www.theguardian.com/football/series/theguardianswomensfootballweekly',
+        'info_dict': {
+            'id': 'theguardianswomensfootballweekly',
+            'title': "The Guardian's Women's Football Weekly",
+            'description': 'md5:e2cc021311e582d29935a73614a43f51'
+        },
+        'playlist_mincount': 69
+    }, {
+        'url': 'https://www.theguardian.com/news/series/todayinfocus?page=2',
+        'info_dict': {
+            'id': 'todayinfocus',
+            'title': 'Today in Focus',
+            'description': 'md5:0f097764fc0d359e0b6eb537be0387e2'
+        },
+        'playlist_mincount': 1261
+    }, {
+        'url': 'https://www.theguardian.com/news/series/the-audio-long-read',
+        'info_dict': {
+            'id': 'the-audio-long-read',
+            'title': 'The Audio Long Read',
+            'description': 'md5:5462994a27527309562b25b6defc4ef3'
+        },
+        'playlist_mincount': 996
+    }]
+
+    def _entries(self, url, playlist_id):
+        for page in itertools.count(1):
+            webpage, urlh = self._download_webpage_handle(
+                url, playlist_id, f'Downloading page {page}', query={'page': page})
+            if 'page' not in parse_qs(urlh.url):
+                break
+
+            episodes = get_elements_html_by_class('fc-item--type-media', webpage)
+            for url_path in traverse_obj(episodes, (..., {extract_attributes}, 'data-id')):
+                yield url_path
+
+    def _real_extract(self, url):
+        podcast_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, podcast_id)
+
+        title = clean_html(get_element_by_class(
+            'index-page-header__title', webpage) or get_element_by_class('flagship-audio__title', webpage))
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage)
+
+        return self.playlist_from_matches(
+            self._entries(url, podcast_id), podcast_id, title, description=description,
+            ie=TheGuardianPodcastIE, getter=lambda x: urljoin('https://www.theguardian.com', x))

From 585d0ed9abcfcb957f2b2684b8ad43c3af160383 Mon Sep 17 00:00:00 2001
From: JC-Chung <52159296+JC-Chung@users.noreply.github.com>
Date: Sun, 19 Nov 2023 06:14:45 +0800
Subject: [PATCH 702/871] [ie/twitcasting] Detect livestreams via API and
 `show` page (#8601)

Authored by: JC-Chung, bashonly
---
 yt_dlp/extractor/twitcasting.py | 41 +++++++++++++++------------------
 1 file changed, 18 insertions(+), 23 deletions(-)

diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py
index 85c7b20de..3c303bdbb 100644
--- a/yt_dlp/extractor/twitcasting.py
+++ b/yt_dlp/extractor/twitcasting.py
@@ -11,7 +11,6 @@
     float_or_none,
     get_element_by_class,
     get_element_by_id,
-    int_or_none,
     parse_duration,
     qualities,
     str_to_int,
@@ -242,35 +241,31 @@ class TwitCastingLiveIE(InfoExtractor):
         'expected_exception': 'UserNotLive',
     }]
 
-    _PROTECTED_LIVE_RE = r'(?s)(<span\s*class="tw-movie-thumbnail2-badge"\s*data-status="live">\s*LIVE)'
-
     def _real_extract(self, url):
         uploader_id = self._match_id(url)
         self.to_screen(
             'Downloading live video of user {0}. '
             'Pass "https://twitcasting.tv/{0}/show" to download the history'.format(uploader_id))
 
-        webpage = self._download_webpage(url, uploader_id)
-        is_live = self._search_regex(  # first pattern is for public live
-            (r'(data-is-onlive="true")', self._PROTECTED_LIVE_RE), webpage, 'is live?', default=None)
-        current_live = int_or_none(self._search_regex(
-            (r'data-type="movie" data-id="(\d+)">',  # not available?
-             r'tw-sound-flag-open-link" data-id="(\d+)" style=',  # not available?
-             r'data-movie-id="(\d+)"'),  # if not currently live, value may be 0
-            webpage, 'current live ID', default=None))
-        if is_live and not current_live:
-            # fetch unfiltered /show to find running livestreams; we can't get ID of the password-protected livestream above
-            webpage = self._download_webpage(
-                f'https://twitcasting.tv/{uploader_id}/show/', uploader_id,
-                note='Downloading live history')
-            is_live = self._search_regex(self._PROTECTED_LIVE_RE, webpage, 'is live?', default=None)
-            if is_live:
-                # get the first live; running live is always at the first
-                current_live = self._search_regex(
-                    r'(?s)<a\s+class="tw-movie-thumbnail2"\s*href="/[^/]+/movie/(?P<video_id>\d+)"\s*>.+?</a>',
-                    webpage, 'current live ID 2', default=None, group='video_id')
-        if not current_live:
+        is_live = traverse_obj(self._download_json(
+            f'https://frontendapi.twitcasting.tv/watch/user/{uploader_id}',
+            uploader_id, 'Checking live status', data=b'', fatal=False), ('is_live', {bool}))
+        if is_live is False:  # only raise here if API response was as expected
             raise UserNotLive(video_id=uploader_id)
+
+        # Use /show/ page so that password-protected and members-only livestreams can be found
+        webpage = self._download_webpage(
+            f'https://twitcasting.tv/{uploader_id}/show/', uploader_id, 'Downloading live history')
+        is_live = is_live or self._search_regex(
+            r'(?s)(<span\s*class="tw-movie-thumbnail2-badge"\s*data-status="live">\s*LIVE)',
+            webpage, 'is live?', default=False)
+        # Current live is always the first match
+        current_live = self._search_regex(
+            r'(?s)<a\s+class="tw-movie-thumbnail2"\s+href="/[^/"]+/movie/(?P<video_id>\d+)"',
+            webpage, 'current live ID', default=None, group='video_id')
+        if not is_live or not current_live:
+            raise UserNotLive(video_id=uploader_id)
+
         return self.url_result(f'https://twitcasting.tv/{uploader_id}/movie/{current_live}', TwitCastingIE)
 
 

From f124fa458826308afc86cf364c509f857686ecfd Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 19 Nov 2023 16:05:13 +0100
Subject: [PATCH 703/871] [ci] Concurrency optimizations (#8614)

Authored by: Grub4K
---
 .github/workflows/core.yml           | 4 ++++
 .github/workflows/release-master.yml | 1 -
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 3c10fc17e..b1ae4ae76 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -3,6 +3,10 @@ on: [push, pull_request]
 permissions:
   contents: read
 
+concurrency:
+  group: core-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   tests:
     name: Core Tests
diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 0208b3bef..0664137a9 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -10,7 +10,6 @@ on:
       - "pyinst.py"
 concurrency:
   group: release-master
-  cancel-in-progress: true
 permissions:
   contents: read
 

From 9f09bdcfcb8e2b4b2decdc30d35d34b993bc7a94 Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Sun, 19 Nov 2023 16:26:46 -0500
Subject: [PATCH 704/871] [ie/bilibili] Support courses and interactive videos
 (#8343)

Closes #6135, Closes #8428
Authored by: c-basalt
---
 yt_dlp/extractor/_extractors.py |   2 +
 yt_dlp/extractor/bilibili.py    | 503 +++++++++++++++++++++++++++++---
 2 files changed, 463 insertions(+), 42 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b8e6a1619..76537e13e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -216,6 +216,8 @@
     BiliBiliBangumiIE,
     BiliBiliBangumiSeasonIE,
     BiliBiliBangumiMediaIE,
+    BilibiliCheeseIE,
+    BilibiliCheeseSeasonIE,
     BiliBiliSearchIE,
     BilibiliCategoryIE,
     BilibiliAudioIE,
diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 9119f396b..bc25dc75e 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -2,6 +2,7 @@
 import functools
 import hashlib
 import itertools
+import json
 import math
 import re
 import time
@@ -16,9 +17,11 @@
     InAdvancePagedList,
     OnDemandPagedList,
     bool_or_none,
+    clean_html,
     filter_dict,
     float_or_none,
     format_field,
+    get_element_by_class,
     int_or_none,
     join_nonempty,
     make_archive_id,
@@ -88,6 +91,12 @@ def extract_formats(self, play_info):
 
         return formats
 
+    def _download_playinfo(self, video_id, cid):
+        return self._download_json(
+            'https://api.bilibili.com/x/player/playurl', video_id,
+            query={'bvid': video_id, 'cid': cid, 'fnval': 4048},
+            note=f'Downloading video formats for cid {cid}')['data']
+
     def json2srt(self, json_data):
         srt_data = ''
         for idx, line in enumerate(json_data.get('body') or []):
@@ -96,7 +105,7 @@ def json2srt(self, json_data):
                          f'{line["content"]}\n\n')
         return srt_data
 
-    def _get_subtitles(self, video_id, aid, cid):
+    def _get_subtitles(self, video_id, cid, aid=None):
         subtitles = {
             'danmaku': [{
                 'ext': 'xml',
@@ -104,8 +113,15 @@ def _get_subtitles(self, video_id, aid, cid):
             }]
         }
 
-        video_info_json = self._download_json(f'https://api.bilibili.com/x/player/v2?aid={aid}&cid={cid}', video_id)
-        for s in traverse_obj(video_info_json, ('data', 'subtitle', 'subtitles', ...)):
+        subtitle_info = traverse_obj(self._download_json(
+            'https://api.bilibili.com/x/player/v2', video_id,
+            query={'aid': aid, 'cid': cid} if aid else {'bvid': video_id, 'cid': cid},
+            note=f'Extracting subtitle info {cid}'), ('data', 'subtitle'))
+        subs_list = traverse_obj(subtitle_info, ('subtitles', lambda _, v: v['subtitle_url'] and v['lan']))
+        if not subs_list and traverse_obj(subtitle_info, 'allow_submit'):
+            if not self._get_cookies('https://api.bilibili.com').get('SESSDATA'):  # no login session cookie
+                self.report_warning(f'CC subtitles (if any) are only visible when logged in. {self._login_hint()}', only_once=True)
+        for s in subs_list:
             subtitles.setdefault(s['lan'], []).append({
                 'ext': 'srt',
                 'data': self.json2srt(self._download_json(s['subtitle_url'], video_id))
@@ -155,7 +171,54 @@ def _get_episodes_from_season(self, ss_id, url):
         for entry in traverse_obj(season_info, (
                 'result', 'main_section', 'episodes',
                 lambda _, v: url_or_none(v['share_url']) and v['id'])):
-            yield self.url_result(entry['share_url'], BiliBiliBangumiIE, f'ep{entry["id"]}')
+            yield self.url_result(entry['share_url'], BiliBiliBangumiIE, str_or_none(entry.get('id')))
+
+    def _get_divisions(self, video_id, graph_version, edges, edge_id, cid_edges=None):
+        cid_edges = cid_edges or {}
+        division_data = self._download_json(
+            'https://api.bilibili.com/x/stein/edgeinfo_v2', video_id,
+            query={'graph_version': graph_version, 'edge_id': edge_id, 'bvid': video_id},
+            note=f'Extracting divisions from edge {edge_id}')
+        edges.setdefault(edge_id, {}).update(
+            traverse_obj(division_data, ('data', 'story_list', lambda _, v: v['edge_id'] == edge_id, {
+                'title': ('title', {str}),
+                'cid': ('cid', {int_or_none}),
+            }), get_all=False))
+
+        edges[edge_id].update(traverse_obj(division_data, ('data', {
+            'title': ('title', {str}),
+            'choices': ('edges', 'questions', ..., 'choices', ..., {
+                'edge_id': ('id', {int_or_none}),
+                'cid': ('cid', {int_or_none}),
+                'text': ('option', {str}),
+            }),
+        })))
+        # use dict to combine edges that use the same video section (same cid)
+        cid_edges.setdefault(edges[edge_id]['cid'], {})[edge_id] = edges[edge_id]
+        for choice in traverse_obj(edges, (edge_id, 'choices', ...)):
+            if choice['edge_id'] not in edges:
+                edges[choice['edge_id']] = {'cid': choice['cid']}
+                self._get_divisions(video_id, graph_version, edges, choice['edge_id'], cid_edges=cid_edges)
+        return cid_edges
+
+    def _get_interactive_entries(self, video_id, cid, metainfo):
+        graph_version = traverse_obj(
+            self._download_json(
+                'https://api.bilibili.com/x/player/wbi/v2', video_id,
+                'Extracting graph version', query={'bvid': video_id, 'cid': cid}),
+            ('data', 'interaction', 'graph_version', {int_or_none}))
+        cid_edges = self._get_divisions(video_id, graph_version, {1: {'cid': cid}}, 1)
+        for cid, edges in cid_edges.items():
+            play_info = self._download_playinfo(video_id, cid)
+            yield {
+                **metainfo,
+                'id': f'{video_id}_{cid}',
+                'title': f'{metainfo.get("title")} - {list(edges.values())[0].get("title")}',
+                'formats': self.extract_formats(play_info),
+                'description': f'{json.dumps(edges, ensure_ascii=False)}\n{metainfo.get("description", "")}',
+                'duration': float_or_none(play_info.get('timelength'), scale=1000),
+                'subtitles': self.extract_subtitles(video_id, cid),
+            }
 
 
 class BiliBiliIE(BilibiliBaseIE):
@@ -180,7 +243,7 @@ class BiliBiliIE(BilibiliBaseIE):
             'view_count': int,
         },
     }, {
-        # old av URL version
+        'note': 'old av URL version',
         'url': 'http://www.bilibili.com/video/av1074402/',
         'info_dict': {
             'thumbnail': r're:^https?://.*\.(jpg|jpeg)$',
@@ -212,7 +275,7 @@ class BiliBiliIE(BilibiliBaseIE):
                 'id': 'BV1bK411W797_p1',
                 'ext': 'mp4',
                 'title': '物语中的人物是如何吐槽自己的OP的 p01 Staple Stable/战场原+羽川',
-                'tags': 'count:11',
+                'tags': 'count:10',
                 'timestamp': 1589601697,
                 'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
                 'uploader': '打牌还是打桩',
@@ -232,7 +295,7 @@ class BiliBiliIE(BilibiliBaseIE):
             'id': 'BV1bK411W797_p1',
             'ext': 'mp4',
             'title': '物语中的人物是如何吐槽自己的OP的 p01 Staple Stable/战场原+羽川',
-            'tags': 'count:11',
+            'tags': 'count:10',
             'timestamp': 1589601697,
             'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
             'uploader': '打牌还是打桩',
@@ -343,18 +406,120 @@ class BiliBiliIE(BilibiliBaseIE):
             'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
         },
         'params': {'skip_download': True},
+    }, {
+        'note': 'interactive/split-path video',
+        'url': 'https://www.bilibili.com/video/BV1af4y1H7ga/',
+        'info_dict': {
+            'id': 'BV1af4y1H7ga',
+            'title': '【互动游戏】花了大半年时间做的自我介绍~请查收！！',
+            'timestamp': 1630500414,
+            'upload_date': '20210901',
+            'description': 'md5:01113e39ab06e28042d74ac356a08786',
+            'tags': list,
+            'uploader': '钉宫妮妮Ninico',
+            'duration': 1503,
+            'uploader_id': '8881297',
+            'comment_count': int,
+            'view_count': int,
+            'like_count': int,
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+        },
+        'playlist_count': 33,
+        'playlist': [{
+            'info_dict': {
+                'id': 'BV1af4y1H7ga_400950101',
+                'ext': 'mp4',
+                'title': '【互动游戏】花了大半年时间做的自我介绍~请查收！！ - 听见猫猫叫~',
+                'timestamp': 1630500414,
+                'upload_date': '20210901',
+                'description': 'md5:db66ac7a2813a94b8291dbce990cc5b2',
+                'tags': list,
+                'uploader': '钉宫妮妮Ninico',
+                'duration': 11.605,
+                'uploader_id': '8881297',
+                'comment_count': int,
+                'view_count': int,
+                'like_count': int,
+                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            },
+        }],
+    }, {
+        'note': '301 redirect to bangumi link',
+        'url': 'https://www.bilibili.com/video/BV1TE411f7f1',
+        'info_dict': {
+            'id': '288525',
+            'title': '李永乐老师 钱学森弹道和乘波体飞行器是什么？',
+            'ext': 'mp4',
+            'series': '我和我的祖国',
+            'series_id': '4780',
+            'season': '幕后纪实',
+            'season_id': '28609',
+            'season_number': 1,
+            'episode': '钱学森弹道和乘波体飞行器是什么？',
+            'episode_id': '288525',
+            'episode_number': 105,
+            'duration': 1183.957,
+            'timestamp': 1571648124,
+            'upload_date': '20191021',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+        },
+    }, {
+        'url': 'https://www.bilibili.com/video/BV1jL41167ZG/',
+        'info_dict': {
+            'id': 'BV1jL41167ZG',
+            'title': '一场大火引发的离奇死亡！古典推理经典短篇集《不可能犯罪诊断书》！',
+            'ext': 'mp4',
+        },
+        'skip': 'supporter-only video',
+    }, {
+        'url': 'https://www.bilibili.com/video/BV1Ks411f7aQ/',
+        'info_dict': {
+            'id': 'BV1Ks411f7aQ',
+            'title': '【BD1080P】狼与香辛料I【华盟】',
+            'ext': 'mp4',
+        },
+        'skip': 'login required',
+    }, {
+        'url': 'https://www.bilibili.com/video/BV1GJ411x7h7/',
+        'info_dict': {
+            'id': 'BV1GJ411x7h7',
+            'title': '【官方 MV】Never Gonna Give You Up - Rick Astley',
+            'ext': 'mp4',
+        },
+        'skip': 'geo-restricted',
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage, urlh = self._download_webpage_handle(url, video_id)
+        if not self._match_valid_url(urlh.url):
+            return self.url_result(urlh.url)
+
         initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', video_id)
 
         is_festival = 'videoData' not in initial_state
         if is_festival:
             video_data = initial_state['videoInfo']
         else:
-            play_info = self._search_json(r'window\.__playinfo__\s*=', webpage, 'play info', video_id)['data']
+            play_info_obj = self._search_json(
+                r'window\.__playinfo__\s*=', webpage, 'play info', video_id, fatal=False)
+            if not play_info_obj:
+                if traverse_obj(initial_state, ('error', 'trueCode')) == -403:
+                    self.raise_login_required()
+                if traverse_obj(initial_state, ('error', 'trueCode')) == -404:
+                    raise ExtractorError(
+                        'This video may be deleted or geo-restricted. '
+                        'You might want to try a VPN or a proxy server (with --proxy)', expected=True)
+            play_info = traverse_obj(play_info_obj, ('data', {dict}))
+            if not play_info:
+                if traverse_obj(play_info_obj, 'code') == 87007:
+                    toast = get_element_by_class('tips-toast', webpage) or ''
+                    msg = clean_html(
+                        f'{get_element_by_class("belongs-to", toast) or ""}，'
+                        + (get_element_by_class('level', toast) or ''))
+                    raise ExtractorError(
+                        f'This is a supporter-only video: {msg}. {self._login_hint()}', expected=True)
+                raise ExtractorError('Failed to extract play info')
             video_data = initial_state['videoData']
 
         video_id, title = video_data['bvid'], video_data.get('title')
@@ -385,10 +550,7 @@ def _real_extract(self, url):
 
         festival_info = {}
         if is_festival:
-            play_info = self._download_json(
-                'https://api.bilibili.com/x/player/playurl', video_id,
-                query={'bvid': video_id, 'cid': cid, 'fnval': 4048},
-                note='Extracting festival video formats')['data']
+            play_info = self._download_playinfo(video_id, cid)
 
             festival_info = traverse_obj(initial_state, {
                 'uploader': ('videoInfo', 'upName'),
@@ -397,7 +559,7 @@ def _real_extract(self, url):
                 'thumbnail': ('sectionEpisodes', lambda _, v: v['bvid'] == video_id, 'cover'),
             }, get_all=False)
 
-        return {
+        metainfo = {
             **traverse_obj(initial_state, {
                 'uploader': ('upData', 'name'),
                 'uploader_id': ('upData', 'mid', {str_or_none}),
@@ -413,28 +575,59 @@ def _real_extract(self, url):
                 'comment_count': ('stat', 'reply', {int_or_none}),
             }, get_all=False),
             'id': f'{video_id}{format_field(part_id, None, "_p%d")}',
-            'formats': self.extract_formats(play_info),
             '_old_archive_ids': [make_archive_id(self, old_video_id)] if old_video_id else None,
             'title': title,
-            'duration': float_or_none(play_info.get('timelength'), scale=1000),
-            'chapters': self._get_chapters(aid, cid),
-            'subtitles': self.extract_subtitles(video_id, aid, cid),
-            '__post_extractor': self.extract_comments(aid),
             'http_headers': {'Referer': url},
         }
 
+        is_interactive = traverse_obj(video_data, ('rights', 'is_stein_gate'))
+        if is_interactive:
+            return self.playlist_result(
+                self._get_interactive_entries(video_id, cid, metainfo), **metainfo, **{
+                    'duration': traverse_obj(initial_state, ('videoData', 'duration', {int_or_none})),
+                    '__post_extractor': self.extract_comments(aid),
+                })
+        else:
+            return {
+                **metainfo,
+                'duration': float_or_none(play_info.get('timelength'), scale=1000),
+                'chapters': self._get_chapters(aid, cid),
+                'subtitles': self.extract_subtitles(video_id, cid),
+                'formats': self.extract_formats(play_info),
+                '__post_extractor': self.extract_comments(aid),
+            }
+
 
 class BiliBiliBangumiIE(BilibiliBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/bangumi/play/(?P<id>ep\d+)'
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/bangumi/play/ep(?P<id>\d+)'
 
     _TESTS = [{
+        'url': 'https://www.bilibili.com/bangumi/play/ep21495/',
+        'info_dict': {
+            'id': '21495',
+            'ext': 'mp4',
+            'series': '悠久之翼',
+            'series_id': '774',
+            'season': '第二季',
+            'season_id': '1182',
+            'season_number': 2,
+            'episode': 'forever／ef',
+            'episode_id': '21495',
+            'episode_number': 12,
+            'title': '12 forever／ef',
+            'duration': 1420.791,
+            'timestamp': 1320412200,
+            'upload_date': '20111104',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+        },
+    }, {
         'url': 'https://www.bilibili.com/bangumi/play/ep267851',
         'info_dict': {
             'id': '267851',
             'ext': 'mp4',
             'series': '鬼灭之刃',
             'series_id': '4358',
-            'season': '鬼灭之刃',
+            'season': '立志篇',
             'season_id': '26801',
             'season_number': 1,
             'episode': '残酷',
@@ -446,13 +639,32 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
             'upload_date': '20190406',
             'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
         },
-        'skip': 'According to the copyright owner\'s request, you may only watch the video after you are premium member.'
+        'skip': 'Geo-restricted',
+    }, {
+        'note': 'a making-of which falls outside main section',
+        'url': 'https://www.bilibili.com/bangumi/play/ep345120',
+        'info_dict': {
+            'id': '345120',
+            'ext': 'mp4',
+            'series': '鬼灭之刃',
+            'series_id': '4358',
+            'season': '立志篇',
+            'season_id': '26801',
+            'season_number': 1,
+            'episode': '炭治郎篇',
+            'episode_id': '345120',
+            'episode_number': 27,
+            'title': '#1 炭治郎篇',
+            'duration': 1922.129,
+            'timestamp': 1602853860,
+            'upload_date': '20201016',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
+        },
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        episode_id = video_id[2:]
-        webpage = self._download_webpage(url, video_id)
+        episode_id = self._match_id(url)
+        webpage = self._download_webpage(url, episode_id)
 
         if '您所在的地区无法观看本片' in webpage:
             raise GeoRestrictedError('This video is restricted')
@@ -461,7 +673,7 @@ def _real_extract(self, url):
 
         headers = {'Referer': url, **self.geo_verification_headers()}
         play_info = self._download_json(
-            'https://api.bilibili.com/pgc/player/web/v2/playurl', video_id,
+            'https://api.bilibili.com/pgc/player/web/v2/playurl', episode_id,
             'Extracting episode', query={'fnval': '4048', 'ep_id': episode_id},
             headers=headers)
         premium_only = play_info.get('code') == -10403
@@ -472,40 +684,43 @@ def _real_extract(self, url):
             self.raise_login_required('This video is for premium members only')
 
         bangumi_info = self._download_json(
-            'https://api.bilibili.com/pgc/view/web/season', video_id, 'Get episode details',
+            'https://api.bilibili.com/pgc/view/web/season', episode_id, 'Get episode details',
             query={'ep_id': episode_id}, headers=headers)['result']
 
         episode_number, episode_info = next((
             (idx, ep) for idx, ep in enumerate(traverse_obj(
-                bangumi_info, ('episodes', ..., {dict})), 1)
+                bangumi_info, (('episodes', ('section', ..., 'episodes')), ..., {dict})), 1)
             if str_or_none(ep.get('id')) == episode_id), (1, {}))
 
         season_id = bangumi_info.get('season_id')
-        season_number = season_id and next((
-            idx + 1 for idx, e in enumerate(
+        season_number, season_title = season_id and next((
+            (idx + 1, e.get('season_title')) for idx, e in enumerate(
                 traverse_obj(bangumi_info, ('seasons', ...)))
             if e.get('season_id') == season_id
-        ), None)
+        ), (None, None))
 
         aid = episode_info.get('aid')
 
         return {
-            'id': video_id,
+            'id': episode_id,
             'formats': formats,
             **traverse_obj(bangumi_info, {
                 'series': ('series', 'series_title', {str}),
                 'series_id': ('series', 'series_id', {str_or_none}),
                 'thumbnail': ('square_cover', {url_or_none}),
             }),
-            'title': join_nonempty('title', 'long_title', delim=' ', from_dict=episode_info),
-            'episode': episode_info.get('long_title'),
+            **traverse_obj(episode_info, {
+                'episode': ('long_title', {str}),
+                'episode_number': ('title', {int_or_none}, {lambda x: x or episode_number}),
+                'timestamp': ('pub_time', {int_or_none}),
+                'title': {lambda v: v and join_nonempty('title', 'long_title', delim=' ', from_dict=v)},
+            }),
             'episode_id': episode_id,
-            'episode_number': int_or_none(episode_info.get('title')) or episode_number,
+            'season': str_or_none(season_title),
             'season_id': str_or_none(season_id),
             'season_number': season_number,
-            'timestamp': int_or_none(episode_info.get('pub_time')),
             'duration': float_or_none(play_info.get('timelength'), scale=1000),
-            'subtitles': self.extract_subtitles(video_id, aid, episode_info.get('cid')),
+            'subtitles': self.extract_subtitles(episode_id, episode_info.get('cid'), aid=aid),
             '__post_extractor': self.extract_comments(aid),
             'http_headers': headers,
         }
@@ -517,17 +732,53 @@ class BiliBiliBangumiMediaIE(BilibiliBaseIE):
         'url': 'https://www.bilibili.com/bangumi/media/md24097891',
         'info_dict': {
             'id': '24097891',
+            'title': 'CAROLE & TUESDAY',
+            'description': 'md5:42417ad33d1eaa1c93bfd2dd1626b829',
         },
         'playlist_mincount': 25,
+    }, {
+        'url': 'https://www.bilibili.com/bangumi/media/md1565/',
+        'info_dict': {
+            'id': '1565',
+            'title': '攻壳机动队 S.A.C. 2nd GIG',
+            'description': 'md5:46cac00bafd645b97f4d6df616fc576d',
+        },
+        'playlist_count': 26,
+        'playlist': [{
+            'info_dict': {
+                'id': '68540',
+                'ext': 'mp4',
+                'series': '攻壳机动队',
+                'series_id': '1077',
+                'season': '第二季',
+                'season_id': '1565',
+                'season_number': 2,
+                'episode': '再启动 REEMBODY',
+                'episode_id': '68540',
+                'episode_number': 1,
+                'title': '1 再启动 REEMBODY',
+                'duration': 1525.777,
+                'timestamp': 1425074413,
+                'upload_date': '20150227',
+                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
+            },
+        }],
     }]
 
     def _real_extract(self, url):
         media_id = self._match_id(url)
         webpage = self._download_webpage(url, media_id)
-        ss_id = self._search_json(
-            r'window\.__INITIAL_STATE__\s*=', webpage, 'initial_state', media_id)['mediaInfo']['season_id']
 
-        return self.playlist_result(self._get_episodes_from_season(ss_id, url), media_id)
+        initial_state = self._search_json(
+            r'window\.__INITIAL_STATE__\s*=', webpage, 'initial_state', media_id)
+        ss_id = initial_state['mediaInfo']['season_id']
+
+        return self.playlist_result(
+            self._get_episodes_from_season(ss_id, url), media_id,
+            **traverse_obj(initial_state, ('mediaInfo', {
+                'title': ('title', {str}),
+                'description': ('evaluate', {str}),
+            })))
 
 
 class BiliBiliBangumiSeasonIE(BilibiliBaseIE):
@@ -535,15 +786,183 @@ class BiliBiliBangumiSeasonIE(BilibiliBaseIE):
     _TESTS = [{
         'url': 'https://www.bilibili.com/bangumi/play/ss26801',
         'info_dict': {
-            'id': '26801'
+            'id': '26801',
+            'title': '鬼灭之刃',
+            'description': 'md5:e2cc9848b6f69be6db79fc2a82d9661b',
         },
         'playlist_mincount': 26
+    }, {
+        'url': 'https://www.bilibili.com/bangumi/play/ss2251',
+        'info_dict': {
+            'id': '2251',
+            'title': '玲音',
+            'description': 'md5:1fd40e3df4c08d4d9d89a6a34844bdc4',
+        },
+        'playlist_count': 13,
+        'playlist': [{
+            'info_dict': {
+                'id': '50188',
+                'ext': 'mp4',
+                'series': '玲音',
+                'series_id': '1526',
+                'season': 'TV',
+                'season_id': '2251',
+                'season_number': 1,
+                'episode': 'WEIRD',
+                'episode_id': '50188',
+                'episode_number': 1,
+                'title': '1 WEIRD',
+                'duration': 1436.992,
+                'timestamp': 1343185080,
+                'upload_date': '20120725',
+                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
+            },
+        }],
     }]
 
     def _real_extract(self, url):
         ss_id = self._match_id(url)
+        webpage = self._download_webpage(url, ss_id)
+        metainfo = traverse_obj(
+            self._search_json(r'<script[^>]+type="application/ld\+json"[^>]*>', webpage, 'info', ss_id),
+            ('itemListElement', ..., {
+                'title': ('name', {str}),
+                'description': ('description', {str}),
+            }), get_all=False)
 
-        return self.playlist_result(self._get_episodes_from_season(ss_id, url), ss_id)
+        return self.playlist_result(self._get_episodes_from_season(ss_id, url), ss_id, **metainfo)
+
+
+class BilibiliCheeseBaseIE(BilibiliBaseIE):
+    _HEADERS = {'Referer': 'https://www.bilibili.com/'}
+
+    def _extract_episode(self, season_info, ep_id):
+        episode_info = traverse_obj(season_info, (
+            'episodes', lambda _, v: v['id'] == int(ep_id)), get_all=False)
+        aid, cid = episode_info['aid'], episode_info['cid']
+
+        if traverse_obj(episode_info, 'ep_status') == -1:
+            raise ExtractorError('This course episode is not yet available.', expected=True)
+        if not traverse_obj(episode_info, 'playable'):
+            self.raise_login_required('You need to purchase the course to download this episode')
+
+        play_info = self._download_json(
+            'https://api.bilibili.com/pugv/player/web/playurl', ep_id,
+            query={'avid': aid, 'cid': cid, 'ep_id': ep_id, 'fnval': 16, 'fourk': 1},
+            headers=self._HEADERS, note='Downloading playinfo')['data']
+
+        return {
+            'id': str_or_none(ep_id),
+            'episode_id': str_or_none(ep_id),
+            'formats': self.extract_formats(play_info),
+            'extractor_key': BilibiliCheeseIE.ie_key(),
+            'extractor': BilibiliCheeseIE.IE_NAME,
+            'webpage_url': f'https://www.bilibili.com/cheese/play/ep{ep_id}',
+            **traverse_obj(episode_info, {
+                'episode': ('title', {str}),
+                'title': {lambda v: v and join_nonempty('index', 'title', delim=' - ', from_dict=v)},
+                'alt_title': ('subtitle', {str}),
+                'duration': ('duration', {int_or_none}),
+                'episode_number': ('index', {int_or_none}),
+                'thumbnail': ('cover', {url_or_none}),
+                'timestamp': ('release_date', {int_or_none}),
+                'view_count': ('play', {int_or_none}),
+            }),
+            **traverse_obj(season_info, {
+                'uploader': ('up_info', 'uname', {str}),
+                'uploader_id': ('up_info', 'mid', {str_or_none}),
+            }),
+            'subtitles': self.extract_subtitles(ep_id, cid, aid=aid),
+            '__post_extractor': self.extract_comments(aid),
+            'http_headers': self._HEADERS,
+        }
+
+    def _download_season_info(self, query_key, video_id):
+        return self._download_json(
+            f'https://api.bilibili.com/pugv/view/web/season?{query_key}={video_id}', video_id,
+            headers=self._HEADERS, note='Downloading season info')['data']
+
+
+class BilibiliCheeseIE(BilibiliCheeseBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/cheese/play/ep(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.bilibili.com/cheese/play/ep229832',
+        'info_dict': {
+            'id': '229832',
+            'ext': 'mp4',
+            'title': '1 - 课程先导片',
+            'alt_title': '视频课 · 3分41秒',
+            'uploader': '马督工',
+            'uploader_id': '316568752',
+            'episode': '课程先导片',
+            'episode_id': '229832',
+            'episode_number': 1,
+            'duration': 221,
+            'timestamp': 1695549606,
+            'upload_date': '20230924',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'view_count': int,
+        }
+    }]
+
+    def _real_extract(self, url):
+        ep_id = self._match_id(url)
+        return self._extract_episode(self._download_season_info('ep_id', ep_id), ep_id)
+
+
+class BilibiliCheeseSeasonIE(BilibiliCheeseBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/cheese/play/ss(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.bilibili.com/cheese/play/ss5918',
+        'info_dict': {
+            'id': '5918',
+            'title': '【限时五折】新闻系学不到：马督工教你做自媒体',
+            'description': '帮普通人建立世界模型，降低人与人的沟通门槛',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '229832',
+                'ext': 'mp4',
+                'title': '1 - 课程先导片',
+                'alt_title': '视频课 · 3分41秒',
+                'uploader': '马督工',
+                'uploader_id': '316568752',
+                'episode': '课程先导片',
+                'episode_id': '229832',
+                'episode_number': 1,
+                'duration': 221,
+                'timestamp': 1695549606,
+                'upload_date': '20230924',
+                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+                'view_count': int,
+            }
+        }],
+        'params': {'playlist_items': '1'},
+    }, {
+        'url': 'https://www.bilibili.com/cheese/play/ss5918',
+        'info_dict': {
+            'id': '5918',
+            'title': '【限时五折】新闻系学不到：马督工教你做自媒体',
+            'description': '帮普通人建立世界模型，降低人与人的沟通门槛',
+        },
+        'playlist_mincount': 5,
+        'skip': 'paid video in list',
+    }]
+
+    def _get_cheese_entries(self, season_info):
+        for ep_id in traverse_obj(season_info, ('episodes', lambda _, v: v['episode_can_view'], 'id')):
+            yield self._extract_episode(season_info, ep_id)
+
+    def _real_extract(self, url):
+        season_id = self._match_id(url)
+        season_info = self._download_season_info('season_id', season_id)
+
+        return self.playlist_result(
+            self._get_cheese_entries(season_info), season_id,
+            **traverse_obj(season_info, {
+                'title': ('title', {str}),
+                'description': ('subtitle', {str}),
+            }))
 
 
 class BilibiliSpaceBaseIE(InfoExtractor):

From 1725e943b0e8a8b585305660d4611e684374409c Mon Sep 17 00:00:00 2001
From: Kyraminol Endyeran <kyraminari@gmail.com>
Date: Sun, 19 Nov 2023 22:30:21 +0100
Subject: [PATCH 705/871] [ie/vvvvid] Set user-agent to fix extraction (#8615)

Authored by: Kyraminol
---
 yt_dlp/extractor/vvvvid.py | 81 +++++++++++++++++++++++++++++---------
 1 file changed, 62 insertions(+), 19 deletions(-)

diff --git a/yt_dlp/extractor/vvvvid.py b/yt_dlp/extractor/vvvvid.py
index ed725a55d..b42ba8537 100644
--- a/yt_dlp/extractor/vvvvid.py
+++ b/yt_dlp/extractor/vvvvid.py
@@ -1,3 +1,4 @@
+import functools
 import re
 
 from .common import InfoExtractor
@@ -14,21 +15,21 @@ class VVVVIDIE(InfoExtractor):
     _VALID_URL = r'%s(?P<show_id>\d+)/[^/]+/(?P<season_id>\d+)/(?P<id>[0-9]+)' % _VALID_URL_BASE
     _TESTS = [{
         # video_type == 'video/vvvvid'
-        'url': 'https://www.vvvvid.it/#!show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048/ping-pong',
-        'md5': 'b8d3cecc2e981adc3835adf07f6df91b',
+        'url': 'https://www.vvvvid.it/show/498/the-power-of-computing/518/505692/playstation-vr-cambiera-il-nostro-modo-di-giocare',
         'info_dict': {
-            'id': '489048',
+            'id': '505692',
             'ext': 'mp4',
-            'title': 'Ping Pong',
-            'duration': 239,
-            'series': '"Perché dovrei guardarlo?" di Dario Moccia',
-            'season_id': '437',
-            'episode': 'Ping Pong',
-            'episode_number': 1,
-            'episode_id': '3334',
+            'title': 'Playstation VR cambierà il nostro modo di giocare',
+            'duration': 93,
+            'series': 'The Power of Computing',
+            'season_id': '518',
+            'episode': 'Playstation VR cambierà il nostro modo di giocare',
+            'episode_number': None,
+            'episode_id': '4747',
             'view_count': int,
             'like_count': int,
             'repost_count': int,
+            'thumbnail': 'https://static.vvvvid.it/img/zoomin/28CA2409-E663-34F0-2B02E72356556EA3_500k.jpg',
         },
         'params': {
             'skip_download': True,
@@ -36,7 +37,6 @@ class VVVVIDIE(InfoExtractor):
     }, {
         # video_type == 'video/rcs'
         'url': 'https://www.vvvvid.it/#!show/376/death-note-live-action/377/482493/episodio-01',
-        'md5': '33e0edfba720ad73a8782157fdebc648',
         'info_dict': {
             'id': '482493',
             'ext': 'mp4',
@@ -45,6 +45,7 @@ class VVVVIDIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'Every video/rcs is not working even in real website',
     }, {
         # video_type == 'video/youtube'
         'url': 'https://www.vvvvid.it/show/404/one-punch-man/406/486683/trailer',
@@ -55,19 +56,54 @@ class VVVVIDIE(InfoExtractor):
             'title': 'Trailer',
             'upload_date': '20150906',
             'description': 'md5:a5e802558d35247fee285875328c0b80',
-            'uploader_id': 'BandaiVisual',
-            'uploader': 'BANDAI NAMCO Arts Channel',
+            'uploader_id': '@EMOTIONLabelChannel',
+            'uploader': 'EMOTION Label Channel',
+            'episode_number': None,
+            'episode_id': '3115',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'availability': str,
+            'categories': list,
+            'age_limit': 0,
+            'channel': 'EMOTION Label Channel',
+            'channel_follower_count': int,
+            'channel_id': 'UCQ5URCSs1f5Cz9rh-cDGxNQ',
+            'channel_url': 'https://www.youtube.com/channel/UCQ5URCSs1f5Cz9rh-cDGxNQ',
+            'comment_count': int,
+            'duration': 133,
+            'episode': 'Trailer',
+            'heatmap': list,
+            'live_status': 'not_live',
+            'playable_in_embed': True,
+            'season_id': '406',
+            'series': 'One-Punch Man',
+            'tags': list,
+            'uploader_url': 'https://www.youtube.com/@EMOTIONLabelChannel',
+            'thumbnail': 'https://i.ytimg.com/vi/RzmFKUDOUgw/maxresdefault.jpg',
         },
         'params': {
             'skip_download': True,
         },
     }, {
         # video_type == 'video/dash'
-        'url': 'https://www.vvvvid.it/show/683/made-in-abyss/1542/693786/nanachi',
+        'url': 'https://www.vvvvid.it/show/844/le-bizzarre-avventure-di-jojo-vento-aureo/938/527551/golden-wind',
         'info_dict': {
-            'id': '693786',
+            'id': '527551',
             'ext': 'mp4',
-            'title': 'Nanachi',
+            'title': 'Golden Wind',
+            'duration': 1430,
+            'series': 'Le bizzarre avventure di Jojo - Vento Aureo',
+            'season_id': '938',
+            'episode': 'Golden Wind',
+            'episode_number': 1,
+            'episode_id': '9089',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'thumbnail': 'https://static.vvvvid.it/img/thumbs/Dynit/Jojo/Jojo_S05Ep01-t.jpg',
+            'season': 'Season 5',
+            'season_number': 5,
         },
         'params': {
             'skip_download': True,
@@ -79,10 +115,17 @@ class VVVVIDIE(InfoExtractor):
     }]
     _conn_id = None
 
+    @functools.cached_property
+    def _headers(self):
+        return {
+            **self.geo_verification_headers(),
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.50 Safari/537.37',
+        }
+
     def _real_initialize(self):
         self._conn_id = self._download_json(
             'https://www.vvvvid.it/user/login',
-            None, headers=self.geo_verification_headers())['data']['conn_id']
+            None, headers=self._headers)['data']['conn_id']
 
     def _download_info(self, show_id, path, video_id, fatal=True, query=None):
         q = {
@@ -92,7 +135,7 @@ def _download_info(self, show_id, path, video_id, fatal=True, query=None):
             q.update(query)
         response = self._download_json(
             'https://www.vvvvid.it/vvvvid/ondemand/%s/%s' % (show_id, path),
-            video_id, headers=self.geo_verification_headers(), query=q, fatal=fatal)
+            video_id, headers=self._headers, query=q, fatal=fatal)
         if not (response or fatal):
             return
         if response.get('result') == 'error':
@@ -219,7 +262,7 @@ def metadata_from_url(r_url):
                     embed_code, video_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
                 formats.extend(self._extract_wowza_formats(
-                    'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))
+                    'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id, skip_protocols=['f4m']))
             metadata_from_url(embed_code)
 
         if not is_youtube:

From 3237f8ba29fe13bf95ff42b1e48b5b5109715feb Mon Sep 17 00:00:00 2001
From: Safouane Aarab <32977495+S-Aarab@users.noreply.github.com>
Date: Mon, 20 Nov 2023 01:07:19 +0100
Subject: [PATCH 706/871] [ie/allstar] Add extractors (#8274)

Closes #6917
Authored by: S-Aarab
---
 yt_dlp/extractor/_extractors.py |   6 +-
 yt_dlp/extractor/allstar.py     | 253 ++++++++++++++++++++++++++++++++
 2 files changed, 258 insertions(+), 1 deletion(-)
 create mode 100644 yt_dlp/extractor/allstar.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 76537e13e..220aa4353 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -81,16 +81,20 @@
 from .airtv import AirTVIE
 from .aitube import AitubeKZVideoIE
 from .aljazeera import AlJazeeraIE
+from .allstar import (
+    AllstarIE,
+    AllstarProfileIE,
+)
 from .alphaporno import AlphaPornoIE
 from .altcensored import (
     AltCensoredIE,
     AltCensoredChannelIE,
 )
-from .amara import AmaraIE
 from .alura import (
     AluraIE,
     AluraCourseIE
 )
+from .amara import AmaraIE
 from .amcnetworks import AMCNetworksIE
 from .amazon import (
     AmazonStoreIE,
diff --git a/yt_dlp/extractor/allstar.py b/yt_dlp/extractor/allstar.py
new file mode 100644
index 000000000..87219f2f8
--- /dev/null
+++ b/yt_dlp/extractor/allstar.py
@@ -0,0 +1,253 @@
+import functools
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    OnDemandPagedList,
+    int_or_none,
+    join_nonempty,
+    parse_qs,
+    urljoin,
+)
+from ..utils.traversal import traverse_obj
+
+
+_FIELDS = '''
+    _id
+    clipImageSource
+    clipImageThumb
+    clipLink
+    clipTitle
+    createdDate
+    shareId
+    user { _id }
+    username
+    views'''
+
+_EXTRA_FIELDS = '''
+    clipLength
+    clipSizeBytes'''
+
+_QUERIES = {
+    'clip': '''query ($id: String!) {
+        video: getClip(clipIdentifier: $id) {
+            %s %s
+        }
+    }''' % (_FIELDS, _EXTRA_FIELDS),
+    'montage': '''query ($id: String!) {
+        video: getMontage(clipIdentifier: $id) {
+            %s
+        }
+    }''' % _FIELDS,
+    'Clips': '''query ($page: Int!, $user: String!, $game: Int) {
+        videos: clips(search: createdDate, page: $page, user: $user, mobile: false, game: $game) {
+            data { %s %s }
+        }
+    }''' % (_FIELDS, _EXTRA_FIELDS),
+    'Montages': '''query ($page: Int!, $user: String!) {
+        videos: montages(search: createdDate, page: $page, user: $user) {
+            data { %s }
+        }
+    }''' % _FIELDS,
+    'Mobile Clips': '''query ($page: Int!, $user: String!) {
+        videos: clips(search: createdDate, page: $page, user: $user, mobile: true) {
+            data { %s %s }
+        }
+    }''' % (_FIELDS, _EXTRA_FIELDS),
+}
+
+
+class AllstarBaseIE(InfoExtractor):
+    @staticmethod
+    def _parse_video_data(video_data):
+        def media_url_or_none(path):
+            return urljoin('https://media.allstar.gg/', path)
+
+        info = traverse_obj(video_data, {
+            'id': ('_id', {str}),
+            'display_id': ('shareId', {str}),
+            'title': ('clipTitle', {str}),
+            'url': ('clipLink', {media_url_or_none}),
+            'thumbnails': (('clipImageThumb', 'clipImageSource'), {'url': {media_url_or_none}}),
+            'duration': ('clipLength', {int_or_none}),
+            'filesize': ('clipSizeBytes', {int_or_none}),
+            'timestamp': ('createdDate', {functools.partial(int_or_none, scale=1000)}),
+            'uploader': ('username', {str}),
+            'uploader_id': ('user', '_id', {str}),
+            'view_count': ('views', {int_or_none}),
+        })
+
+        if info.get('id') and info.get('url'):
+            basename = 'clip' if '/clips/' in info['url'] else 'montage'
+            info['webpage_url'] = f'https://allstar.gg/{basename}?{basename}={info["id"]}'
+
+        info.update({
+            'extractor_key': AllstarIE.ie_key(),
+            'extractor': AllstarIE.IE_NAME,
+            'uploader_url': urljoin('https://allstar.gg/u/', info.get('uploader_id')),
+        })
+
+        return info
+
+    def _call_api(self, query, variables, path, video_id=None, note=None):
+        response = self._download_json(
+            'https://a1.allstar.gg/graphql', video_id, note=note,
+            headers={'content-type': 'application/json'},
+            data=json.dumps({'variables': variables, 'query': query}).encode())
+
+        errors = traverse_obj(response, ('errors', ..., 'message', {str}))
+        if errors:
+            raise ExtractorError('; '.join(errors))
+
+        return traverse_obj(response, path)
+
+
+class AllstarIE(AllstarBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?P<type>(?:clip|montage))\?(?P=type)=(?P<id>[^/?#&]+)'
+
+    _TESTS = [{
+        'url': 'https://allstar.gg/clip?clip=64482c2da9eec30008a67d1b',
+        'info_dict': {
+            'id': '64482c2da9eec30008a67d1b',
+            'title': '4K on Inferno',
+            'url': 'md5:66befb5381eef0c9456026386c25fa55',
+            'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
+            'uploader': 'chrk.',
+            'ext': 'mp4',
+            'duration': 20,
+            'filesize': 21199257,
+            'timestamp': 1682451501,
+            'uploader_id': '62b8bdfc9021052f7905882d',
+            'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
+            'upload_date': '20230425',
+            'view_count': int,
+        }
+    }, {
+        'url': 'https://allstar.gg/clip?clip=8LJLY4JKB',
+        'info_dict': {
+            'id': '64a1ec6b887f4c0008dc50b8',
+            'display_id': '8LJLY4JKB',
+            'title': 'AK-47 3K on Mirage',
+            'url': 'md5:dde224fd12f035c0e2529a4ae34c4283',
+            'ext': 'mp4',
+            'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
+            'duration': 16,
+            'filesize': 30175859,
+            'timestamp': 1688333419,
+            'uploader': 'cherokee',
+            'uploader_id': '62b8bdfc9021052f7905882d',
+            'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
+            'upload_date': '20230702',
+            'view_count': int,
+        }
+    }, {
+        'url': 'https://allstar.gg/montage?montage=643e64089da7e9363e1fa66c',
+        'info_dict': {
+            'id': '643e64089da7e9363e1fa66c',
+            'display_id': 'APQLGM2IMXW',
+            'title': 'cherokee Rapid Fire Snipers Montage',
+            'url': 'md5:a3ee356022115db2b27c81321d195945',
+            'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
+            'ext': 'mp4',
+            'timestamp': 1681810448,
+            'uploader': 'cherokee',
+            'uploader_id': '62b8bdfc9021052f7905882d',
+            'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
+            'upload_date': '20230418',
+            'view_count': int,
+        }
+    }, {
+        'url': 'https://allstar.gg/montage?montage=RILJMH6QOS',
+        'info_dict': {
+            'id': '64a2697372ce3703de29e868',
+            'display_id': 'RILJMH6QOS',
+            'title': 'cherokee Rapid Fire Snipers Montage',
+            'url': 'md5:d5672e6f88579730c2310a80fdbc4030',
+            'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
+            'ext': 'mp4',
+            'timestamp': 1688365434,
+            'uploader': 'cherokee',
+            'uploader_id': '62b8bdfc9021052f7905882d',
+            'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
+            'upload_date': '20230703',
+            'view_count': int,
+        }
+    }]
+
+    def _real_extract(self, url):
+        query_id, video_id = self._match_valid_url(url).group('type', 'id')
+
+        return self._parse_video_data(
+            self._call_api(
+                _QUERIES.get(query_id), {'id': video_id}, ('data', 'video'), video_id))
+
+
+class AllstarProfileIE(AllstarBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?:profile\?user=|u/)(?P<id>[^/?#&]+)'
+
+    _TESTS = [{
+        'url': 'https://allstar.gg/profile?user=62b8bdfc9021052f7905882d',
+        'info_dict': {
+            'id': '62b8bdfc9021052f7905882d-clips',
+            'title': 'cherokee - Clips',
+        },
+        'playlist_mincount': 15
+    }, {
+        'url': 'https://allstar.gg/u/cherokee?game=730&view=Clips',
+        'info_dict': {
+            'id': '62b8bdfc9021052f7905882d-clips-730',
+            'title': 'cherokee - Clips - 730',
+        },
+        'playlist_mincount': 15
+    }, {
+        'url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d?view=Montages',
+        'info_dict': {
+            'id': '62b8bdfc9021052f7905882d-montages',
+            'title': 'cherokee - Montages',
+        },
+        'playlist_mincount': 4
+    }, {
+        'url': 'https://allstar.gg/profile?user=cherokee&view=Mobile Clips',
+        'info_dict': {
+            'id': '62b8bdfc9021052f7905882d-mobile',
+            'title': 'cherokee - Mobile Clips',
+        },
+        'playlist_mincount': 1
+    }]
+
+    _PAGE_SIZE = 10
+
+    def _get_page(self, user_id, display_id, game, query, page_num):
+        page_num += 1
+
+        for video_data in self._call_api(
+                query, {
+                    'user': user_id,
+                    'page': page_num,
+                    'game': game,
+                }, ('data', 'videos', 'data'), display_id, f'Downloading page {page_num}'):
+            yield self._parse_video_data(video_data)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        profile_data = self._download_json(
+            urljoin('https://api.allstar.gg/v1/users/profile/', display_id), display_id)
+        user_id = traverse_obj(profile_data, ('data', ('_id'), {str}))
+        if not user_id:
+            raise ExtractorError('Unable to extract the user id')
+
+        username = traverse_obj(profile_data, ('data', 'profile', ('username'), {str}))
+        url_query = parse_qs(url)
+        game = traverse_obj(url_query, ('game', 0, {int_or_none}))
+        query_id = traverse_obj(url_query, ('view', 0), default='Clips')
+
+        if query_id not in ('Clips', 'Montages', 'Mobile Clips'):
+            raise ExtractorError(f'Unsupported playlist URL type {query_id!r}')
+
+        return self.playlist_result(
+            OnDemandPagedList(
+                functools.partial(
+                    self._get_page, user_id, display_id, game, _QUERIES.get(query_id)), self._PAGE_SIZE),
+            playlist_id=join_nonempty(user_id, query_id.lower().split()[0], game),
+            playlist_title=join_nonempty((username or display_id), query_id, game, delim=' - '))

From 45d82be65f71bb05506bd55376c6fdb36bc54142 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Mon, 20 Nov 2023 02:03:33 +0100
Subject: [PATCH 707/871] [ie/nebula] Overhaul extractors (#8566)

Closes #4300, Closes #5814, Closes #7588, Closes #6334, Closes #6538
Authored by: elyse0, pukkandan, seproDev

Co-authored-by: Elyse <26639800+elyse0@users.noreply.github.com>
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
---
 test/helper.py                  |   5 +-
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/nebula.py      | 562 +++++++++++++++++++-------------
 3 files changed, 336 insertions(+), 232 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 539b2f618..7503840be 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -214,8 +214,9 @@ def sanitize(key, value):
 
     test_info_dict = {
         key: sanitize(key, value) for key, value in got_dict.items()
-        if value is not None and key not in IGNORED_FIELDS and not any(
-            key.startswith(f'{prefix}_') for prefix in IGNORED_PREFIXES)
+        if value is not None and key not in IGNORED_FIELDS and (
+            not any(key.startswith(f'{prefix}_') for prefix in IGNORED_PREFIXES)
+            or key == '_old_archive_ids')
     }
 
     # display_id may be generated from id
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 220aa4353..128b86c1a 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1247,6 +1247,7 @@
 from .ndtv import NDTVIE
 from .nebula import (
     NebulaIE,
+    NebulaClassIE,
     NebulaSubscriptionsIE,
     NebulaChannelIE,
 )
diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py
index 8fba2bcf7..136b0e10a 100644
--- a/yt_dlp/extractor/nebula.py
+++ b/yt_dlp/extractor/nebula.py
@@ -3,230 +3,306 @@
 
 from .common import InfoExtractor
 from ..networking.exceptions import HTTPError
-from ..utils import ExtractorError, make_archive_id, parse_iso8601, remove_start
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    make_archive_id,
+    parse_iso8601,
+    smuggle_url,
+    try_call,
+    unsmuggle_url,
+    update_url_query,
+    url_or_none,
+    urljoin,
+)
+from ..utils.traversal import traverse_obj
 
 _BASE_URL_RE = r'https?://(?:www\.|beta\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)'
 
 
 class NebulaBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'watchnebula'
+    _token = _api_token = None
 
-    _nebula_api_token = None
-    _nebula_bearer_token = None
-
-    def _perform_nebula_auth(self, username, password):
-        if not username or not password:
-            self.raise_login_required(method='password')
-
-        data = json.dumps({'email': username, 'password': password}).encode('utf8')
-        response = self._download_json(
-            'https://api.watchnebula.com/api/v1/auth/login/',
-            data=data, fatal=False, video_id=None,
-            headers={
-                'content-type': 'application/json',
-                # Submitting the 'sessionid' cookie always causes a 403 on auth endpoint
-                'cookie': ''
-            },
-            note='Logging in to Nebula with supplied credentials',
-            errnote='Authentication failed or rejected')
-        if not response or not response.get('key'):
-            self.raise_login_required(method='password')
-
-        return response['key']
-
-    def _call_nebula_api(self, url, video_id=None, method='GET', auth_type='api', note=''):
-        assert method in ('GET', 'POST',)
-        assert auth_type in ('api', 'bearer',)
-
-        def inner_call():
-            authorization = f'Token {self._nebula_api_token}' if auth_type == 'api' else f'Bearer {self._nebula_bearer_token}'
-            return self._download_json(
-                url, video_id, note=note, headers={'Authorization': authorization},
-                data=b'' if method == 'POST' else None)
-
+    def _perform_login(self, username, password):
         try:
-            return inner_call()
-        except ExtractorError as exc:
-            # if 401 or 403, attempt credential re-auth and retry
-            if exc.cause and isinstance(exc.cause, HTTPError) and exc.cause.status in (401, 403):
-                self.to_screen(f'Reauthenticating to Nebula and retrying, because last {auth_type} call resulted in error {exc.cause.code}')
-                self._perform_login()
-                return inner_call()
-            else:
+            response = self._download_json(
+                'https://nebula.tv/auth/login/', None,
+                'Logging in to Nebula', 'Login failed',
+                data=json.dumps({'email': username, 'password': password}).encode(),
+                headers={'content-type': 'application/json'})
+        except ExtractorError as e:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
+                raise ExtractorError('Login failed: Invalid username or password', expected=True)
+            raise
+        self._api_token = traverse_obj(response, ('key', {str}))
+        if not self._api_token:
+            raise ExtractorError('Login failed: No token')
+
+    def _call_api(self, *args, **kwargs):
+        if self._token:
+            kwargs.setdefault('headers', {})['Authorization'] = f'Bearer {self._token}'
+        try:
+            return self._download_json(*args, **kwargs)
+        except ExtractorError as e:
+            if not isinstance(e.cause, HTTPError) or e.cause.status not in (401, 403):
+                raise
+            self.to_screen(
+                f'Reauthorizing with Nebula and retrying, because last API call resulted in error {e.cause.status}')
+            self._real_initialize()
+            if self._token:
+                kwargs.setdefault('headers', {})['Authorization'] = f'Bearer {self._token}'
+            return self._download_json(*args, **kwargs)
+
+    def _real_initialize(self):
+        if not self._api_token:
+            self._api_token = try_call(
+                lambda: self._get_cookies('https://nebula.tv')['nebula_auth.apiToken'].value)
+        self._token = self._download_json(
+            'https://users.api.nebula.app/api/v1/authorization/', None,
+            headers={'Authorization': f'Token {self._api_token}'} if self._api_token else None,
+            note='Authorizing to Nebula', data=b'')['token']
+
+    def _extract_formats(self, content_id, slug):
+        for retry in (False, True):
+            try:
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    f'https://content.api.nebula.app/{content_id.split(":")[0]}s/{content_id}/manifest.m3u8',
+                    slug, 'mp4', query={
+                        'token': self._token,
+                        'app_version': '23.10.0',
+                        'platform': 'ios',
+                    })
+                return {'formats': fmts, 'subtitles': subs}
+            except ExtractorError as e:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                    self.raise_login_required()
+                if not retry and isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                    self.to_screen('Reauthorizing with Nebula and retrying, because fetching video resulted in error')
+                    self._real_initialize()
+                    continue
                 raise
 
-    def _fetch_nebula_bearer_token(self):
-        """
-        Get a Bearer token for the Nebula API. This will be required to fetch video meta data.
-        """
-        response = self._call_nebula_api('https://api.watchnebula.com/api/v1/authorization/',
-                                         method='POST',
-                                         note='Authorizing to Nebula')
-        return response['token']
-
-    def _fetch_video_formats(self, slug):
-        stream_info = self._call_nebula_api(f'https://content.api.nebula.app/video/{slug}/stream/',
-                                            video_id=slug,
-                                            auth_type='bearer',
-                                            note='Fetching video stream info')
-        manifest_url = stream_info['manifest']
-        return self._extract_m3u8_formats_and_subtitles(manifest_url, slug, 'mp4')
-
-    def _build_video_info(self, episode):
-        fmts, subs = self._fetch_video_formats(episode['slug'])
-        channel_slug = episode['channel_slug']
-        channel_title = episode['channel_title']
-        zype_id = episode.get('zype_id')
+    def _extract_video_metadata(self, episode):
+        channel_url = traverse_obj(
+            episode, (('channel_slug', 'class_slug'), {lambda x: urljoin('https://nebula.tv/', x)}), get_all=False)
         return {
-            'id': remove_start(episode['id'], 'video_episode:'),
-            'display_id': episode['slug'],
-            'formats': fmts,
-            'subtitles': subs,
-            'webpage_url': f'https://nebula.tv/{episode["slug"]}',
-            'title': episode['title'],
-            'description': episode['description'],
-            'timestamp': parse_iso8601(episode['published_at']),
-            'thumbnails': [{
-                # 'id': tn.get('name'),  # this appears to be null
-                'url': tn['original'],
-                'height': key,
-            } for key, tn in episode['assets']['thumbnail'].items()],
-            'duration': episode['duration'],
-            'channel': channel_title,
-            'channel_id': channel_slug,
-            'channel_url': f'https://nebula.tv/{channel_slug}',
-            'uploader': channel_title,
-            'uploader_id': channel_slug,
-            'uploader_url': f'https://nebula.tv/{channel_slug}',
-            'series': channel_title,
-            'creator': channel_title,
-            'extractor_key': NebulaIE.ie_key(),
-            'extractor': NebulaIE.IE_NAME,
-            '_old_archive_ids': [make_archive_id(NebulaIE, zype_id)] if zype_id else None,
+            'id': episode['id'].partition(':')[2],
+            **traverse_obj(episode, {
+                'display_id': 'slug',
+                'title': 'title',
+                'description': 'description',
+                'timestamp': ('published_at', {parse_iso8601}),
+                'duration': ('duration', {int_or_none}),
+                'channel_id': 'channel_slug',
+                'uploader_id': 'channel_slug',
+                'channel': 'channel_title',
+                'uploader': 'channel_title',
+                'series': 'channel_title',
+                'creator': 'channel_title',
+                'thumbnail': ('images', 'thumbnail', 'src', {url_or_none}),
+                'episode_number': ('order', {int_or_none}),
+                # Old code was wrongly setting extractor_key from NebulaSubscriptionsIE
+                '_old_archive_ids': ('zype_id', {lambda x: [
+                    make_archive_id(NebulaIE, x), make_archive_id(NebulaSubscriptionsIE, x)] if x else None}),
+            }),
+            'channel_url': channel_url,
+            'uploader_url': channel_url,
         }
 
-    def _perform_login(self, username=None, password=None):
-        self._nebula_api_token = self._perform_nebula_auth(username, password)
-        self._nebula_bearer_token = self._fetch_nebula_bearer_token()
-
 
 class NebulaIE(NebulaBaseIE):
     _VALID_URL = rf'{_BASE_URL_RE}/videos/(?P<id>[-\w]+)'
-    _TESTS = [
-        {
-            'url': 'https://nebula.tv/videos/that-time-disney-remade-beauty-and-the-beast',
-            'md5': '14944cfee8c7beeea106320c47560efc',
-            'info_dict': {
-                'id': '84ed544d-4afd-4723-8cd5-2b95261f0abf',
-                'ext': 'mp4',
-                'title': 'That Time Disney Remade Beauty and the Beast',
-                'description': 'Note: this video was originally posted on YouTube with the sponsor read included. We weren’t able to remove it without reducing video quality, so it’s presented here in its original context.',
-                'upload_date': '20180731',
-                'timestamp': 1533009600,
-                'channel': 'Lindsay Ellis',
-                'channel_id': 'lindsayellis',
-                'uploader': 'Lindsay Ellis',
-                'uploader_id': 'lindsayellis',
-                'uploader_url': 'https://nebula.tv/lindsayellis',
-                'series': 'Lindsay Ellis',
-                'display_id': 'that-time-disney-remade-beauty-and-the-beast',
-                'channel_url': 'https://nebula.tv/lindsayellis',
-                'creator': 'Lindsay Ellis',
-                'duration': 2212,
-                'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*',
-            },
+    _TESTS = [{
+        'url': 'https://nebula.tv/videos/that-time-disney-remade-beauty-and-the-beast',
+        'info_dict': {
+            'id': '84ed544d-4afd-4723-8cd5-2b95261f0abf',
+            'ext': 'mp4',
+            'title': 'That Time Disney Remade Beauty and the Beast',
+            'description': 'md5:2aae3c4cfc5ee09a1ecdff0909618cf4',
+            'upload_date': '20180731',
+            'timestamp': 1533009600,
+            'channel': 'Lindsay Ellis',
+            'channel_id': 'lindsayellis',
+            'uploader': 'Lindsay Ellis',
+            'uploader_id': 'lindsayellis',
+            'uploader_url': r're:https://nebula\.(tv|app)/lindsayellis',
+            'series': 'Lindsay Ellis',
+            'display_id': 'that-time-disney-remade-beauty-and-the-beast',
+            'channel_url': r're:https://nebula\.(tv|app)/lindsayellis',
+            'creator': 'Lindsay Ellis',
+            'duration': 2212,
+            'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+',
+            '_old_archive_ids': ['nebula 5c271b40b13fd613090034fd', 'nebulasubscriptions 5c271b40b13fd613090034fd'],
         },
-        {
-            'url': 'https://nebula.tv/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
-            'md5': 'd05739cf6c38c09322422f696b569c23',
-            'info_dict': {
-                'id': '7e623145-1b44-4ca3-aa0b-ed25a247ea34',
-                'ext': 'mp4',
-                'title': 'Landing Craft - How The Allies Got Ashore',
-                'description': r're:^In this episode we explore the unsung heroes of D-Day, the landing craft.',
-                'upload_date': '20200327',
-                'timestamp': 1585348140,
-                'channel': 'Real Engineering — The Logistics of D-Day',
-                'channel_id': 'd-day',
-                'uploader': 'Real Engineering — The Logistics of D-Day',
-                'uploader_id': 'd-day',
-                'series': 'Real Engineering — The Logistics of D-Day',
-                'display_id': 'the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
-                'creator': 'Real Engineering — The Logistics of D-Day',
-                'duration': 841,
-                'channel_url': 'https://nebula.tv/d-day',
-                'uploader_url': 'https://nebula.tv/d-day',
-                'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*',
-            },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://nebula.tv/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
+        'md5': 'd05739cf6c38c09322422f696b569c23',
+        'info_dict': {
+            'id': '7e623145-1b44-4ca3-aa0b-ed25a247ea34',
+            'ext': 'mp4',
+            'title': 'Landing Craft - How The Allies Got Ashore',
+            'description': r're:^In this episode we explore the unsung heroes of D-Day, the landing craft.',
+            'upload_date': '20200327',
+            'timestamp': 1585348140,
+            'channel': 'Real Engineering — The Logistics of D-Day',
+            'channel_id': 'd-day',
+            'uploader': 'Real Engineering — The Logistics of D-Day',
+            'uploader_id': 'd-day',
+            'series': 'Real Engineering — The Logistics of D-Day',
+            'display_id': 'the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
+            'creator': 'Real Engineering — The Logistics of D-Day',
+            'duration': 841,
+            'channel_url': 'https://nebula.tv/d-day',
+            'uploader_url': 'https://nebula.tv/d-day',
+            'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+',
+            '_old_archive_ids': ['nebula 5e7e78171aaf320001fbd6be', 'nebulasubscriptions 5e7e78171aaf320001fbd6be'],
         },
-        {
-            'url': 'https://nebula.tv/videos/money-episode-1-the-draw',
-            'md5': 'ebe28a7ad822b9ee172387d860487868',
-            'info_dict': {
-                'id': 'b96c5714-9e2b-4ec3-b3f1-20f6e89cc553',
-                'ext': 'mp4',
-                'title': 'Episode 1: The Draw',
-                'description': r'contains:There’s free money on offer… if the players can all work together.',
-                'upload_date': '20200323',
-                'timestamp': 1584980400,
-                'channel': 'Tom Scott Presents: Money',
-                'channel_id': 'tom-scott-presents-money',
-                'uploader': 'Tom Scott Presents: Money',
-                'uploader_id': 'tom-scott-presents-money',
-                'uploader_url': 'https://nebula.tv/tom-scott-presents-money',
-                'duration': 825,
-                'channel_url': 'https://nebula.tv/tom-scott-presents-money',
-                'series': 'Tom Scott Presents: Money',
-                'display_id': 'money-episode-1-the-draw',
-                'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*',
-                'creator': 'Tom Scott Presents: Money',
-            },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://nebula.tv/videos/money-episode-1-the-draw',
+        'md5': 'ebe28a7ad822b9ee172387d860487868',
+        'info_dict': {
+            'id': 'b96c5714-9e2b-4ec3-b3f1-20f6e89cc553',
+            'ext': 'mp4',
+            'title': 'Episode 1: The Draw',
+            'description': r'contains:There’s free money on offer… if the players can all work together.',
+            'upload_date': '20200323',
+            'timestamp': 1584980400,
+            'channel': 'Tom Scott Presents: Money',
+            'channel_id': 'tom-scott-presents-money',
+            'uploader': 'Tom Scott Presents: Money',
+            'uploader_id': 'tom-scott-presents-money',
+            'uploader_url': 'https://nebula.tv/tom-scott-presents-money',
+            'duration': 825,
+            'channel_url': 'https://nebula.tv/tom-scott-presents-money',
+            'series': 'Tom Scott Presents: Money',
+            'display_id': 'money-episode-1-the-draw',
+            'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+',
+            'creator': 'Tom Scott Presents: Money',
+            '_old_archive_ids': ['nebula 5e779ebdd157bc0001d1c75a', 'nebulasubscriptions 5e779ebdd157bc0001d1c75a'],
         },
-        {
-            'url': 'https://watchnebula.com/videos/money-episode-1-the-draw',
-            'only_matching': True,
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://watchnebula.com/videos/money-episode-1-the-draw',
+        'only_matching': True,
+    }, {
+        'url': 'https://nebula.tv/videos/tldrnewseu-did-the-us-really-blow-up-the-nordstream-pipelines',
+        'info_dict': {
+            'id': 'e389af9d-1dab-44f2-8788-ee24deb7ff0d',
+            'ext': 'mp4',
+            'display_id': 'tldrnewseu-did-the-us-really-blow-up-the-nordstream-pipelines',
+            'title': 'Did the US Really Blow Up the NordStream Pipelines?',
+            'description': 'md5:b4e2a14e3ff08f546a3209c75261e789',
+            'upload_date': '20230223',
+            'timestamp': 1677144070,
+            'channel': 'TLDR News EU',
+            'channel_id': 'tldrnewseu',
+            'uploader': 'TLDR News EU',
+            'uploader_id': 'tldrnewseu',
+            'uploader_url': r're:https://nebula\.(tv|app)/tldrnewseu',
+            'duration': 524,
+            'channel_url': r're:https://nebula\.(tv|app)/tldrnewseu',
+            'series': 'TLDR News EU',
+            'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+',
+            'creator': 'TLDR News EU',
+            '_old_archive_ids': ['nebula 63f64c74366fcd00017c1513', 'nebulasubscriptions 63f64c74366fcd00017c1513'],
         },
-        {
-            'url': 'https://beta.nebula.tv/videos/money-episode-1-the-draw',
-            'only_matching': True,
-        },
-    ]
-
-    def _fetch_video_metadata(self, slug):
-        return self._call_nebula_api(f'https://content.api.nebula.app/video/{slug}/',
-                                     video_id=slug,
-                                     auth_type='bearer',
-                                     note='Fetching video meta data')
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://beta.nebula.tv/videos/money-episode-1-the-draw',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         slug = self._match_id(url)
-        video = self._fetch_video_metadata(slug)
-        return self._build_video_info(video)
+        url, smuggled_data = unsmuggle_url(url, {})
+        if smuggled_data.get('id'):
+            return {
+                'id': smuggled_data['id'],
+                'display_id': slug,
+                'title': '',
+                **self._extract_formats(smuggled_data['id'], slug),
+            }
+
+        metadata = self._call_api(
+            f'https://content.api.nebula.app/content/videos/{slug}',
+            slug, note='Fetching video metadata')
+        return {
+            **self._extract_video_metadata(metadata),
+            **self._extract_formats(metadata['id'], slug),
+        }
+
+
+class NebulaClassIE(NebulaBaseIE):
+    IE_NAME = 'nebula:class'
+    _VALID_URL = rf'{_BASE_URL_RE}/(?P<id>[-\w]+)/(?P<ep>\d+)'
+    _TESTS = [{
+        'url': 'https://nebula.tv/copyright-for-fun-and-profit/14',
+        'info_dict': {
+            'id': 'd7432cdc-c608-474d-942c-f74345daed7b',
+            'ext': 'mp4',
+            'display_id': '14',
+            'channel_url': 'https://nebula.tv/copyright-for-fun-and-profit',
+            'episode_number': 14,
+            'thumbnail': 'https://dj423fildxgac.cloudfront.net/d533718d-9307-42d4-8fb0-e283285e99c9',
+            'uploader_url': 'https://nebula.tv/copyright-for-fun-and-profit',
+            'duration': 646,
+            'episode': 'Episode 14',
+            'title': 'Photos, Sculpture, and Video',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        slug, episode = self._match_valid_url(url).group('id', 'ep')
+        url, smuggled_data = unsmuggle_url(url, {})
+        if smuggled_data.get('id'):
+            return {
+                'id': smuggled_data['id'],
+                'display_id': slug,
+                'title': '',
+                **self._extract_formats(smuggled_data['id'], slug),
+            }
+
+        metadata = self._call_api(
+            f'https://content.api.nebula.app/content/{slug}/{episode}/?include=lessons',
+            slug, note='Fetching video metadata')
+        return {
+            **self._extract_video_metadata(metadata),
+            **self._extract_formats(metadata['id'], slug),
+        }
 
 
 class NebulaSubscriptionsIE(NebulaBaseIE):
     IE_NAME = 'nebula:subscriptions'
-    _VALID_URL = rf'{_BASE_URL_RE}/myshows'
-    _TESTS = [
-        {
-            'url': 'https://nebula.tv/myshows',
-            'playlist_mincount': 1,
-            'info_dict': {
-                'id': 'myshows',
-            },
+    _VALID_URL = rf'{_BASE_URL_RE}/(?P<id>myshows|library/latest-videos)'
+    _TESTS = [{
+        'url': 'https://nebula.tv/myshows',
+        'playlist_mincount': 1,
+        'info_dict': {
+            'id': 'myshows',
         },
-    ]
+    }]
 
     def _generate_playlist_entries(self):
-        next_url = 'https://content.watchnebula.com/library/video/?page_size=100'
-        page_num = 1
-        while next_url:
-            channel = self._call_nebula_api(next_url, 'myshows', auth_type='bearer',
-                                            note=f'Retrieving subscriptions page {page_num}')
+        next_url = update_url_query('https://content.api.nebula.app/video_episodes/', {
+            'following': 'true',
+            'include': 'engagement',
+            'ordering': '-published_at',
+        })
+        for page_num in itertools.count(1):
+            channel = self._call_api(
+                next_url, 'myshows', note=f'Retrieving subscriptions page {page_num}')
             for episode in channel['results']:
-                yield self._build_video_info(episode)
-            next_url = channel['next']
-            page_num += 1
+                metadata = self._extract_video_metadata(episode)
+                yield self.url_result(smuggle_url(
+                    f'https://nebula.tv/videos/{metadata["display_id"]}',
+                    {'id': episode['id']}), NebulaIE, url_transparent=True, **metadata)
+            next_url = channel.get('next')
+            if not next_url:
+                return
 
     def _real_extract(self, url):
         return self.playlist_result(self._generate_playlist_entries(), 'myshows')
@@ -234,48 +310,74 @@ def _real_extract(self, url):
 
 class NebulaChannelIE(NebulaBaseIE):
     IE_NAME = 'nebula:channel'
-    _VALID_URL = rf'{_BASE_URL_RE}/(?!myshows|videos/)(?P<id>[-\w]+)'
-    _TESTS = [
-        {
-            'url': 'https://nebula.tv/tom-scott-presents-money',
-            'info_dict': {
-                'id': 'tom-scott-presents-money',
-                'title': 'Tom Scott Presents: Money',
-                'description': 'Tom Scott hosts a series all about trust, negotiation and money.',
-            },
-            'playlist_count': 5,
-        }, {
-            'url': 'https://nebula.tv/lindsayellis',
-            'info_dict': {
-                'id': 'lindsayellis',
-                'title': 'Lindsay Ellis',
-                'description': 'Enjoy these hottest of takes on Disney, Transformers, and Musicals.',
-            },
-            'playlist_mincount': 2,
+    _VALID_URL = rf'{_BASE_URL_RE}/(?!myshows|library|videos/)(?P<id>[-\w]+)/?(?:$|[?#])'
+    _TESTS = [{
+        'url': 'https://nebula.tv/tom-scott-presents-money',
+        'info_dict': {
+            'id': 'tom-scott-presents-money',
+            'title': 'Tom Scott Presents: Money',
+            'description': 'Tom Scott hosts a series all about trust, negotiation and money.',
         },
-    ]
+        'playlist_count': 5,
+    }, {
+        'url': 'https://nebula.tv/lindsayellis',
+        'info_dict': {
+            'id': 'lindsayellis',
+            'title': 'Lindsay Ellis',
+            'description': 'Enjoy these hottest of takes on Disney, Transformers, and Musicals.',
+        },
+        'playlist_mincount': 2,
+    }, {
+        'url': 'https://nebula.tv/johnnyharris',
+        'info_dict': {
+            'id': 'johnnyharris',
+            'title': 'Johnny Harris',
+            'description': 'I make videos about maps and many other things.',
+        },
+        'playlist_mincount': 90,
+    }, {
+        'url': 'https://nebula.tv/copyright-for-fun-and-profit',
+        'info_dict': {
+            'id': 'copyright-for-fun-and-profit',
+            'title': 'Copyright for Fun and Profit',
+            'description': 'md5:6690248223eed044a9f11cd5a24f9742',
+        },
+        'playlist_count': 23,
+    }]
 
-    def _generate_playlist_entries(self, collection_id, channel):
-        episodes = channel['episodes']['results']
-        for page_num in itertools.count(2):
-            for episode in episodes:
-                yield self._build_video_info(episode)
-            next_url = channel['episodes']['next']
+    def _generate_playlist_entries(self, collection_id, collection_slug):
+        next_url = f'https://content.api.nebula.app/video_channels/{collection_id}/video_episodes/?ordering=-published_at'
+        for page_num in itertools.count(1):
+            episodes = self._call_api(next_url, collection_slug, note=f'Retrieving channel page {page_num}')
+            for episode in episodes['results']:
+                metadata = self._extract_video_metadata(episode)
+                yield self.url_result(smuggle_url(
+                    episode.get('share_url') or f'https://nebula.tv/videos/{metadata["display_id"]}',
+                    {'id': episode['id']}), NebulaIE, url_transparent=True, **metadata)
+            next_url = episodes.get('next')
             if not next_url:
                 break
-            channel = self._call_nebula_api(next_url, collection_id, auth_type='bearer',
-                                            note=f'Retrieving channel page {page_num}')
-            episodes = channel['episodes']['results']
+
+    def _generate_class_entries(self, channel):
+        for lesson in channel['lessons']:
+            metadata = self._extract_video_metadata(lesson)
+            yield self.url_result(smuggle_url(
+                lesson.get('share_url') or f'https://nebula.tv/{metadata["class_slug"]}/{metadata["slug"]}',
+                {'id': lesson['id']}), NebulaClassIE, url_transparent=True, **metadata)
 
     def _real_extract(self, url):
-        collection_id = self._match_id(url)
-        channel_url = f'https://content.watchnebula.com/video/channels/{collection_id}/'
-        channel = self._call_nebula_api(channel_url, collection_id, auth_type='bearer', note='Retrieving channel')
-        channel_details = channel['details']
+        collection_slug = self._match_id(url)
+        channel = self._call_api(
+            f'https://content.api.nebula.app/content/{collection_slug}/?include=lessons',
+            collection_slug, note='Retrieving channel')
+
+        if channel.get('type') == 'class':
+            entries = self._generate_class_entries(channel)
+        else:
+            entries = self._generate_playlist_entries(channel['id'], collection_slug)
 
         return self.playlist_result(
-            entries=self._generate_playlist_entries(collection_id, channel),
-            playlist_id=collection_id,
-            playlist_title=channel_details['title'],
-            playlist_description=channel_details['description']
-        )
+            entries=entries,
+            playlist_id=collection_slug,
+            playlist_title=channel.get('title'),
+            playlist_description=channel.get('description'))

From ccfd70f4c24b579c72123ca76ab50164f8f122b7 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Mon, 20 Nov 2023 08:04:04 +0000
Subject: [PATCH 708/871] [rh:websockets] Migrate websockets to networking
 framework (#7720)

* Adds a basic WebSocket framework
* Introduces new minimum `websockets` version of 12.0
* Deprecates `WebSocketsWrapper`

Fixes https://github.com/yt-dlp/yt-dlp/issues/8439

Authored by: coletdjnz
---
 requirements.txt                 |   1 +
 test/conftest.py                 |   5 +
 test/test_networking.py          |  79 +++++--
 test/test_socks.py               |  62 +++--
 test/test_websockets.py          | 380 +++++++++++++++++++++++++++++++
 yt_dlp/YoutubeDL.py              |  10 +
 yt_dlp/downloader/niconico.py    |  10 +-
 yt_dlp/extractor/fc2.py          |  12 +-
 yt_dlp/extractor/niconico.py     |  16 +-
 yt_dlp/networking/__init__.py    |   8 +
 yt_dlp/networking/_websockets.py | 159 +++++++++++++
 yt_dlp/networking/websocket.py   |  23 ++
 yt_dlp/utils/_legacy.py          |  73 ++++++
 yt_dlp/utils/_utils.py           |  75 +-----
 14 files changed, 766 insertions(+), 147 deletions(-)
 create mode 100644 test/test_websockets.py
 create mode 100644 yt_dlp/networking/_websockets.py
 create mode 100644 yt_dlp/networking/websocket.py

diff --git a/requirements.txt b/requirements.txt
index 5b6270a7d..d983fa03f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ brotlicffi; implementation_name!='cpython'
 certifi
 requests>=2.31.0,<3
 urllib3>=1.26.17,<3
+websockets>=12.0
diff --git a/test/conftest.py b/test/conftest.py
index 15549d30b..2fbc269e1 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -19,3 +19,8 @@ def handler(request):
         pytest.skip(f'{RH_KEY} request handler is not available')
 
     return functools.partial(handler, logger=FakeLogger)
+
+
+def validate_and_send(rh, req):
+    rh.validate(req)
+    return rh.send(req)
diff --git a/test/test_networking.py b/test/test_networking.py
index 4466fc048..64af6e459 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -52,6 +52,8 @@
 from yt_dlp.utils._utils import _YDLLogger as FakeLogger
 from yt_dlp.utils.networking import HTTPHeaderDict
 
+from test.conftest import validate_and_send
+
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
@@ -275,11 +277,6 @@ def send_header(self, keyword, value):
         self._headers_buffer.append(f'{keyword}: {value}\r\n'.encode())
 
 
-def validate_and_send(rh, req):
-    rh.validate(req)
-    return rh.send(req)
-
-
 class TestRequestHandlerBase:
     @classmethod
     def setup_class(cls):
@@ -872,8 +869,9 @@ def request(self, *args, **kwargs):
     ])
     @pytest.mark.parametrize('handler', ['Requests'], indirect=True)
     def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
-        from urllib3.response import HTTPResponse as Urllib3Response
         from requests.models import Response as RequestsResponse
+        from urllib3.response import HTTPResponse as Urllib3Response
+
         from yt_dlp.networking._requests import RequestsResponseAdapter
         requests_res = RequestsResponse()
         requests_res.raw = Urllib3Response(body=b'', status=200)
@@ -929,13 +927,17 @@ class HTTPSupportedRH(ValidationRH):
             ('http', False, {}),
             ('https', False, {}),
         ]),
+        ('Websockets', [
+            ('ws', False, {}),
+            ('wss', False, {}),
+        ]),
         (NoCheckRH, [('http', False, {})]),
         (ValidationRH, [('http', UnsupportedRequest, {})])
     ]
 
     PROXY_SCHEME_TESTS = [
         # scheme, expected to fail
-        ('Urllib', [
+        ('Urllib', 'http', [
             ('http', False),
             ('https', UnsupportedRequest),
             ('socks4', False),
@@ -944,7 +946,7 @@ class HTTPSupportedRH(ValidationRH):
             ('socks5h', False),
             ('socks', UnsupportedRequest),
         ]),
-        ('Requests', [
+        ('Requests', 'http', [
             ('http', False),
             ('https', False),
             ('socks4', False),
@@ -952,8 +954,11 @@ class HTTPSupportedRH(ValidationRH):
             ('socks5', False),
             ('socks5h', False),
         ]),
-        (NoCheckRH, [('http', False)]),
-        (HTTPSupportedRH, [('http', UnsupportedRequest)]),
+        (NoCheckRH, 'http', [('http', False)]),
+        (HTTPSupportedRH, 'http', [('http', UnsupportedRequest)]),
+        ('Websockets', 'ws', [('http', UnsupportedRequest)]),
+        (NoCheckRH, 'http', [('http', False)]),
+        (HTTPSupportedRH, 'http', [('http', UnsupportedRequest)]),
     ]
 
     PROXY_KEY_TESTS = [
@@ -972,7 +977,7 @@ class HTTPSupportedRH(ValidationRH):
     ]
 
     EXTENSION_TESTS = [
-        ('Urllib', [
+        ('Urllib', 'http', [
             ({'cookiejar': 'notacookiejar'}, AssertionError),
             ({'cookiejar': YoutubeDLCookieJar()}, False),
             ({'cookiejar': CookieJar()}, AssertionError),
@@ -980,17 +985,21 @@ class HTTPSupportedRH(ValidationRH):
             ({'timeout': 'notatimeout'}, AssertionError),
             ({'unsupported': 'value'}, UnsupportedRequest),
         ]),
-        ('Requests', [
+        ('Requests', 'http', [
             ({'cookiejar': 'notacookiejar'}, AssertionError),
             ({'cookiejar': YoutubeDLCookieJar()}, False),
             ({'timeout': 1}, False),
             ({'timeout': 'notatimeout'}, AssertionError),
             ({'unsupported': 'value'}, UnsupportedRequest),
         ]),
-        (NoCheckRH, [
+        (NoCheckRH, 'http', [
             ({'cookiejar': 'notacookiejar'}, False),
             ({'somerandom': 'test'}, False),  # but any extension is allowed through
         ]),
+        ('Websockets', 'ws', [
+            ({'cookiejar': YoutubeDLCookieJar()}, False),
+            ({'timeout': 2}, False),
+        ]),
     ]
 
     @pytest.mark.parametrize('handler,scheme,fail,handler_kwargs', [
@@ -1016,14 +1025,14 @@ def test_proxy_key(self, handler, proxy_key, fail):
         run_validation(handler, fail, Request('http://', proxies={proxy_key: 'http://example.com'}))
         run_validation(handler, fail, Request('http://'), proxies={proxy_key: 'http://example.com'})
 
-    @pytest.mark.parametrize('handler,scheme,fail', [
-        (handler_tests[0], scheme, fail)
+    @pytest.mark.parametrize('handler,req_scheme,scheme,fail', [
+        (handler_tests[0], handler_tests[1], scheme, fail)
         for handler_tests in PROXY_SCHEME_TESTS
-        for scheme, fail in handler_tests[1]
+        for scheme, fail in handler_tests[2]
     ], indirect=['handler'])
-    def test_proxy_scheme(self, handler, scheme, fail):
-        run_validation(handler, fail, Request('http://', proxies={'http': f'{scheme}://example.com'}))
-        run_validation(handler, fail, Request('http://'), proxies={'http': f'{scheme}://example.com'})
+    def test_proxy_scheme(self, handler, req_scheme, scheme, fail):
+        run_validation(handler, fail, Request(f'{req_scheme}://', proxies={req_scheme: f'{scheme}://example.com'}))
+        run_validation(handler, fail, Request(f'{req_scheme}://'), proxies={req_scheme: f'{scheme}://example.com'})
 
     @pytest.mark.parametrize('handler', ['Urllib', HTTPSupportedRH, 'Requests'], indirect=True)
     def test_empty_proxy(self, handler):
@@ -1035,14 +1044,14 @@ def test_empty_proxy(self, handler):
     def test_invalid_proxy_url(self, handler, proxy_url):
         run_validation(handler, UnsupportedRequest, Request('http://', proxies={'http': proxy_url}))
 
-    @pytest.mark.parametrize('handler,extensions,fail', [
-        (handler_tests[0], extensions, fail)
+    @pytest.mark.parametrize('handler,scheme,extensions,fail', [
+        (handler_tests[0], handler_tests[1], extensions, fail)
         for handler_tests in EXTENSION_TESTS
-        for extensions, fail in handler_tests[1]
+        for extensions, fail in handler_tests[2]
     ], indirect=['handler'])
-    def test_extension(self, handler, extensions, fail):
+    def test_extension(self, handler, scheme, extensions, fail):
         run_validation(
-            handler, fail, Request('http://', extensions=extensions))
+            handler, fail, Request(f'{scheme}://', extensions=extensions))
 
     def test_invalid_request_type(self):
         rh = self.ValidationRH(logger=FakeLogger())
@@ -1075,6 +1084,22 @@ def __init__(self, *args, **kwargs):
         self._request_director = self.build_request_director([FakeRH])
 
 
+class AllUnsupportedRHYDL(FakeYDL):
+
+    def __init__(self, *args, **kwargs):
+
+        class UnsupportedRH(RequestHandler):
+            def _send(self, request: Request):
+                pass
+
+            _SUPPORTED_FEATURES = ()
+            _SUPPORTED_PROXY_SCHEMES = ()
+            _SUPPORTED_URL_SCHEMES = ()
+
+        super().__init__(*args, **kwargs)
+        self._request_director = self.build_request_director([UnsupportedRH])
+
+
 class TestRequestDirector:
 
     def test_handler_operations(self):
@@ -1234,6 +1259,12 @@ def test_file_urls_error(self):
             with pytest.raises(RequestError, match=r'file:// URLs are disabled by default'):
                 ydl.urlopen('file://')
 
+    @pytest.mark.parametrize('scheme', (['ws', 'wss']))
+    def test_websocket_unavailable_error(self, scheme):
+        with AllUnsupportedRHYDL() as ydl:
+            with pytest.raises(RequestError, match=r'This request requires WebSocket support'):
+                ydl.urlopen(f'{scheme}://')
+
     def test_legacy_server_connect_error(self):
         with FakeRHYDL() as ydl:
             for error in ('UNSAFE_LEGACY_RENEGOTIATION_DISABLED', 'SSLV3_ALERT_HANDSHAKE_FAILURE'):
diff --git a/test/test_socks.py b/test/test_socks.py
index d8ac88dad..71f783e13 100644
--- a/test/test_socks.py
+++ b/test/test_socks.py
@@ -210,6 +210,16 @@ def do_GET(self):
             self.wfile.write(payload.encode())
 
 
+class SocksWebSocketTestRequestHandler(SocksTestRequestHandler):
+    def handle(self):
+        import websockets.sync.server
+        protocol = websockets.ServerProtocol()
+        connection = websockets.sync.server.ServerConnection(socket=self.request, protocol=protocol, close_timeout=0)
+        connection.handshake()
+        connection.send(json.dumps(self.socks_info))
+        connection.close()
+
+
 @contextlib.contextmanager
 def socks_server(socks_server_class, request_handler, bind_ip=None, **socks_server_kwargs):
     server = server_thread = None
@@ -252,8 +262,22 @@ def socks_info_request(self, handler, target_domain=None, target_port=None, **re
         return json.loads(handler.send(request).read().decode())
 
 
+class WebSocketSocksTestProxyContext(SocksProxyTestContext):
+    REQUEST_HANDLER_CLASS = SocksWebSocketTestRequestHandler
+
+    def socks_info_request(self, handler, target_domain=None, target_port=None, **req_kwargs):
+        request = Request(f'ws://{target_domain or "127.0.0.1"}:{target_port or "40000"}', **req_kwargs)
+        handler.validate(request)
+        ws = handler.send(request)
+        ws.send('socks_info')
+        socks_info = ws.recv()
+        ws.close()
+        return json.loads(socks_info)
+
+
 CTX_MAP = {
     'http': HTTPSocksTestProxyContext,
+    'ws': WebSocketSocksTestProxyContext,
 }
 
 
@@ -263,7 +287,7 @@ def ctx(request):
 
 
 class TestSocks4Proxy:
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks4_no_auth(self, handler, ctx):
         with handler() as rh:
             with ctx.socks_server(Socks4ProxyHandler) as server_address:
@@ -271,7 +295,7 @@ def test_socks4_no_auth(self, handler, ctx):
                     rh, proxies={'all': f'socks4://{server_address}'})
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks4_auth(self, handler, ctx):
         with handler() as rh:
             with ctx.socks_server(Socks4ProxyHandler, user_id='user') as server_address:
@@ -281,7 +305,7 @@ def test_socks4_auth(self, handler, ctx):
                     rh, proxies={'all': f'socks4://user:@{server_address}'})
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks4a_ipv4_target(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
@@ -289,7 +313,7 @@ def test_socks4a_ipv4_target(self, handler, ctx):
                 assert response['version'] == 4
                 assert (response['ipv4_address'] == '127.0.0.1') != (response['domain_address'] == '127.0.0.1')
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks4a_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks4a://{server_address}'}) as rh:
@@ -298,7 +322,7 @@ def test_socks4a_domain_target(self, handler, ctx):
                 assert response['ipv4_address'] is None
                 assert response['domain_address'] == 'localhost'
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
@@ -308,7 +332,7 @@ def test_ipv4_client_source_address(self, handler, ctx):
                 assert response['client_address'][0] == source_address
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     @pytest.mark.parametrize('reply_code', [
         Socks4CD.REQUEST_REJECTED_OR_FAILED,
         Socks4CD.REQUEST_REJECTED_CANNOT_CONNECT_TO_IDENTD,
@@ -320,7 +344,7 @@ def test_socks4_errors(self, handler, ctx, reply_code):
                 with pytest.raises(ProxyError):
                     ctx.socks_info_request(rh)
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_ipv6_socks4_proxy(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler, bind_ip='::1') as server_address:
             with handler(proxies={'all': f'socks4://{server_address}'}) as rh:
@@ -329,7 +353,7 @@ def test_ipv6_socks4_proxy(self, handler, ctx):
                 assert response['ipv4_address'] == '127.0.0.1'
                 assert response['version'] == 4
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_timeout(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler, sleep=2) as server_address:
             with handler(proxies={'all': f'socks4://{server_address}'}, timeout=0.5) as rh:
@@ -339,7 +363,7 @@ def test_timeout(self, handler, ctx):
 
 class TestSocks5Proxy:
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks5_no_auth(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -347,7 +371,7 @@ def test_socks5_no_auth(self, handler, ctx):
                 assert response['auth_methods'] == [0x0]
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks5_user_pass(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler, auth=('test', 'testpass')) as server_address:
             with handler() as rh:
@@ -360,7 +384,7 @@ def test_socks5_user_pass(self, handler, ctx):
                 assert response['auth_methods'] == [Socks5Auth.AUTH_NONE, Socks5Auth.AUTH_USER_PASS]
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks5_ipv4_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -368,7 +392,7 @@ def test_socks5_ipv4_target(self, handler, ctx):
                 assert response['ipv4_address'] == '127.0.0.1'
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks5_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -376,7 +400,7 @@ def test_socks5_domain_target(self, handler, ctx):
                 assert (response['ipv4_address'] == '127.0.0.1') != (response['ipv6_address'] == '::1')
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks5h_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
@@ -385,7 +409,7 @@ def test_socks5h_domain_target(self, handler, ctx):
                 assert response['domain_address'] == 'localhost'
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks5h_ip_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5h://{server_address}'}) as rh:
@@ -394,7 +418,7 @@ def test_socks5h_ip_target(self, handler, ctx):
                 assert response['domain_address'] is None
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_socks5_ipv6_destination(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -402,7 +426,7 @@ def test_socks5_ipv6_destination(self, handler, ctx):
                 assert response['ipv6_address'] == '::1'
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_ipv6_socks5_proxy(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler, bind_ip='::1') as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -413,7 +437,7 @@ def test_ipv6_socks5_proxy(self, handler, ctx):
 
     # XXX: is there any feasible way of testing IPv6 source addresses?
     # Same would go for non-proxy source_address test...
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
@@ -422,7 +446,7 @@ def test_ipv4_client_source_address(self, handler, ctx):
                 assert response['client_address'][0] == source_address
                 assert response['version'] == 5
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws')], indirect=True)
     @pytest.mark.parametrize('reply_code', [
         Socks5Reply.GENERAL_FAILURE,
         Socks5Reply.CONNECTION_NOT_ALLOWED,
@@ -439,7 +463,7 @@ def test_socks5_errors(self, handler, ctx, reply_code):
                 with pytest.raises(ProxyError):
                     ctx.socks_info_request(rh)
 
-    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http')], indirect=True)
+    @pytest.mark.parametrize('handler,ctx', [('Urllib', 'http'), ('Websockets', 'ws')], indirect=True)
     def test_timeout(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler, sleep=2) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}, timeout=1) as rh:
diff --git a/test/test_websockets.py b/test/test_websockets.py
new file mode 100644
index 000000000..39d3c7d72
--- /dev/null
+++ b/test/test_websockets.py
@@ -0,0 +1,380 @@
+#!/usr/bin/env python3
+
+# Allow direct execution
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import http.client
+import http.cookiejar
+import http.server
+import json
+import random
+import ssl
+import threading
+
+from yt_dlp import socks
+from yt_dlp.cookies import YoutubeDLCookieJar
+from yt_dlp.dependencies import websockets
+from yt_dlp.networking import Request
+from yt_dlp.networking.exceptions import (
+    CertificateVerifyError,
+    HTTPError,
+    ProxyError,
+    RequestError,
+    SSLError,
+    TransportError,
+)
+from yt_dlp.utils.networking import HTTPHeaderDict
+
+from test.conftest import validate_and_send
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def websocket_handler(websocket):
+    for message in websocket:
+        if isinstance(message, bytes):
+            if message == b'bytes':
+                return websocket.send('2')
+        elif isinstance(message, str):
+            if message == 'headers':
+                return websocket.send(json.dumps(dict(websocket.request.headers)))
+            elif message == 'path':
+                return websocket.send(websocket.request.path)
+            elif message == 'source_address':
+                return websocket.send(websocket.remote_address[0])
+            elif message == 'str':
+                return websocket.send('1')
+        return websocket.send(message)
+
+
+def process_request(self, request):
+    if request.path.startswith('/gen_'):
+        status = http.HTTPStatus(int(request.path[5:]))
+        if 300 <= status.value <= 300:
+            return websockets.http11.Response(
+                status.value, status.phrase, websockets.datastructures.Headers([('Location', '/')]), b'')
+        return self.protocol.reject(status.value, status.phrase)
+    return self.protocol.accept(request)
+
+
+def create_websocket_server(**ws_kwargs):
+    import websockets.sync.server
+    wsd = websockets.sync.server.serve(websocket_handler, '127.0.0.1', 0, process_request=process_request, **ws_kwargs)
+    ws_port = wsd.socket.getsockname()[1]
+    ws_server_thread = threading.Thread(target=wsd.serve_forever)
+    ws_server_thread.daemon = True
+    ws_server_thread.start()
+    return ws_server_thread, ws_port
+
+
+def create_ws_websocket_server():
+    return create_websocket_server()
+
+
+def create_wss_websocket_server():
+    certfn = os.path.join(TEST_DIR, 'testcert.pem')
+    sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    sslctx.load_cert_chain(certfn, None)
+    return create_websocket_server(ssl_context=sslctx)
+
+
+MTLS_CERT_DIR = os.path.join(TEST_DIR, 'testdata', 'certificate')
+
+
+def create_mtls_wss_websocket_server():
+    certfn = os.path.join(TEST_DIR, 'testcert.pem')
+    cacertfn = os.path.join(MTLS_CERT_DIR, 'ca.crt')
+
+    sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    sslctx.verify_mode = ssl.CERT_REQUIRED
+    sslctx.load_verify_locations(cafile=cacertfn)
+    sslctx.load_cert_chain(certfn, None)
+
+    return create_websocket_server(ssl_context=sslctx)
+
+
+@pytest.mark.skipif(not websockets, reason='websockets must be installed to test websocket request handlers')
+class TestWebsSocketRequestHandlerConformance:
+    @classmethod
+    def setup_class(cls):
+        cls.ws_thread, cls.ws_port = create_ws_websocket_server()
+        cls.ws_base_url = f'ws://127.0.0.1:{cls.ws_port}'
+
+        cls.wss_thread, cls.wss_port = create_wss_websocket_server()
+        cls.wss_base_url = f'wss://127.0.0.1:{cls.wss_port}'
+
+        cls.bad_wss_thread, cls.bad_wss_port = create_websocket_server(ssl_context=ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER))
+        cls.bad_wss_host = f'wss://127.0.0.1:{cls.bad_wss_port}'
+
+        cls.mtls_wss_thread, cls.mtls_wss_port = create_mtls_wss_websocket_server()
+        cls.mtls_wss_base_url = f'wss://127.0.0.1:{cls.mtls_wss_port}'
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_basic_websockets(self, handler):
+        with handler() as rh:
+            ws = validate_and_send(rh, Request(self.ws_base_url))
+            assert 'upgrade' in ws.headers
+            assert ws.status == 101
+            ws.send('foo')
+            assert ws.recv() == 'foo'
+            ws.close()
+
+    # https://www.rfc-editor.org/rfc/rfc6455.html#section-5.6
+    @pytest.mark.parametrize('msg,opcode', [('str', 1), (b'bytes', 2)])
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_send_types(self, handler, msg, opcode):
+        with handler() as rh:
+            ws = validate_and_send(rh, Request(self.ws_base_url))
+            ws.send(msg)
+            assert int(ws.recv()) == opcode
+            ws.close()
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_verify_cert(self, handler):
+        with handler() as rh:
+            with pytest.raises(CertificateVerifyError):
+                validate_and_send(rh, Request(self.wss_base_url))
+
+        with handler(verify=False) as rh:
+            ws = validate_and_send(rh, Request(self.wss_base_url))
+            assert ws.status == 101
+            ws.close()
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_ssl_error(self, handler):
+        with handler(verify=False) as rh:
+            with pytest.raises(SSLError, match='sslv3 alert handshake failure') as exc_info:
+                validate_and_send(rh, Request(self.bad_wss_host))
+            assert not issubclass(exc_info.type, CertificateVerifyError)
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    @pytest.mark.parametrize('path,expected', [
+        # Unicode characters should be encoded with uppercase percent-encoding
+        ('/中文', '/%E4%B8%AD%E6%96%87'),
+        # don't normalize existing percent encodings
+        ('/%c7%9f', '/%c7%9f'),
+    ])
+    def test_percent_encode(self, handler, path, expected):
+        with handler() as rh:
+            ws = validate_and_send(rh, Request(f'{self.ws_base_url}{path}'))
+            ws.send('path')
+            assert ws.recv() == expected
+            assert ws.status == 101
+            ws.close()
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_remove_dot_segments(self, handler):
+        with handler() as rh:
+            # This isn't a comprehensive test,
+            # but it should be enough to check whether the handler is removing dot segments
+            ws = validate_and_send(rh, Request(f'{self.ws_base_url}/a/b/./../../test'))
+            assert ws.status == 101
+            ws.send('path')
+            assert ws.recv() == '/test'
+            ws.close()
+
+    # We are restricted to known HTTP status codes in http.HTTPStatus
+    # Redirects are not supported for websockets
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    @pytest.mark.parametrize('status', (200, 204, 301, 302, 303, 400, 500, 511))
+    def test_raise_http_error(self, handler, status):
+        with handler() as rh:
+            with pytest.raises(HTTPError) as exc_info:
+                validate_and_send(rh, Request(f'{self.ws_base_url}/gen_{status}'))
+            assert exc_info.value.status == status
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    @pytest.mark.parametrize('params,extensions', [
+        ({'timeout': 0.00001}, {}),
+        ({}, {'timeout': 0.00001}),
+    ])
+    def test_timeout(self, handler, params, extensions):
+        with handler(**params) as rh:
+            with pytest.raises(TransportError):
+                validate_and_send(rh, Request(self.ws_base_url, extensions=extensions))
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_cookies(self, handler):
+        cookiejar = YoutubeDLCookieJar()
+        cookiejar.set_cookie(http.cookiejar.Cookie(
+            version=0, name='test', value='ytdlp', port=None, port_specified=False,
+            domain='127.0.0.1', domain_specified=True, domain_initial_dot=False, path='/',
+            path_specified=True, secure=False, expires=None, discard=False, comment=None,
+            comment_url=None, rest={}))
+
+        with handler(cookiejar=cookiejar) as rh:
+            ws = validate_and_send(rh, Request(self.ws_base_url))
+            ws.send('headers')
+            assert json.loads(ws.recv())['cookie'] == 'test=ytdlp'
+            ws.close()
+
+        with handler() as rh:
+            ws = validate_and_send(rh, Request(self.ws_base_url))
+            ws.send('headers')
+            assert 'cookie' not in json.loads(ws.recv())
+            ws.close()
+
+            ws = validate_and_send(rh, Request(self.ws_base_url, extensions={'cookiejar': cookiejar}))
+            ws.send('headers')
+            assert json.loads(ws.recv())['cookie'] == 'test=ytdlp'
+            ws.close()
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_source_address(self, handler):
+        source_address = f'127.0.0.{random.randint(5, 255)}'
+        with handler(source_address=source_address) as rh:
+            ws = validate_and_send(rh, Request(self.ws_base_url))
+            ws.send('source_address')
+            assert source_address == ws.recv()
+            ws.close()
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_response_url(self, handler):
+        with handler() as rh:
+            url = f'{self.ws_base_url}/something'
+            ws = validate_and_send(rh, Request(url))
+            assert ws.url == url
+            ws.close()
+
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_request_headers(self, handler):
+        with handler(headers=HTTPHeaderDict({'test1': 'test', 'test2': 'test2'})) as rh:
+            # Global Headers
+            ws = validate_and_send(rh, Request(self.ws_base_url))
+            ws.send('headers')
+            headers = HTTPHeaderDict(json.loads(ws.recv()))
+            assert headers['test1'] == 'test'
+            ws.close()
+
+            # Per request headers, merged with global
+            ws = validate_and_send(rh, Request(
+                self.ws_base_url, headers={'test2': 'changed', 'test3': 'test3'}))
+            ws.send('headers')
+            headers = HTTPHeaderDict(json.loads(ws.recv()))
+            assert headers['test1'] == 'test'
+            assert headers['test2'] == 'changed'
+            assert headers['test3'] == 'test3'
+            ws.close()
+
+    @pytest.mark.parametrize('client_cert', (
+        {'client_certificate': os.path.join(MTLS_CERT_DIR, 'clientwithkey.crt')},
+        {
+            'client_certificate': os.path.join(MTLS_CERT_DIR, 'client.crt'),
+            'client_certificate_key': os.path.join(MTLS_CERT_DIR, 'client.key'),
+        },
+        {
+            'client_certificate': os.path.join(MTLS_CERT_DIR, 'clientwithencryptedkey.crt'),
+            'client_certificate_password': 'foobar',
+        },
+        {
+            'client_certificate': os.path.join(MTLS_CERT_DIR, 'client.crt'),
+            'client_certificate_key': os.path.join(MTLS_CERT_DIR, 'clientencrypted.key'),
+            'client_certificate_password': 'foobar',
+        }
+    ))
+    @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+    def test_mtls(self, handler, client_cert):
+        with handler(
+            # Disable client-side validation of unacceptable self-signed testcert.pem
+            # The test is of a check on the server side, so unaffected
+            verify=False,
+            client_cert=client_cert
+        ) as rh:
+            validate_and_send(rh, Request(self.mtls_wss_base_url)).close()
+
+
+def create_fake_ws_connection(raised):
+    import websockets.sync.client
+
+    class FakeWsConnection(websockets.sync.client.ClientConnection):
+        def __init__(self, *args, **kwargs):
+            class FakeResponse:
+                body = b''
+                headers = {}
+                status_code = 101
+                reason_phrase = 'test'
+
+            self.response = FakeResponse()
+
+        def send(self, *args, **kwargs):
+            raise raised()
+
+        def recv(self, *args, **kwargs):
+            raise raised()
+
+        def close(self, *args, **kwargs):
+            return
+
+    return FakeWsConnection()
+
+
+@pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
+class TestWebsocketsRequestHandler:
+    @pytest.mark.parametrize('raised,expected', [
+        # https://websockets.readthedocs.io/en/stable/reference/exceptions.html
+        (lambda: websockets.exceptions.InvalidURI(msg='test', uri='test://'), RequestError),
+        # Requires a response object. Should be covered by HTTP error tests.
+        # (lambda: websockets.exceptions.InvalidStatus(), TransportError),
+        (lambda: websockets.exceptions.InvalidHandshake(), TransportError),
+        # These are subclasses of InvalidHandshake
+        (lambda: websockets.exceptions.InvalidHeader(name='test'), TransportError),
+        (lambda: websockets.exceptions.NegotiationError(), TransportError),
+        # Catch-all
+        (lambda: websockets.exceptions.WebSocketException(), TransportError),
+        (lambda: TimeoutError(), TransportError),
+        # These may be raised by our create_connection implementation, which should also be caught
+        (lambda: OSError(), TransportError),
+        (lambda: ssl.SSLError(), SSLError),
+        (lambda: ssl.SSLCertVerificationError(), CertificateVerifyError),
+        (lambda: socks.ProxyError(), ProxyError),
+    ])
+    def test_request_error_mapping(self, handler, monkeypatch, raised, expected):
+        import websockets.sync.client
+
+        import yt_dlp.networking._websockets
+        with handler() as rh:
+            def fake_connect(*args, **kwargs):
+                raise raised()
+            monkeypatch.setattr(yt_dlp.networking._websockets, 'create_connection', lambda *args, **kwargs: None)
+            monkeypatch.setattr(websockets.sync.client, 'connect', fake_connect)
+            with pytest.raises(expected) as exc_info:
+                rh.send(Request('ws://fake-url'))
+            assert exc_info.type is expected
+
+    @pytest.mark.parametrize('raised,expected,match', [
+        # https://websockets.readthedocs.io/en/stable/reference/sync/client.html#websockets.sync.client.ClientConnection.send
+        (lambda: websockets.exceptions.ConnectionClosed(None, None), TransportError, None),
+        (lambda: RuntimeError(), TransportError, None),
+        (lambda: TimeoutError(), TransportError, None),
+        (lambda: TypeError(), RequestError, None),
+        (lambda: socks.ProxyError(), ProxyError, None),
+        # Catch-all
+        (lambda: websockets.exceptions.WebSocketException(), TransportError, None),
+    ])
+    def test_ws_send_error_mapping(self, handler, monkeypatch, raised, expected, match):
+        from yt_dlp.networking._websockets import WebsocketsResponseAdapter
+        ws = WebsocketsResponseAdapter(create_fake_ws_connection(raised), url='ws://fake-url')
+        with pytest.raises(expected, match=match) as exc_info:
+            ws.send('test')
+        assert exc_info.type is expected
+
+    @pytest.mark.parametrize('raised,expected,match', [
+        # https://websockets.readthedocs.io/en/stable/reference/sync/client.html#websockets.sync.client.ClientConnection.recv
+        (lambda: websockets.exceptions.ConnectionClosed(None, None), TransportError, None),
+        (lambda: RuntimeError(), TransportError, None),
+        (lambda: TimeoutError(), TransportError, None),
+        (lambda: socks.ProxyError(), ProxyError, None),
+        # Catch-all
+        (lambda: websockets.exceptions.WebSocketException(), TransportError, None),
+    ])
+    def test_ws_recv_error_mapping(self, handler, monkeypatch, raised, expected, match):
+        from yt_dlp.networking._websockets import WebsocketsResponseAdapter
+        ws = WebsocketsResponseAdapter(create_fake_ws_connection(raised), url='ws://fake-url')
+        with pytest.raises(expected, match=match) as exc_info:
+            ws.recv()
+        assert exc_info.type is expected
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 740826b45..85b282bd5 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -4052,6 +4052,7 @@ def urlopen(self, req):
             return self._request_director.send(req)
         except NoSupportingHandlers as e:
             for ue in e.unsupported_errors:
+                # FIXME: This depends on the order of errors.
                 if not (ue.handler and ue.msg):
                     continue
                 if ue.handler.RH_KEY == 'Urllib' and 'unsupported url scheme: "file"' in ue.msg.lower():
@@ -4061,6 +4062,15 @@ def urlopen(self, req):
                 if 'unsupported proxy type: "https"' in ue.msg.lower():
                     raise RequestError(
                         'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests')
+
+                elif (
+                    re.match(r'unsupported url scheme: "wss?"', ue.msg.lower())
+                    and 'websockets' not in self._request_director.handlers
+                ):
+                    raise RequestError(
+                        'This request requires WebSocket support. '
+                        'Ensure one of the following dependencies are installed: websockets',
+                        cause=ue) from ue
             raise
         except SSLError as e:
             if 'UNSAFE_LEGACY_RENEGOTIATION_DISABLED' in str(e):
diff --git a/yt_dlp/downloader/niconico.py b/yt_dlp/downloader/niconico.py
index 5720f6eb8..fef8bff73 100644
--- a/yt_dlp/downloader/niconico.py
+++ b/yt_dlp/downloader/niconico.py
@@ -6,7 +6,7 @@
 from .common import FileDownloader
 from .external import FFmpegFD
 from ..networking import Request
-from ..utils import DownloadError, WebSocketsWrapper, str_or_none, try_get
+from ..utils import DownloadError, str_or_none, try_get
 
 
 class NiconicoDmcFD(FileDownloader):
@@ -64,7 +64,6 @@ def real_download(self, filename, info_dict):
         ws_url = info_dict['url']
         ws_extractor = info_dict['ws']
         ws_origin_host = info_dict['origin']
-        cookies = info_dict.get('cookies')
         live_quality = info_dict.get('live_quality', 'high')
         live_latency = info_dict.get('live_latency', 'high')
         dl = FFmpegFD(self.ydl, self.params or {})
@@ -76,12 +75,7 @@ def real_download(self, filename, info_dict):
 
         def communicate_ws(reconnect):
             if reconnect:
-                ws = WebSocketsWrapper(ws_url, {
-                    'Cookies': str_or_none(cookies) or '',
-                    'Origin': f'https://{ws_origin_host}',
-                    'Accept': '*/*',
-                    'User-Agent': self.params['http_headers']['User-Agent'],
-                })
+                ws = self.ydl.urlopen(Request(ws_url, headers={'Origin': f'https://{ws_origin_host}'}))
                 if self.ydl.params.get('verbose', False):
                     self.to_screen('[debug] Sending startWatching request')
                 ws.send(json.dumps({
diff --git a/yt_dlp/extractor/fc2.py b/yt_dlp/extractor/fc2.py
index ba19b6cab..bbc4b5693 100644
--- a/yt_dlp/extractor/fc2.py
+++ b/yt_dlp/extractor/fc2.py
@@ -2,11 +2,9 @@
 
 from .common import InfoExtractor
 from ..compat import compat_parse_qs
-from ..dependencies import websockets
 from ..networking import Request
 from ..utils import (
     ExtractorError,
-    WebSocketsWrapper,
     js_to_json,
     traverse_obj,
     update_url_query,
@@ -167,8 +165,6 @@ class FC2LiveIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        if not websockets:
-            raise ExtractorError('websockets library is not available. Please install it.', expected=True)
         video_id = self._match_id(url)
         webpage = self._download_webpage('https://live.fc2.com/%s/' % video_id, video_id)
 
@@ -199,13 +195,9 @@ def _real_extract(self, url):
         ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']})
         playlist_data = None
 
-        self.to_screen('%s: Fetching HLS playlist info via WebSocket' % video_id)
-        ws = WebSocketsWrapper(ws_url, {
-            'Cookie': str(self._get_cookies('https://live.fc2.com/'))[12:],
+        ws = self._request_webpage(Request(ws_url, headers={
             'Origin': 'https://live.fc2.com',
-            'Accept': '*/*',
-            'User-Agent': self.get_param('http_headers')['User-Agent'],
-        })
+        }), video_id, note='Fetching HLS playlist info via WebSocket')
 
         self.write_debug('Sending HLS server request')
 
diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py
index fa2d709d2..797b5268a 100644
--- a/yt_dlp/extractor/niconico.py
+++ b/yt_dlp/extractor/niconico.py
@@ -8,12 +8,11 @@
 from urllib.parse import urlparse
 
 from .common import InfoExtractor, SearchInfoExtractor
-from ..dependencies import websockets
+from ..networking import Request
 from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
-    WebSocketsWrapper,
     bug_reports_message,
     clean_html,
     float_or_none,
@@ -934,8 +933,6 @@ class NiconicoLiveIE(InfoExtractor):
     _KNOWN_LATENCY = ('high', 'low')
 
     def _real_extract(self, url):
-        if not websockets:
-            raise ExtractorError('websockets library is not available. Please install it.', expected=True)
         video_id = self._match_id(url)
         webpage, urlh = self._download_webpage_handle(f'https://live.nicovideo.jp/watch/{video_id}', video_id)
 
@@ -950,17 +947,13 @@ def _real_extract(self, url):
         })
 
         hostname = remove_start(urlparse(urlh.url).hostname, 'sp.')
-        cookies = try_get(urlh.url, self._downloader._calc_cookies)
         latency = try_get(self._configuration_arg('latency'), lambda x: x[0])
         if latency not in self._KNOWN_LATENCY:
             latency = 'high'
 
-        ws = WebSocketsWrapper(ws_url, {
-            'Cookies': str_or_none(cookies) or '',
-            'Origin': f'https://{hostname}',
-            'Accept': '*/*',
-            'User-Agent': self.get_param('http_headers')['User-Agent'],
-        })
+        ws = self._request_webpage(
+            Request(ws_url, headers={'Origin': f'https://{hostname}'}),
+            video_id=video_id, note='Connecting to WebSocket server')
 
         self.write_debug('[debug] Sending HLS server request')
         ws.send(json.dumps({
@@ -1034,7 +1027,6 @@ def _real_extract(self, url):
                 'protocol': 'niconico_live',
                 'ws': ws,
                 'video_id': video_id,
-                'cookies': cookies,
                 'live_latency': latency,
                 'origin': hostname,
             })
diff --git a/yt_dlp/networking/__init__.py b/yt_dlp/networking/__init__.py
index aa8d0eabe..96c5a0678 100644
--- a/yt_dlp/networking/__init__.py
+++ b/yt_dlp/networking/__init__.py
@@ -21,3 +21,11 @@
     pass
 except Exception as e:
     warnings.warn(f'Failed to import "requests" request handler: {e}' + bug_reports_message())
+
+try:
+    from . import _websockets
+except ImportError:
+    pass
+except Exception as e:
+    warnings.warn(f'Failed to import "websockets" request handler: {e}' + bug_reports_message())
+
diff --git a/yt_dlp/networking/_websockets.py b/yt_dlp/networking/_websockets.py
new file mode 100644
index 000000000..ad85554e4
--- /dev/null
+++ b/yt_dlp/networking/_websockets.py
@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import io
+import logging
+import ssl
+import sys
+
+from ._helper import create_connection, select_proxy, make_socks_proxy_opts, create_socks_proxy_socket
+from .common import Response, register_rh, Features
+from .exceptions import (
+    CertificateVerifyError,
+    HTTPError,
+    RequestError,
+    SSLError,
+    TransportError, ProxyError,
+)
+from .websocket import WebSocketRequestHandler, WebSocketResponse
+from ..compat import functools
+from ..dependencies import websockets
+from ..utils import int_or_none
+from ..socks import ProxyError as SocksProxyError
+
+if not websockets:
+    raise ImportError('websockets is not installed')
+
+import websockets.version
+
+websockets_version = tuple(map(int_or_none, websockets.version.version.split('.')))
+if websockets_version < (12, 0):
+    raise ImportError('Only websockets>=12.0 is supported')
+
+import websockets.sync.client
+from websockets.uri import parse_uri
+
+
+class WebsocketsResponseAdapter(WebSocketResponse):
+
+    def __init__(self, wsw: websockets.sync.client.ClientConnection, url):
+        super().__init__(
+            fp=io.BytesIO(wsw.response.body or b''),
+            url=url,
+            headers=wsw.response.headers,
+            status=wsw.response.status_code,
+            reason=wsw.response.reason_phrase,
+        )
+        self.wsw = wsw
+
+    def close(self):
+        self.wsw.close()
+        super().close()
+
+    def send(self, message):
+        # https://websockets.readthedocs.io/en/stable/reference/sync/client.html#websockets.sync.client.ClientConnection.send
+        try:
+            return self.wsw.send(message)
+        except (websockets.exceptions.WebSocketException, RuntimeError, TimeoutError) as e:
+            raise TransportError(cause=e) from e
+        except SocksProxyError as e:
+            raise ProxyError(cause=e) from e
+        except TypeError as e:
+            raise RequestError(cause=e) from e
+
+    def recv(self):
+        # https://websockets.readthedocs.io/en/stable/reference/sync/client.html#websockets.sync.client.ClientConnection.recv
+        try:
+            return self.wsw.recv()
+        except SocksProxyError as e:
+            raise ProxyError(cause=e) from e
+        except (websockets.exceptions.WebSocketException, RuntimeError, TimeoutError) as e:
+            raise TransportError(cause=e) from e
+
+
+@register_rh
+class WebsocketsRH(WebSocketRequestHandler):
+    """
+    Websockets request handler
+    https://websockets.readthedocs.io
+    https://github.com/python-websockets/websockets
+    """
+    _SUPPORTED_URL_SCHEMES = ('wss', 'ws')
+    _SUPPORTED_PROXY_SCHEMES = ('socks4', 'socks4a', 'socks5', 'socks5h')
+    _SUPPORTED_FEATURES = (Features.ALL_PROXY, Features.NO_PROXY)
+    RH_NAME = 'websockets'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        for name in ('websockets.client', 'websockets.server'):
+            logger = logging.getLogger(name)
+            handler = logging.StreamHandler(stream=sys.stdout)
+            handler.setFormatter(logging.Formatter(f'{self.RH_NAME}: %(message)s'))
+            logger.addHandler(handler)
+            if self.verbose:
+                logger.setLevel(logging.DEBUG)
+
+    def _check_extensions(self, extensions):
+        super()._check_extensions(extensions)
+        extensions.pop('timeout', None)
+        extensions.pop('cookiejar', None)
+
+    def _send(self, request):
+        timeout = float(request.extensions.get('timeout') or self.timeout)
+        headers = self._merge_headers(request.headers)
+        if 'cookie' not in headers:
+            cookiejar = request.extensions.get('cookiejar') or self.cookiejar
+            cookie_header = cookiejar.get_cookie_header(request.url)
+            if cookie_header:
+                headers['cookie'] = cookie_header
+
+        wsuri = parse_uri(request.url)
+        create_conn_kwargs = {
+            'source_address': (self.source_address, 0) if self.source_address else None,
+            'timeout': timeout
+        }
+        proxy = select_proxy(request.url, request.proxies or self.proxies or {})
+        try:
+            if proxy:
+                socks_proxy_options = make_socks_proxy_opts(proxy)
+                sock = create_connection(
+                    address=(socks_proxy_options['addr'], socks_proxy_options['port']),
+                    _create_socket_func=functools.partial(
+                        create_socks_proxy_socket, (wsuri.host, wsuri.port), socks_proxy_options),
+                    **create_conn_kwargs
+                )
+            else:
+                sock = create_connection(
+                    address=(wsuri.host, wsuri.port),
+                    **create_conn_kwargs
+                )
+            conn = websockets.sync.client.connect(
+                sock=sock,
+                uri=request.url,
+                additional_headers=headers,
+                open_timeout=timeout,
+                user_agent_header=None,
+                ssl_context=self._make_sslcontext() if wsuri.secure else None,
+                close_timeout=0,  # not ideal, but prevents yt-dlp hanging
+            )
+            return WebsocketsResponseAdapter(conn, url=request.url)
+
+        # Exceptions as per https://websockets.readthedocs.io/en/stable/reference/sync/client.html
+        except SocksProxyError as e:
+            raise ProxyError(cause=e) from e
+        except websockets.exceptions.InvalidURI as e:
+            raise RequestError(cause=e) from e
+        except ssl.SSLCertVerificationError as e:
+            raise CertificateVerifyError(cause=e) from e
+        except ssl.SSLError as e:
+            raise SSLError(cause=e) from e
+        except websockets.exceptions.InvalidStatus as e:
+            raise HTTPError(
+                Response(
+                    fp=io.BytesIO(e.response.body),
+                    url=request.url,
+                    headers=e.response.headers,
+                    status=e.response.status_code,
+                    reason=e.response.reason_phrase),
+            ) from e
+        except (OSError, TimeoutError, websockets.exceptions.WebSocketException) as e:
+            raise TransportError(cause=e) from e
diff --git a/yt_dlp/networking/websocket.py b/yt_dlp/networking/websocket.py
new file mode 100644
index 000000000..09fcf78ac
--- /dev/null
+++ b/yt_dlp/networking/websocket.py
@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import abc
+
+from .common import Response, RequestHandler
+
+
+class WebSocketResponse(Response):
+
+    def send(self, message: bytes | str):
+        """
+        Send a message to the server.
+
+        @param message: The message to send. A string (str) is sent as a text frame, bytes is sent as a binary frame.
+        """
+        raise NotImplementedError
+
+    def recv(self):
+        raise NotImplementedError
+
+
+class WebSocketRequestHandler(RequestHandler, abc.ABC):
+    pass
diff --git a/yt_dlp/utils/_legacy.py b/yt_dlp/utils/_legacy.py
index dde02092c..aa9f46d20 100644
--- a/yt_dlp/utils/_legacy.py
+++ b/yt_dlp/utils/_legacy.py
@@ -1,4 +1,6 @@
 """No longer used and new code should not use. Exists only for API compat."""
+import asyncio
+import atexit
 import platform
 import struct
 import sys
@@ -32,6 +34,77 @@
 has_websockets = bool(websockets)
 
 
+class WebSocketsWrapper:
+    """Wraps websockets module to use in non-async scopes"""
+    pool = None
+
+    def __init__(self, url, headers=None, connect=True, **ws_kwargs):
+        self.loop = asyncio.new_event_loop()
+        # XXX: "loop" is deprecated
+        self.conn = websockets.connect(
+            url, extra_headers=headers, ping_interval=None,
+            close_timeout=float('inf'), loop=self.loop, ping_timeout=float('inf'), **ws_kwargs)
+        if connect:
+            self.__enter__()
+        atexit.register(self.__exit__, None, None, None)
+
+    def __enter__(self):
+        if not self.pool:
+            self.pool = self.run_with_loop(self.conn.__aenter__(), self.loop)
+        return self
+
+    def send(self, *args):
+        self.run_with_loop(self.pool.send(*args), self.loop)
+
+    def recv(self, *args):
+        return self.run_with_loop(self.pool.recv(*args), self.loop)
+
+    def __exit__(self, type, value, traceback):
+        try:
+            return self.run_with_loop(self.conn.__aexit__(type, value, traceback), self.loop)
+        finally:
+            self.loop.close()
+            self._cancel_all_tasks(self.loop)
+
+    # taken from https://github.com/python/cpython/blob/3.9/Lib/asyncio/runners.py with modifications
+    # for contributors: If there's any new library using asyncio needs to be run in non-async, move these function out of this class
+    @staticmethod
+    def run_with_loop(main, loop):
+        if not asyncio.iscoroutine(main):
+            raise ValueError(f'a coroutine was expected, got {main!r}')
+
+        try:
+            return loop.run_until_complete(main)
+        finally:
+            loop.run_until_complete(loop.shutdown_asyncgens())
+            if hasattr(loop, 'shutdown_default_executor'):
+                loop.run_until_complete(loop.shutdown_default_executor())
+
+    @staticmethod
+    def _cancel_all_tasks(loop):
+        to_cancel = asyncio.all_tasks(loop)
+
+        if not to_cancel:
+            return
+
+        for task in to_cancel:
+            task.cancel()
+
+        # XXX: "loop" is removed in python 3.10+
+        loop.run_until_complete(
+            asyncio.gather(*to_cancel, loop=loop, return_exceptions=True))
+
+        for task in to_cancel:
+            if task.cancelled():
+                continue
+            if task.exception() is not None:
+                loop.call_exception_handler({
+                    'message': 'unhandled exception during asyncio.run() shutdown',
+                    'exception': task.exception(),
+                    'task': task,
+                })
+
+
 def load_plugins(name, suffix, namespace):
     from ..plugins import load_plugins
     ret = load_plugins(name, suffix)
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 10c7c4311..b0164a895 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -1,5 +1,3 @@
-import asyncio
-import atexit
 import base64
 import binascii
 import calendar
@@ -54,7 +52,7 @@
     compat_os_name,
     compat_shlex_quote,
 )
-from ..dependencies import websockets, xattr
+from ..dependencies import xattr
 
 __name__ = __name__.rsplit('.', 1)[0]  # Pretend to be the parent module
 
@@ -4923,77 +4921,6 @@ def parse_args(self):
         return self.parser.parse_args(self.all_args)
 
 
-class WebSocketsWrapper:
-    """Wraps websockets module to use in non-async scopes"""
-    pool = None
-
-    def __init__(self, url, headers=None, connect=True):
-        self.loop = asyncio.new_event_loop()
-        # XXX: "loop" is deprecated
-        self.conn = websockets.connect(
-            url, extra_headers=headers, ping_interval=None,
-            close_timeout=float('inf'), loop=self.loop, ping_timeout=float('inf'))
-        if connect:
-            self.__enter__()
-        atexit.register(self.__exit__, None, None, None)
-
-    def __enter__(self):
-        if not self.pool:
-            self.pool = self.run_with_loop(self.conn.__aenter__(), self.loop)
-        return self
-
-    def send(self, *args):
-        self.run_with_loop(self.pool.send(*args), self.loop)
-
-    def recv(self, *args):
-        return self.run_with_loop(self.pool.recv(*args), self.loop)
-
-    def __exit__(self, type, value, traceback):
-        try:
-            return self.run_with_loop(self.conn.__aexit__(type, value, traceback), self.loop)
-        finally:
-            self.loop.close()
-            self._cancel_all_tasks(self.loop)
-
-    # taken from https://github.com/python/cpython/blob/3.9/Lib/asyncio/runners.py with modifications
-    # for contributors: If there's any new library using asyncio needs to be run in non-async, move these function out of this class
-    @staticmethod
-    def run_with_loop(main, loop):
-        if not asyncio.iscoroutine(main):
-            raise ValueError(f'a coroutine was expected, got {main!r}')
-
-        try:
-            return loop.run_until_complete(main)
-        finally:
-            loop.run_until_complete(loop.shutdown_asyncgens())
-            if hasattr(loop, 'shutdown_default_executor'):
-                loop.run_until_complete(loop.shutdown_default_executor())
-
-    @staticmethod
-    def _cancel_all_tasks(loop):
-        to_cancel = asyncio.all_tasks(loop)
-
-        if not to_cancel:
-            return
-
-        for task in to_cancel:
-            task.cancel()
-
-        # XXX: "loop" is removed in python 3.10+
-        loop.run_until_complete(
-            asyncio.gather(*to_cancel, loop=loop, return_exceptions=True))
-
-        for task in to_cancel:
-            if task.cancelled():
-                continue
-            if task.exception() is not None:
-                loop.call_exception_handler({
-                    'message': 'unhandled exception during asyncio.run() shutdown',
-                    'exception': task.exception(),
-                    'task': task,
-                })
-
-
 def merge_headers(*dicts):
     """Merge dicts of http headers case insensitively, prioritizing the latter ones"""
     return {k.title(): v for k, v in itertools.chain.from_iterable(map(dict.items, dicts))}

From cc07f5cc85d9e2a6cd0bedb9d961665eea0d6047 Mon Sep 17 00:00:00 2001
From: middlingphys <38708390+middlingphys@users.noreply.github.com>
Date: Tue, 21 Nov 2023 07:39:12 +0900
Subject: [PATCH 709/871] [ie/abematv] Fix season metadata (#8607)

Authored by: middlingphys
---
 yt_dlp/extractor/abematv.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index 2a093580c..57ccb928b 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -211,7 +211,8 @@ class AbemaTVIE(AbemaTVBaseIE):
             'id': '194-25_s2_p1',
             'title': '第1話 「チーズケーキ」　「モーニング再び」',
             'series': '異世界食堂２',
-            'series_number': 2,
+            'season': 'シーズン2',
+            'season_number': 2,
             'episode': '第1話 「チーズケーキ」　「モーニング再び」',
             'episode_number': 1,
         },
@@ -347,12 +348,12 @@ def _real_extract(self, url):
                     )?
                 ''', r'\1', og_desc)
 
-        # canonical URL may contain series and episode number
+        # canonical URL may contain season and episode number
         mobj = re.search(r's(\d+)_p(\d+)$', canonical_url)
         if mobj:
             seri = int_or_none(mobj.group(1), default=float('inf'))
             epis = int_or_none(mobj.group(2), default=float('inf'))
-            info['series_number'] = seri if seri < 100 else None
+            info['season_number'] = seri if seri < 100 else None
             # some anime like Detective Conan (though not available in AbemaTV)
             # has more than 1000 episodes (1026 as of 2021/11/15)
             info['episode_number'] = epis if epis < 2000 else None
@@ -381,7 +382,7 @@ def _real_extract(self, url):
                 self.report_warning('This is a premium-only stream')
             info.update(traverse_obj(api_response, {
                 'series': ('series', 'title'),
-                'season': ('season', 'title'),
+                'season': ('season', 'name'),
                 'season_number': ('season', 'sequence'),
                 'episode_number': ('episode', 'number'),
             }))

From a0b19d319a6ce8b7059318fa17a34b144fde1785 Mon Sep 17 00:00:00 2001
From: pk <prettykool@protonmail.com>
Date: Mon, 20 Nov 2023 16:43:52 -0600
Subject: [PATCH 710/871] [core] Support `NO_COLOR` environment variable
 (#8385)

Authored by: prettykool, Grub4K
---
 yt_dlp/YoutubeDL.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 85b282bd5..32ae25aa0 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -625,13 +625,16 @@ def __init__(self, params=None, auto_init=True):
                     'Overwriting params from "color" with "no_color"')
             self.params['color'] = 'no_color'
 
-        term_allow_color = os.environ.get('TERM', '').lower() != 'dumb'
+        term_allow_color = os.getenv('TERM', '').lower() != 'dumb'
+        no_color = bool(os.getenv('NO_COLOR'))
 
         def process_color_policy(stream):
             stream_name = {sys.stdout: 'stdout', sys.stderr: 'stderr'}[stream]
             policy = traverse_obj(self.params, ('color', (stream_name, None), {str}), get_all=False)
             if policy in ('auto', None):
-                return term_allow_color and supports_terminal_sequences(stream)
+                if term_allow_color and supports_terminal_sequences(stream):
+                    return 'no_color' if no_color else True
+                return False
             assert policy in ('always', 'never', 'no_color'), policy
             return {'always': True, 'never': False}.get(policy, policy)
 

From 1732eccc0a40256e076bf0435a29f0f1d8419280 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sun, 26 Nov 2023 03:12:05 +0100
Subject: [PATCH 711/871] [core] Parse `release_year` from `release_date`
 (#8524)

Closes #7263
Authored by: seproDev
---
 README.md                      |  2 +-
 test/helper.py                 |  6 +++++-
 yt_dlp/YoutubeDL.py            |  3 +++
 yt_dlp/extractor/archiveorg.py |  2 --
 yt_dlp/extractor/common.py     |  4 +++-
 yt_dlp/extractor/harpodeon.py  | 10 +++++-----
 yt_dlp/extractor/monstercat.py |  2 --
 yt_dlp/extractor/youtube.py    |  1 -
 8 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 6fe7fab6a..f67cab572 100644
--- a/README.md
+++ b/README.md
@@ -1309,6 +1309,7 @@ # OUTPUT TEMPLATE
  - `upload_date` (string): Video upload date in UTC (YYYYMMDD)
  - `release_timestamp` (numeric): UNIX timestamp of the moment the video was released
  - `release_date` (string): The date (YYYYMMDD) when the video was released in UTC
+ - `release_year` (numeric): Year (YYYY) when the video or album was released
  - `modified_timestamp` (numeric): UNIX timestamp of the moment the video was last modified
  - `modified_date` (string): The date (YYYYMMDD) when the video was last modified in UTC
  - `uploader_id` (string): Nickname or id of the video uploader
@@ -1382,7 +1383,6 @@ # OUTPUT TEMPLATE
  - `album_type` (string): Type of the album
  - `album_artist` (string): List of all artists appeared on the album
  - `disc_number` (numeric): Number of the disc or other physical medium the track belongs to
- - `release_year` (numeric): Year (YYYY) when the album was released
 
 Available only when using `--download-sections` and for `chapter:` prefix when using `--split-chapters` for videos with internal chapters:
 
diff --git a/test/helper.py b/test/helper.py
index 7503840be..e5ace8fe2 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -10,7 +10,7 @@
 import yt_dlp.extractor
 from yt_dlp import YoutubeDL
 from yt_dlp.compat import compat_os_name
-from yt_dlp.utils import preferredencoding, write_string
+from yt_dlp.utils import preferredencoding, try_call, write_string
 
 if 'pytest' in sys.modules:
     import pytest
@@ -223,6 +223,10 @@ def sanitize(key, value):
     if test_info_dict.get('display_id') == test_info_dict.get('id'):
         test_info_dict.pop('display_id')
 
+    # release_year may be generated from release_date
+    if try_call(lambda: test_info_dict['release_year'] == int(test_info_dict['release_date'][:4])):
+        test_info_dict.pop('release_year')
+
     # Check url for flat entries
     if got_dict.get('_type', 'video') != 'video' and got_dict.get('url'):
         test_info_dict['url'] = got_dict['url']
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 32ae25aa0..e65bef862 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2589,6 +2589,9 @@ def _fill_common_fields(self, info_dict, final=True):
                     upload_date = datetime.datetime.fromtimestamp(info_dict[ts_key], datetime.timezone.utc)
                     info_dict[date_key] = upload_date.strftime('%Y%m%d')
 
+        if not info_dict.get('release_year'):
+            info_dict['release_year'] = traverse_obj(info_dict, ('release_date', {lambda x: int(x[:4])}))
+
         live_keys = ('is_live', 'was_live')
         live_status = info_dict.get('live_status')
         if live_status is None:
diff --git a/yt_dlp/extractor/archiveorg.py b/yt_dlp/extractor/archiveorg.py
index a0b26ac5a..3bb6f2e31 100644
--- a/yt_dlp/extractor/archiveorg.py
+++ b/yt_dlp/extractor/archiveorg.py
@@ -52,7 +52,6 @@ class ArchiveOrgIE(InfoExtractor):
             'creator': 'SRI International',
             'uploader': 'laura@archive.org',
             'thumbnail': r're:https://archive\.org/download/.*\.jpg',
-            'release_year': 1968,
             'display_id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.cdr',
             'track': 'XD300-23 68HighlightsAResearchCntAugHumanIntellect',
 
@@ -134,7 +133,6 @@ class ArchiveOrgIE(InfoExtractor):
             'album': '1977-05-08 - Barton Hall - Cornell University',
             'release_date': '19770508',
             'display_id': 'gd1977-05-08d01t07.flac',
-            'release_year': 1977,
             'track_number': 7,
         },
     }, {
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 507ef34cf..b179f4038 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -286,6 +286,9 @@ class InfoExtractor:
                     If it is not clear whether to use timestamp or this, use the former
     release_date:   The date (YYYYMMDD) when the video was released in UTC.
                     If not explicitly set, calculated from release_timestamp
+    release_year:   Year (YYYY) as integer when the video or album was released.
+                    To be used if no exact release date is known.
+                    If not explicitly set, calculated from release_date.
     modified_timestamp: UNIX timestamp of the moment the video was last modified.
     modified_date:   The date (YYYYMMDD) when the video was last modified in UTC.
                     If not explicitly set, calculated from modified_timestamp
@@ -427,7 +430,6 @@ class InfoExtractor:
                     and compilations).
     disc_number:    Number of the disc or other physical medium the track belongs to,
                     as an integer.
-    release_year:   Year (YYYY) when the album was released.
     composer:       Composer of the piece
 
     The following fields should only be set for clips that should be cut from the original video:
diff --git a/yt_dlp/extractor/harpodeon.py b/yt_dlp/extractor/harpodeon.py
index 0aa47337f..46eaddb32 100644
--- a/yt_dlp/extractor/harpodeon.py
+++ b/yt_dlp/extractor/harpodeon.py
@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import int_or_none
 
 
 class HarpodeonIE(InfoExtractor):
@@ -14,7 +14,7 @@ class HarpodeonIE(InfoExtractor):
             'title': 'The Smoking Out of Bella Butts',
             'description': 'md5:47e16bdb41fc8a79c83ab83af11c8b77',
             'creator': 'Vitagraph Company of America',
-            'release_date': '19150101'
+            'release_year': 1915,
         }
     }, {
         'url': 'https://www.harpodeon.com/preview/The_Smoking_Out_of_Bella_Butts/268068288',
@@ -25,7 +25,7 @@ class HarpodeonIE(InfoExtractor):
             'title': 'The Smoking Out of Bella Butts',
             'description': 'md5:47e16bdb41fc8a79c83ab83af11c8b77',
             'creator': 'Vitagraph Company of America',
-            'release_date': '19150101'
+            'release_year': 1915,
         }
     }, {
         'url': 'https://www.harpodeon.com/preview/Behind_the_Screen/421838710',
@@ -36,7 +36,7 @@ class HarpodeonIE(InfoExtractor):
             'title': 'Behind the Screen',
             'description': 'md5:008972a3dc51fba3965ee517d2ba9155',
             'creator': 'Lone Star Corporation',
-            'release_date': '19160101'
+            'release_year': 1916,
         }
     }]
 
@@ -66,5 +66,5 @@ def _real_extract(self, url):
             'http_headers': {'Referer': url},
             'description': self._html_search_meta('description', webpage, fatal=False),
             'creator': creator,
-            'release_date': unified_strdate(f'{release_year}0101')
+            'release_year': int_or_none(release_year),
         }
diff --git a/yt_dlp/extractor/monstercat.py b/yt_dlp/extractor/monstercat.py
index 7f04825fc..cf5e09969 100644
--- a/yt_dlp/extractor/monstercat.py
+++ b/yt_dlp/extractor/monstercat.py
@@ -24,7 +24,6 @@ class MonstercatIE(InfoExtractor):
             'title': 'The Secret Language of Trees',
             'id': '742779548009',
             'thumbnail': 'https://www.monstercat.com/release/742779548009/cover',
-            'release_year': 2023,
             'release_date': '20230711',
             'album': 'The Secret Language of Trees',
             'album_artist': 'BT',
@@ -71,7 +70,6 @@ def _real_extract(self, url):
             'thumbnail': f'https://www.monstercat.com/release/{url_id}/cover',
             'album_artist': try_call(
                 lambda: get_element_by_class('h-normal text-uppercase mb-desktop-medium mb-smallish', html)),
-            'release_year': int_or_none(date[:4]) if date else None,
             'release_date': date,
         }
 
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index f6caf0970..73fe233ef 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -2072,7 +2072,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'track': 'Voyeur Girl',
                 'album': 'it\'s too much love to know my dear',
                 'release_date': '20190313',
-                'release_year': 2019,
                 'alt_title': 'Voyeur Girl',
                 'view_count': int,
                 'playable_in_embed': True,

From 9cafb9ff17e14475a35c9a58b5bb010c86c9db4b Mon Sep 17 00:00:00 2001
From: kclauhk <78251477+kclauhk@users.noreply.github.com>
Date: Sun, 26 Nov 2023 10:17:16 +0800
Subject: [PATCH 712/871] [ie/facebook] Improve subtitles extraction (#8296)

Authored by: kclauhk
---
 yt_dlp/extractor/facebook.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 50a750d3b..58162cc5f 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -16,6 +16,7 @@
     determine_ext,
     error_to_compat_str,
     float_or_none,
+    format_field,
     get_element_by_id,
     get_first,
     int_or_none,
@@ -420,6 +421,29 @@ def extract_metadata(webpage):
                 r'data-sjs>({.*?ScheduledServerJS.*?})</script>', webpage)]
             post = traverse_obj(post_data, (
                 ..., 'require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., '__bbox', 'result', 'data'), expected_type=dict) or []
+
+            automatic_captions, subtitles = {}, {}
+            subs_data = traverse_obj(post, (..., 'video', ..., 'attachments', ..., lambda k, v: (
+                k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')))
+            is_video_broadcast = get_first(subs_data, 'is_video_broadcast', expected_type=bool)
+            captions = get_first(subs_data, 'video_available_captions_locales', 'captions_url')
+            if url_or_none(captions):  # if subs_data only had a 'captions_url'
+                locale = self._html_search_meta(['og:locale', 'twitter:locale'], webpage, 'locale', default='en_US')
+                subtitles[locale] = [{'url': captions}]
+            # or else subs_data had 'video_available_captions_locales', a list of dicts
+            for caption in traverse_obj(captions, (
+                {lambda x: sorted(x, key=lambda c: c['locale'])}, lambda _, v: v['captions_url'])
+            ):
+                lang = caption.get('localized_language') or ''
+                subs = {
+                    'url': caption['captions_url'],
+                    'name': format_field(caption, 'localized_country', f'{lang} (%s)', default=lang),
+                }
+                if caption.get('localized_creation_method') or is_video_broadcast:
+                    automatic_captions.setdefault(caption['locale'], []).append(subs)
+                else:
+                    subtitles.setdefault(caption['locale'], []).append(subs)
+
             media = traverse_obj(post, (..., 'attachments', ..., lambda k, v: (
                 k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')), expected_type=dict)
             title = get_first(media, ('title', 'text'))
@@ -463,6 +487,8 @@ def extract_metadata(webpage):
                     webpage, 'view count', default=None)),
                 'concurrent_view_count': get_first(post, (
                     ('video', (..., ..., 'attachments', ..., 'media')), 'liveViewerCount', {int_or_none})),
+                'automatic_captions': automatic_captions,
+                'subtitles': subtitles,
             }
 
             info_json_ld = self._search_json_ld(webpage, video_id, default={})

From 628fa244bbce2ad39775a5959e99588f30cac152 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sun, 26 Nov 2023 03:20:10 +0100
Subject: [PATCH 713/871] [ie/floatplane] Add extractors (#8639)

Closes #5877, Closes #5912
Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/floatplane.py  | 268 ++++++++++++++++++++++++++++++++
 2 files changed, 272 insertions(+)
 create mode 100644 yt_dlp/extractor/floatplane.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 128b86c1a..ad8c7d661 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -642,6 +642,10 @@
 from .firsttv import FirstTVIE
 from .fivetv import FiveTVIE
 from .flickr import FlickrIE
+from .floatplane import (
+    FloatplaneIE,
+    FloatplaneChannelIE,
+)
 from .folketinget import FolketingetIE
 from .footyroom import FootyRoomIE
 from .formula1 import Formula1IE
diff --git a/yt_dlp/extractor/floatplane.py b/yt_dlp/extractor/floatplane.py
new file mode 100644
index 000000000..09abb40bf
--- /dev/null
+++ b/yt_dlp/extractor/floatplane.py
@@ -0,0 +1,268 @@
+import functools
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    OnDemandPagedList,
+    clean_html,
+    determine_ext,
+    format_field,
+    int_or_none,
+    join_nonempty,
+    parse_codecs,
+    parse_iso8601,
+    urljoin,
+)
+from ..utils.traversal import traverse_obj
+
+
+class FloatplaneIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/post/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://www.floatplane.com/post/2Yf3UedF7C',
+        'info_dict': {
+            'id': 'yuleLogLTT',
+            'ext': 'mp4',
+            'display_id': '2Yf3UedF7C',
+            'title': '8K Yule Log Fireplace with Crackling Fire Sounds - 10 Hours',
+            'description': 'md5:adf2970e0de1c5e3df447818bb0309f6',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'duration': 36035,
+            'comment_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'release_date': '20191206',
+            'release_timestamp': 1575657000,
+            'uploader': 'LinusTechTips',
+            'uploader_id': '59f94c0bdd241b70349eb72b',
+            'uploader_url': 'https://www.floatplane.com/channel/linustechtips/home',
+            'channel': 'Linus Tech Tips',
+            'channel_id': '63fe42c309e691e4e36de93d',
+            'channel_url': 'https://www.floatplane.com/channel/linustechtips/home/main',
+            'availability': 'subscriber_only',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.floatplane.com/post/j2jqG3JmgJ',
+        'info_dict': {
+            'id': 'j2jqG3JmgJ',
+            'title': 'TJM: Does Anyone Care About Avatar: The Way of Water?',
+            'description': 'md5:00bf17dc5733e4031e99b7fd6489f274',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'comment_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'release_timestamp': 1671915900,
+            'release_date': '20221224',
+            'uploader': 'LinusTechTips',
+            'uploader_id': '59f94c0bdd241b70349eb72b',
+            'uploader_url': 'https://www.floatplane.com/channel/linustechtips/home',
+            'channel': "They're Just Movies",
+            'channel_id': '64135f82fc76ab7f9fbdc876',
+            'channel_url': 'https://www.floatplane.com/channel/linustechtips/home/tajm',
+            'availability': 'subscriber_only',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'https://www.floatplane.com/post/3tK2tInhoN',
+        'info_dict': {
+            'id': '3tK2tInhoN',
+            'title': 'Extras - How Linus Communicates with Editors (Compensator 4)',
+            'description': 'md5:83cd40aae1ce124df33769600c80ca5b',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'comment_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'release_timestamp': 1700529120,
+            'release_date': '20231121',
+            'uploader': 'LinusTechTips',
+            'uploader_id': '59f94c0bdd241b70349eb72b',
+            'uploader_url': 'https://www.floatplane.com/channel/linustechtips/home',
+            'channel': 'FP Exclusives',
+            'channel_id': '6413623f5b12cca228a28e78',
+            'channel_url': 'https://www.floatplane.com/channel/linustechtips/home/fpexclusive',
+            'availability': 'subscriber_only',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'https://beta.floatplane.com/post/d870PEFXS1',
+        'info_dict': {
+            'id': 'bg9SuYKEww',
+            'ext': 'mp4',
+            'display_id': 'd870PEFXS1',
+            'title': 'LCS Drama, TLOU 2 Remaster, Destiny 2 Player Count Drops, + More!',
+            'description': 'md5:80d612dcabf41b17487afcbe303ec57d',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'release_timestamp': 1700622000,
+            'release_date': '20231122',
+            'duration': 513,
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'uploader': 'LinusTechTips',
+            'uploader_id': '59f94c0bdd241b70349eb72b',
+            'uploader_url': 'https://www.floatplane.com/channel/linustechtips/home',
+            'channel': 'GameLinked',
+            'channel_id': '649dbade3540dbc3945eeda7',
+            'channel_url': 'https://www.floatplane.com/channel/linustechtips/home/gamelinked',
+            'availability': 'subscriber_only',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_initialize(self):
+        if not self._get_cookies('https://www.floatplane.com').get('sails.sid'):
+            self.raise_login_required()
+
+    def _real_extract(self, url):
+        post_id = self._match_id(url)
+
+        post_data = self._download_json(
+            'https://www.floatplane.com/api/v3/content/post', post_id, query={'id': post_id},
+            note='Downloading post data', errnote='Unable to download post data')
+
+        if not any(traverse_obj(post_data, ('metadata', ('hasVideo', 'hasAudio')))):
+            raise ExtractorError('Post does not contain a video or audio track', expected=True)
+
+        items = []
+        for media in traverse_obj(post_data, (('videoAttachments', 'audioAttachments'), ...)):
+            media_id = media['id']
+            media_typ = media.get('type') or 'video'
+
+            metadata = self._download_json(
+                f'https://www.floatplane.com/api/v3/content/{media_typ}', media_id, query={'id': media_id},
+                note=f'Downloading {media_typ} metadata')
+
+            stream = self._download_json(
+                'https://www.floatplane.com/api/v2/cdn/delivery', media_id, query={
+                    'type': 'vod' if media_typ == 'video' else 'aod',
+                    'guid': metadata['guid']
+                }, note=f'Downloading {media_typ} stream data')
+
+            path_template = traverse_obj(stream, ('resource', 'uri', {str}))
+
+            def format_path(params):
+                path = path_template
+                for i, val in (params or {}).items():
+                    path = path.replace(f'{{qualityLevelParams.{i}}}', val)
+                return path
+
+            formats = []
+            for quality in traverse_obj(stream, ('resource', 'data', 'qualityLevels', ...)):
+                url = urljoin(stream['cdn'], format_path(traverse_obj(
+                    stream, ('resource', 'data', 'qualityLevelParams', quality['name']))))
+                formats.append({
+                    **traverse_obj(quality, {
+                        'format_id': 'name',
+                        'format_note': 'label',
+                        'width': ('width', {int}),
+                        'height': ('height', {int}),
+                    }),
+                    **parse_codecs(quality.get('codecs')),
+                    'url': url,
+                    'ext': determine_ext(url.partition('/chunk.m3u8')[0], 'mp4'),
+                })
+
+            items.append({
+                'id': media_id,
+                **traverse_obj(metadata, {
+                    'title': 'title',
+                    'duration': ('duration', {int_or_none}),
+                    'thumbnail': ('thumbnail', 'path'),
+                }),
+                'formats': formats,
+            })
+
+        uploader_url = format_field(traverse_obj(
+            post_data, 'creator'), 'urlname', 'https://www.floatplane.com/channel/%s/home', default=None)
+        channel_url = urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname')))
+
+        post_info = {
+            'id': post_id,
+            'display_id': post_id,
+            **traverse_obj(post_data, {
+                'title': 'title',
+                'description': ('text', {clean_html}),
+                'uploader': ('creator', 'title'),
+                'uploader_id': ('creator', 'id'),
+                'channel': ('channel', 'title'),
+                'channel_id': ('channel', 'id'),
+                'like_count': ('likes', {int_or_none}),
+                'dislike_count': ('dislikes', {int_or_none}),
+                'comment_count': ('comments', {int_or_none}),
+                'release_timestamp': ('releaseDate', {parse_iso8601}),
+                'thumbnail': ('thumbnail', 'path'),
+            }),
+            'uploader_url': uploader_url,
+            'channel_url': channel_url,
+            'availability': self._availability(needs_subscription=True),
+        }
+
+        if len(items) > 1:
+            return self.playlist_result(items, **post_info)
+
+        post_info.update(items[0])
+        return post_info
+
+
+class FloatplaneChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|beta)\.)?floatplane\.com/channel/(?P<id>[\w-]+)/home(?:/(?P<channel>[\w-]+))?'
+    _PAGE_SIZE = 20
+    _TESTS = [{
+        'url': 'https://www.floatplane.com/channel/linustechtips/home/ltxexpo',
+        'info_dict': {
+            'id': 'linustechtips/ltxexpo',
+            'title': 'LTX Expo',
+            'description': 'md5:9819002f9ebe7fd7c75a3a1d38a59149',
+        },
+        'playlist_mincount': 51,
+    }, {
+        'url': 'https://www.floatplane.com/channel/ShankMods/home',
+        'info_dict': {
+            'id': 'ShankMods',
+            'title': 'Shank Mods',
+            'description': 'md5:6dff1bb07cad8e5448e04daad9be1b30',
+        },
+        'playlist_mincount': 14,
+    }, {
+        'url': 'https://beta.floatplane.com/channel/bitwit_ultra/home',
+        'info_dict': {
+            'id': 'bitwit_ultra',
+            'title': 'Bitwit Ultra',
+            'description': 'md5:1452f280bb45962976d4789200f676dd',
+        },
+        'playlist_mincount': 200,
+    }]
+
+    def _fetch_page(self, display_id, creator_id, channel_id, page):
+        query = {
+            'id': creator_id,
+            'limit': self._PAGE_SIZE,
+            'fetchAfter': page * self._PAGE_SIZE,
+        }
+        if channel_id:
+            query['channel'] = channel_id
+        page_data = self._download_json(
+            'https://www.floatplane.com/api/v3/content/creator', display_id,
+            query=query, note=f'Downloading page {page + 1}')
+        for post in page_data or []:
+            yield self.url_result(
+                f'https://www.floatplane.com/post/{post["id"]}',
+                ie=FloatplaneIE, video_id=post['id'], video_title=post.get('title'),
+                release_timestamp=parse_iso8601(post.get('releaseDate')))
+
+    def _real_extract(self, url):
+        creator, channel = self._match_valid_url(url).group('id', 'channel')
+        display_id = join_nonempty(creator, channel, delim='/')
+
+        creator_data = self._download_json(
+            'https://www.floatplane.com/api/v3/creator/named',
+            display_id, query={'creatorURL[0]': creator})[0]
+
+        channel_data = traverse_obj(
+            creator_data, ('channels', lambda _, v: v['urlname'] == channel), get_all=False) or {}
+
+        return self.playlist_result(OnDemandPagedList(functools.partial(
+            self._fetch_page, display_id, creator_data['id'], channel_data.get('id')), self._PAGE_SIZE),
+            display_id, playlist_title=channel_data.get('title') or creator_data.get('title'),
+            playlist_description=channel_data.get('about') or creator_data.get('about'))

From bb5a54e6db2422bbd155d93a0e105b6616c09467 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 25 Nov 2023 20:21:29 -0600
Subject: [PATCH 714/871] [ie/youtube] Improve detection of faulty HLS formats
 (#8646)

Closes #7747
Authored by: bashonly
---
 yt_dlp/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 73fe233ef..449d9d1a5 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4562,7 +4562,7 @@ def process_language(container, base_url, lang_code, sub_name, query):
         if upload_date and live_status not in ('is_live', 'post_live', 'is_upcoming'):
             # Newly uploaded videos' HLS formats are potentially problematic and need to be checked
             upload_datetime = datetime_from_str(upload_date).replace(tzinfo=datetime.timezone.utc)
-            if upload_datetime >= datetime_from_str('today-1day'):
+            if upload_datetime >= datetime_from_str('today-2days'):
                 for fmt in info['formats']:
                     if fmt.get('protocol') == 'm3u8_native':
                         fmt['__needs_testing'] = True

From deeb13eae82e60f82a2c0c5861f460399a997528 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 25 Nov 2023 20:40:09 -0600
Subject: [PATCH 715/871] [pp/FFmpegMetadata] Embed stream metadata in single
 format downloads (#8647)

Closes #8568
Authored by: bashonly
---
 yt_dlp/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py
index 323f4303c..7c904417b 100644
--- a/yt_dlp/postprocessor/ffmpeg.py
+++ b/yt_dlp/postprocessor/ffmpeg.py
@@ -780,7 +780,7 @@ def add(meta_list, info_list=None):
             yield ('-metadata', f'{name}={value}')
 
         stream_idx = 0
-        for fmt in info.get('requested_formats') or []:
+        for fmt in info.get('requested_formats') or [info]:
             stream_count = 2 if 'none' not in (fmt.get('vcodec'), fmt.get('acodec')) else 1
             lang = ISO639Utils.short2long(fmt.get('language') or '') or fmt.get('language')
             for i in range(stream_idx, stream_idx + stream_count):

From ff2fde1b8f922fd34bae6172602008cd67c07c93 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 25 Nov 2023 20:47:48 -0600
Subject: [PATCH 716/871] [ie/TwitCastingUser] Fix extraction (#8650)

Closes #8653
Authored by: bashonly
---
 yt_dlp/extractor/twitcasting.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py
index 3c303bdbb..28ea16cc2 100644
--- a/yt_dlp/extractor/twitcasting.py
+++ b/yt_dlp/extractor/twitcasting.py
@@ -289,8 +289,7 @@ def _entries(self, uploader_id):
             webpage = self._download_webpage(
                 next_url, uploader_id, query={'filter': 'watchable'}, note='Downloading page %d' % page_num)
             matches = re.finditer(
-                r'''(?isx)<a\s+class="tw-movie-thumbnail"\s*href="(?P<url>/[^/]+/movie/\d+)"\s*>.+?</a>''',
-                webpage)
+                r'(?s)<a\s+class="tw-movie-thumbnail2"\s+href="(?P<url>/[^/"]+/movie/\d+)"', webpage)
             for mobj in matches:
                 yield self.url_result(urljoin(base_url, mobj.group('url')))
 

From 4903f452b68efb62dadf22e81be8c7934fc743e7 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 25 Nov 2023 20:49:18 -0600
Subject: [PATCH 717/871] [ie/bfmtv] Fix extractors (#8651)

Closes #8425
Authored by: bashonly
---
 yt_dlp/extractor/bfmtv.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/bfmtv.py b/yt_dlp/extractor/bfmtv.py
index a7be0e67d..5d0c73ff3 100644
--- a/yt_dlp/extractor/bfmtv.py
+++ b/yt_dlp/extractor/bfmtv.py
@@ -7,7 +7,7 @@
 class BFMTVBaseIE(InfoExtractor):
     _VALID_URL_BASE = r'https?://(?:www\.|rmc\.)?bfmtv\.com/'
     _VALID_URL_TMPL = _VALID_URL_BASE + r'(?:[^/]+/)*[^/?&#]+_%s[A-Z]-(?P<id>\d{12})\.html'
-    _VIDEO_BLOCK_REGEX = r'(<div[^>]+class="video_block"[^>]*>)'
+    _VIDEO_BLOCK_REGEX = r'(<div[^>]+class="video_block[^"]*"[^>]*>)'
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
 
     def _brightcove_url_result(self, video_id, video_block):
@@ -55,8 +55,11 @@ class BFMTVLiveIE(BFMTVIE):  # XXX: Do not subclass from concrete IE
             'ext': 'mp4',
             'title': r're:^le direct BFMTV WEB \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
             'uploader_id': '876450610001',
-            'upload_date': '20171018',
-            'timestamp': 1508329950,
+            'upload_date': '20220926',
+            'timestamp': 1664207191,
+            'live_status': 'is_live',
+            'thumbnail': r're:https://.+/image\.jpg',
+            'tags': [],
         },
         'params': {
             'skip_download': True,

From 5a230233d6fce06f4abd1fce0dc92b948e6f780b Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sat, 25 Nov 2023 20:50:23 -0600
Subject: [PATCH 718/871] [ie/box] Fix formats extraction (#8649)

Closes #5098
Authored by: bashonly
---
 yt_dlp/extractor/box.py | 41 ++++++++++++++++-------------------------
 1 file changed, 16 insertions(+), 25 deletions(-)

diff --git a/yt_dlp/extractor/box.py b/yt_dlp/extractor/box.py
index 8ab149626..7281b3c6a 100644
--- a/yt_dlp/extractor/box.py
+++ b/yt_dlp/extractor/box.py
@@ -1,16 +1,17 @@
 import json
+import urllib.parse
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     parse_iso8601,
-    # try_get,
     update_url_query,
+    url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class BoxIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^.]+\.)?app\.box\.com/s/(?P<shared_name>[^/]+)/file/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:[^.]+\.)?app\.box\.com/s/(?P<shared_name>[^/?#]+)/file/(?P<id>\d+)'
     _TEST = {
         'url': 'https://mlssoccer.app.box.com/s/0evd2o3e08l60lr4ygukepvnkord1o1x/file/510727257538',
         'md5': '1f81b2fd3960f38a40a3b8823e5fcd43',
@@ -18,11 +19,12 @@ class BoxIE(InfoExtractor):
             'id': '510727257538',
             'ext': 'mp4',
             'title': 'Garber   St. Louis will be 28th MLS team  +scarving.mp4',
-            'uploader': 'MLS Video',
+            'uploader': '',
             'timestamp': 1566320259,
             'upload_date': '20190820',
             'uploader_id': '235196876',
-        }
+        },
+        'params': {'skip_download': 'dash fragment too small'},
     }
 
     def _real_extract(self, url):
@@ -58,26 +60,15 @@ def _real_extract(self, url):
 
         formats = []
 
-        # for entry in (try_get(f, lambda x: x['representations']['entries'], list) or []):
-        #     entry_url_template = try_get(
-        #         entry, lambda x: x['content']['url_template'])
-        #     if not entry_url_template:
-        #         continue
-        #     representation = entry.get('representation')
-        #     if representation == 'dash':
-        #         TODO: append query to every fragment URL
-        #         formats.extend(self._extract_mpd_formats(
-        #             entry_url_template.replace('{+asset_path}', 'manifest.mpd'),
-        #             file_id, query=query))
-
-        authenticated_download_url = f.get('authenticated_download_url')
-        if authenticated_download_url and f.get('is_download_available'):
-            formats.append({
-                'ext': f.get('extension') or determine_ext(title),
-                'filesize': f.get('size'),
-                'format_id': 'download',
-                'url': update_url_query(authenticated_download_url, query),
-            })
+        for url_tmpl in traverse_obj(f, (
+            'representations', 'entries', lambda _, v: v['representation'] == 'dash',
+            'content', 'url_template', {url_or_none}
+        )):
+            manifest_url = update_url_query(url_tmpl.replace('{+asset_path}', 'manifest.mpd'), query)
+            fmts = self._extract_mpd_formats(manifest_url, file_id)
+            for fmt in fmts:
+                fmt['extra_param_to_segment_url'] = urllib.parse.urlparse(manifest_url).query
+            formats.extend(fmts)
 
         creator = f.get('created_by') or {}
 

From 9751a457cfdb18bf99d9ee0d10e4e6a594502bbf Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sun, 26 Nov 2023 04:09:59 +0100
Subject: [PATCH 719/871] [cleanup] Remove dead extractors (#8604)

Closes #1609, Closes #3232, Closes #4763, Closes #6026, Closes #6322, Closes #7912
Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py     | 156 +------
 yt_dlp/extractor/airmozilla.py      |  63 ---
 yt_dlp/extractor/aol.py             |   1 +
 yt_dlp/extractor/atttechchannel.py  |  53 ---
 yt_dlp/extractor/behindkink.py      |   1 +
 yt_dlp/extractor/bet.py             |   3 +-
 yt_dlp/extractor/bfi.py             |   1 +
 yt_dlp/extractor/biqle.py           | 110 -----
 yt_dlp/extractor/bitwave.py         |  58 ---
 yt_dlp/extractor/bleacherreport.py  |   4 +-
 yt_dlp/extractor/br.py              | 144 +------
 yt_dlp/extractor/breakcom.py        |  86 ----
 yt_dlp/extractor/byutv.py           |  16 +-
 yt_dlp/extractor/camwithher.py      |  87 ----
 yt_dlp/extractor/carambatv.py       | 105 -----
 yt_dlp/extractor/channel9.py        | 252 -----------
 yt_dlp/extractor/chirbit.py         |  88 ----
 yt_dlp/extractor/cinchcast.py       |  56 ---
 yt_dlp/extractor/clipsyndicate.py   |  52 ---
 yt_dlp/extractor/cloudy.py          |  57 ---
 yt_dlp/extractor/clubic.py          |   1 +
 yt_dlp/extractor/cmt.py             |   1 +
 yt_dlp/extractor/daftsex.py         | 150 -------
 yt_dlp/extractor/defense.py         |  37 --
 yt_dlp/extractor/dhm.py             |   1 +
 yt_dlp/extractor/dotsub.py          |  81 ----
 yt_dlp/extractor/echomsk.py         |  43 --
 yt_dlp/extractor/ehow.py            |  36 --
 yt_dlp/extractor/elevensports.py    |  59 ---
 yt_dlp/extractor/ellentube.py       | 130 ------
 yt_dlp/extractor/engadget.py        |  15 -
 yt_dlp/extractor/escapist.py        | 108 -----
 yt_dlp/extractor/esri.py            |  70 ---
 yt_dlp/extractor/expotv.py          |  74 ----
 yt_dlp/extractor/extremetube.py     |  48 ---
 yt_dlp/extractor/fourzerostudio.py  | 106 -----
 yt_dlp/extractor/foxgay.py          |  58 ---
 yt_dlp/extractor/fusion.py          |  81 ----
 yt_dlp/extractor/generic.py         |  61 +--
 yt_dlp/extractor/gfycat.py          | 145 -------
 yt_dlp/extractor/groupon.py         |   1 -
 yt_dlp/extractor/helsinki.py        |  38 --
 yt_dlp/extractor/hitbox.py          | 209 ---------
 yt_dlp/extractor/howcast.py         |  41 --
 yt_dlp/extractor/howstuffworks.py   |  86 ----
 yt_dlp/extractor/keezmovies.py      | 125 ------
 yt_dlp/extractor/kinja.py           |   7 -
 yt_dlp/extractor/laola1tv.py        | 261 ------------
 yt_dlp/extractor/linuxacademy.py    | 238 -----------
 yt_dlp/extractor/m6.py              |  22 -
 yt_dlp/extractor/meta.py            |  70 ---
 yt_dlp/extractor/metacafe.py        | 281 ------------
 yt_dlp/extractor/mgoon.py           |  81 ----
 yt_dlp/extractor/miomio.py          | 134 ------
 yt_dlp/extractor/mnet.py            |  85 ----
 yt_dlp/extractor/moevideo.py        |  74 ----
 yt_dlp/extractor/mofosex.py         |  70 ---
 yt_dlp/extractor/movieclips.py      |  47 --
 yt_dlp/extractor/msn.py             |   1 +
 yt_dlp/extractor/mwave.py           |  87 ----
 yt_dlp/extractor/mychannels.py      |  35 --
 yt_dlp/extractor/myvi.py            | 100 -----
 yt_dlp/extractor/newstube.py        |  75 ----
 yt_dlp/extractor/nick.py            |  20 -
 yt_dlp/extractor/normalboots.py     |  51 ---
 yt_dlp/extractor/nosvideo.py        |  72 ----
 yt_dlp/extractor/nrl.py             |   1 +
 yt_dlp/extractor/ooyala.py          | 230 ----------
 yt_dlp/extractor/pandoratv.py       | 128 ------
 yt_dlp/extractor/people.py          |  29 --
 yt_dlp/extractor/playfm.py          |  70 ---
 yt_dlp/extractor/plays.py           |  49 ---
 yt_dlp/extractor/playvid.py         |  90 ----
 yt_dlp/extractor/porncom.py         |  99 -----
 yt_dlp/extractor/pornez.py          |  60 ---
 yt_dlp/extractor/pornhd.py          | 116 -----
 yt_dlp/extractor/radiobremen.py     |  59 ---
 yt_dlp/extractor/recurbate.py       |  42 --
 yt_dlp/extractor/rice.py            | 112 -----
 yt_dlp/extractor/rtl2.py            | 100 +----
 yt_dlp/extractor/rtvnh.py           |  58 ---
 yt_dlp/extractor/ruhd.py            |  42 --
 yt_dlp/extractor/scte.py            |   2 +
 yt_dlp/extractor/shared.py          | 138 ------
 yt_dlp/extractor/sky.py             |  33 +-
 yt_dlp/extractor/spankwire.py       | 174 --------
 yt_dlp/extractor/srmediathek.py     |   1 +
 yt_dlp/extractor/streamcloud.py     |  75 ----
 yt_dlp/extractor/swrmediathek.py    | 111 -----
 yt_dlp/extractor/techtalks.py       |  80 ----
 yt_dlp/extractor/telecinco.py       |   1 -
 yt_dlp/extractor/tinypic.py         |  54 ---
 yt_dlp/extractor/tokentube.py       | 153 -------
 yt_dlp/extractor/toypics.py         |   2 +
 yt_dlp/extractor/trilulilu.py       | 100 -----
 yt_dlp/extractor/tube8.py           |  95 ++++-
 yt_dlp/extractor/tunepk.py          |  87 ----
 yt_dlp/extractor/tvnet.py           | 138 ------
 yt_dlp/extractor/tvnow.py           | 639 ----------------------------
 yt_dlp/extractor/twentyfourvideo.py | 128 ------
 yt_dlp/extractor/unscripted.py      |  53 ---
 yt_dlp/extractor/veehd.py           | 116 -----
 yt_dlp/extractor/vice.py            |   6 -
 yt_dlp/extractor/vidbit.py          |  82 ----
 yt_dlp/extractor/vimple.py          |  58 ---
 yt_dlp/extractor/vodlocker.py       |  73 ----
 yt_dlp/extractor/voicerepublic.py   |  59 ---
 yt_dlp/extractor/voot.py            |   2 +
 yt_dlp/extractor/voxmedia.py        |   8 +-
 yt_dlp/extractor/vrak.py            |  77 ----
 yt_dlp/extractor/vrv.py             | 269 ------------
 yt_dlp/extractor/vshare.py          |  57 ---
 yt_dlp/extractor/vupload.py         |  52 ---
 yt_dlp/extractor/vyborymos.py       |  52 ---
 yt_dlp/extractor/vzaar.py           | 100 -----
 yt_dlp/extractor/wakanim.py         |  75 ----
 yt_dlp/extractor/watchbox.py        | 153 -------
 yt_dlp/extractor/watchindianporn.py |  65 ---
 yt_dlp/extractor/willow.py          |  56 ---
 yt_dlp/extractor/xbef.py            |  42 --
 yt_dlp/extractor/xtube.py           | 214 ----------
 yt_dlp/extractor/xuite.py           | 149 -------
 yt_dlp/extractor/yesjapan.py        |  56 ---
 yt_dlp/extractor/yinyuetai.py       |  52 ---
 yt_dlp/extractor/ynet.py            |  48 ---
 125 files changed, 129 insertions(+), 10150 deletions(-)
 delete mode 100644 yt_dlp/extractor/airmozilla.py
 delete mode 100644 yt_dlp/extractor/atttechchannel.py
 delete mode 100644 yt_dlp/extractor/biqle.py
 delete mode 100644 yt_dlp/extractor/bitwave.py
 delete mode 100644 yt_dlp/extractor/breakcom.py
 delete mode 100644 yt_dlp/extractor/camwithher.py
 delete mode 100644 yt_dlp/extractor/carambatv.py
 delete mode 100644 yt_dlp/extractor/channel9.py
 delete mode 100644 yt_dlp/extractor/chirbit.py
 delete mode 100644 yt_dlp/extractor/cinchcast.py
 delete mode 100644 yt_dlp/extractor/clipsyndicate.py
 delete mode 100644 yt_dlp/extractor/cloudy.py
 delete mode 100644 yt_dlp/extractor/daftsex.py
 delete mode 100644 yt_dlp/extractor/defense.py
 delete mode 100644 yt_dlp/extractor/dotsub.py
 delete mode 100644 yt_dlp/extractor/echomsk.py
 delete mode 100644 yt_dlp/extractor/ehow.py
 delete mode 100644 yt_dlp/extractor/elevensports.py
 delete mode 100644 yt_dlp/extractor/ellentube.py
 delete mode 100644 yt_dlp/extractor/engadget.py
 delete mode 100644 yt_dlp/extractor/escapist.py
 delete mode 100644 yt_dlp/extractor/esri.py
 delete mode 100644 yt_dlp/extractor/expotv.py
 delete mode 100644 yt_dlp/extractor/extremetube.py
 delete mode 100644 yt_dlp/extractor/fourzerostudio.py
 delete mode 100644 yt_dlp/extractor/foxgay.py
 delete mode 100644 yt_dlp/extractor/fusion.py
 delete mode 100644 yt_dlp/extractor/gfycat.py
 delete mode 100644 yt_dlp/extractor/helsinki.py
 delete mode 100644 yt_dlp/extractor/hitbox.py
 delete mode 100644 yt_dlp/extractor/howcast.py
 delete mode 100644 yt_dlp/extractor/howstuffworks.py
 delete mode 100644 yt_dlp/extractor/keezmovies.py
 delete mode 100644 yt_dlp/extractor/laola1tv.py
 delete mode 100644 yt_dlp/extractor/linuxacademy.py
 delete mode 100644 yt_dlp/extractor/m6.py
 delete mode 100644 yt_dlp/extractor/meta.py
 delete mode 100644 yt_dlp/extractor/metacafe.py
 delete mode 100644 yt_dlp/extractor/mgoon.py
 delete mode 100644 yt_dlp/extractor/miomio.py
 delete mode 100644 yt_dlp/extractor/mnet.py
 delete mode 100644 yt_dlp/extractor/moevideo.py
 delete mode 100644 yt_dlp/extractor/mofosex.py
 delete mode 100644 yt_dlp/extractor/movieclips.py
 delete mode 100644 yt_dlp/extractor/mwave.py
 delete mode 100644 yt_dlp/extractor/mychannels.py
 delete mode 100644 yt_dlp/extractor/myvi.py
 delete mode 100644 yt_dlp/extractor/newstube.py
 delete mode 100644 yt_dlp/extractor/normalboots.py
 delete mode 100644 yt_dlp/extractor/nosvideo.py
 delete mode 100644 yt_dlp/extractor/ooyala.py
 delete mode 100644 yt_dlp/extractor/pandoratv.py
 delete mode 100644 yt_dlp/extractor/people.py
 delete mode 100644 yt_dlp/extractor/playfm.py
 delete mode 100644 yt_dlp/extractor/plays.py
 delete mode 100644 yt_dlp/extractor/playvid.py
 delete mode 100644 yt_dlp/extractor/porncom.py
 delete mode 100644 yt_dlp/extractor/pornez.py
 delete mode 100644 yt_dlp/extractor/pornhd.py
 delete mode 100644 yt_dlp/extractor/radiobremen.py
 delete mode 100644 yt_dlp/extractor/recurbate.py
 delete mode 100644 yt_dlp/extractor/rice.py
 delete mode 100644 yt_dlp/extractor/rtvnh.py
 delete mode 100644 yt_dlp/extractor/ruhd.py
 delete mode 100644 yt_dlp/extractor/shared.py
 delete mode 100644 yt_dlp/extractor/spankwire.py
 delete mode 100644 yt_dlp/extractor/streamcloud.py
 delete mode 100644 yt_dlp/extractor/swrmediathek.py
 delete mode 100644 yt_dlp/extractor/techtalks.py
 delete mode 100644 yt_dlp/extractor/tinypic.py
 delete mode 100644 yt_dlp/extractor/tokentube.py
 delete mode 100644 yt_dlp/extractor/trilulilu.py
 delete mode 100644 yt_dlp/extractor/tunepk.py
 delete mode 100644 yt_dlp/extractor/tvnet.py
 delete mode 100644 yt_dlp/extractor/tvnow.py
 delete mode 100644 yt_dlp/extractor/twentyfourvideo.py
 delete mode 100644 yt_dlp/extractor/unscripted.py
 delete mode 100644 yt_dlp/extractor/veehd.py
 delete mode 100644 yt_dlp/extractor/vidbit.py
 delete mode 100644 yt_dlp/extractor/vimple.py
 delete mode 100644 yt_dlp/extractor/vodlocker.py
 delete mode 100644 yt_dlp/extractor/voicerepublic.py
 delete mode 100644 yt_dlp/extractor/vrak.py
 delete mode 100644 yt_dlp/extractor/vrv.py
 delete mode 100644 yt_dlp/extractor/vshare.py
 delete mode 100644 yt_dlp/extractor/vupload.py
 delete mode 100644 yt_dlp/extractor/vyborymos.py
 delete mode 100644 yt_dlp/extractor/vzaar.py
 delete mode 100644 yt_dlp/extractor/wakanim.py
 delete mode 100644 yt_dlp/extractor/watchbox.py
 delete mode 100644 yt_dlp/extractor/watchindianporn.py
 delete mode 100644 yt_dlp/extractor/willow.py
 delete mode 100644 yt_dlp/extractor/xbef.py
 delete mode 100644 yt_dlp/extractor/xtube.py
 delete mode 100644 yt_dlp/extractor/xuite.py
 delete mode 100644 yt_dlp/extractor/yesjapan.py
 delete mode 100644 yt_dlp/extractor/yinyuetai.py
 delete mode 100644 yt_dlp/extractor/ynet.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ad8c7d661..9b96bd5b4 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -77,7 +77,6 @@
     WyborczaPodcastIE,
     WyborczaVideoIE,
 )
-from .airmozilla import AirMozillaIE
 from .airtv import AirTVIE
 from .aitube import AitubeKZVideoIE
 from .aljazeera import AlJazeeraIE
@@ -147,7 +146,6 @@
 from .arnes import ArnesIE
 from .atresplayer import AtresPlayerIE
 from .atscaleconf import AtScaleConfEventIE
-from .atttechchannel import ATTTechChannelIE
 from .atvat import ATVAtIE
 from .audimedia import AudiMediaIE
 from .audioboom import AudioBoomIE
@@ -243,11 +241,6 @@
     BitChuteIE,
     BitChuteChannelIE,
 )
-from .bitwave import (
-    BitwaveReplayIE,
-    BitwaveStreamIE,
-)
-from .biqle import BIQLEIE
 from .blackboardcollaborate import BlackboardCollaborateIE
 from .bleacherreport import (
     BleacherReportIE,
@@ -262,10 +255,7 @@
 from .box import BoxIE
 from .boxcast import BoxCastVideoIE
 from .bpb import BpbIE
-from .br import (
-    BRIE,
-    BRMediathekIE,
-)
+from .br import BRIE
 from .bravotv import BravoTVIE
 from .brainpop import (
     BrainPOPIE,
@@ -275,7 +265,6 @@
     BrainPOPFrIE,
     BrainPOPIlIE,
 )
-from .breakcom import BreakIE
 from .breitbart import BreitBartIE
 from .brightcove import (
     BrightcoveLegacyIE,
@@ -305,16 +294,11 @@
 from .cammodels import CamModelsIE
 from .camsoda import CamsodaIE
 from .camtasia import CamtasiaEmbedIE
-from .camwithher import CamWithHerIE
 from .canal1 import Canal1IE
 from .canalalpha import CanalAlphaIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .caracoltv import CaracolTvPlayIE
-from .carambatv import (
-    CarambaTVIE,
-    CarambaTVPageIE,
-)
 from .cartoonnetwork import CartoonNetworkIE
 from .cbc import (
     CBCIE,
@@ -353,7 +337,6 @@
 from .cellebrite import CellebriteIE
 from .ceskatelevize import CeskaTelevizeIE
 from .cgtn import CGTNIE
-from .channel9 import Channel9IE
 from .charlierose import CharlieRoseIE
 from .chaturbate import ChaturbateIE
 from .chilloutzone import ChilloutzoneIE
@@ -361,11 +344,6 @@
     ChingariIE,
     ChingariUserIE,
 )
-from .chirbit import (
-    ChirbitIE,
-    ChirbitProfileIE,
-)
-from .cinchcast import CinchcastIE
 from .cinemax import CinemaxIE
 from .cinetecamilano import CinetecaMilanoIE
 from .cineverse import (
@@ -382,10 +360,8 @@
 from .cliphunter import CliphunterIE
 from .clippit import ClippitIE
 from .cliprs import ClipRsIE
-from .clipsyndicate import ClipsyndicateIE
 from .closertotruth import CloserToTruthIE
 from .cloudflarestream import CloudflareStreamIE
-from .cloudy import CloudyIE
 from .clubic import ClubicIE
 from .clyp import ClypIE
 from .cmt import CMTIE
@@ -452,7 +428,6 @@
     DacastVODIE,
     DacastPlaylistIE,
 )
-from .daftsex import DaftsexIE
 from .dailymail import DailyMailIE
 from .dailymotion import (
     DailymotionIE,
@@ -489,7 +464,6 @@
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .digg import DiggIE
-from .dotsub import DotsubIE
 from .douyutv import (
     DouyuShowIE,
     DouyuTVIE,
@@ -536,7 +510,6 @@
     DubokuPlaylistIE
 )
 from .dumpert import DumpertIE
-from .defense import DefenseGouvFrIE
 from .deuxm import (
     DeuxMIE,
     DeuxMNewsIE
@@ -559,26 +532,17 @@
 from .eagleplatform import EaglePlatformIE, ClipYouEmbedIE
 from .ebaumsworld import EbaumsWorldIE
 from .ebay import EbayIE
-from .echomsk import EchoMskIE
 from .egghead import (
     EggheadCourseIE,
     EggheadLessonIE,
 )
-from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE
 from .eitb import EitbIE
-from .elevensports import ElevenSportsIE
-from .ellentube import (
-    EllenTubeIE,
-    EllenTubeVideoIE,
-    EllenTubePlaylistIE,
-)
 from .elonet import ElonetIE
 from .elpais import ElPaisIE
 from .eltrecetv import ElTreceTVIE
 from .embedly import EmbedlyIE
-from .engadget import EngadgetIE
 from .epicon import (
     EpiconIE,
     EpiconSeriesIE,
@@ -596,7 +560,6 @@
     ERTFlixIE,
     ERTWebtvEmbedIE,
 )
-from .escapist import EscapistIE
 from .espn import (
     ESPNIE,
     WatchESPNIE,
@@ -604,15 +567,12 @@
     FiveThirtyEightIE,
     ESPNCricInfoIE,
 )
-from .esri import EsriVideoIE
 from .ettutv import EttuTvIE
 from .europa import EuropaIE, EuroParlWebstreamIE
 from .europeantour import EuropeanTourIE
 from .eurosport import EurosportIE
 from .euscreen import EUScreenIE
-from .expotv import ExpoTVIE
 from .expressen import ExpressenIE
-from .extremetube import ExtremeTubeIE
 from .eyedotv import EyedoTVIE
 from .facebook import (
     FacebookIE,
@@ -655,16 +615,11 @@
     PornerBrosIE,
     FuxIE,
 )
-from .fourzerostudio import (
-    FourZeroStudioArchiveIE,
-    FourZeroStudioClipIE,
-)
 from .fox import FOXIE
 from .fox9 import (
     FOX9IE,
     FOX9NewsIE,
 )
-from .foxgay import FoxgayIE
 from .foxnews import (
     FoxNewsIE,
     FoxNewsArticleIE,
@@ -697,7 +652,6 @@
 )
 from .funk import FunkIE
 from .funker530 import Funker530IE
-from .fusion import FusionIE
 from .fuyintv import FuyinTVIE
 from .gab import (
     GabTVIE,
@@ -728,7 +682,6 @@
     GettrIE,
     GettrStreamingIE,
 )
-from .gfycat import GfycatIE
 from .giantbomb import GiantBombIE
 from .giga import GigaIE
 from .glide import GlideIE
@@ -774,12 +727,10 @@
 from .hearthisat import HearThisAtIE
 from .heise import HeiseIE
 from .hellporno import HellPornoIE
-from .helsinki import HelsinkiIE
 from .hgtv import HGTVComShowIE
 from .hketv import HKETVIE
 from .hidive import HiDiveIE
 from .historicfilms import HistoricFilmsIE
-from .hitbox import HitboxIE, HitboxLiveIE
 from .hitrecord import HitRecordIE
 from .hollywoodreporter import (
     HollywoodReporterIE,
@@ -794,8 +745,6 @@
     HotStarSeasonIE,
     HotStarSeriesIE,
 )
-from .howcast import HowcastIE
-from .howstuffworks import HowStuffWorksIE
 from .hrefli import HrefLiRedirectIE
 from .hrfensehen import HRFernsehenIE
 from .hrti import (
@@ -927,7 +876,6 @@
 from .kankanews import KankaNewsIE
 from .karaoketv import KaraoketvIE
 from .karrierevideos import KarriereVideosIE
-from .keezmovies import KeezMoviesIE
 from .kelbyone import KelbyOneIE
 from .khanacademy import (
     KhanAcademyIE,
@@ -962,12 +910,6 @@
     LA7PodcastEpisodeIE,
     LA7PodcastIE,
 )
-from .laola1tv import (
-    Laola1TvEmbedIE,
-    Laola1TvIE,
-    EHFTVIE,
-    ITTFIE,
-)
 from .lastfm import (
     LastFMIE,
     LastFMPlaylistIE,
@@ -1022,7 +964,6 @@
     LinkedInLearningIE,
     LinkedInLearningCourseIE,
 )
-from .linuxacademy import LinuxAcademyIE
 from .liputan6 import Liputan6IE
 from .listennotes import ListenNotesIE
 from .litv import LiTVIE
@@ -1050,7 +991,6 @@
     LyndaIE,
     LyndaCourseIE
 )
-from .m6 import M6IE
 from .magellantv import MagellanTVIE
 from .magentamusik360 import MagentaMusik360IE
 from .mailru import (
@@ -1101,10 +1041,7 @@
 from .megaphone import MegaphoneIE
 from .meipai import MeipaiIE
 from .melonvod import MelonVODIE
-from .meta import METAIE
-from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
-from .mgoon import MgoonIE
 from .mgtv import MGTVIE
 from .miaopai import MiaoPaiIE
 from .microsoftstream import MicrosoftStreamIE
@@ -1126,7 +1063,6 @@
 )
 from .ministrygrid import MinistryGridIE
 from .minoto import MinotoIE
-from .miomio import MioMioIE
 from .mirrativ import (
     MirrativIE,
     MirrativUserIE,
@@ -1150,13 +1086,7 @@
     MLBArticleIE,
 )
 from .mlssoccer import MLSSoccerIE
-from .mnet import MnetIE
 from .mocha import MochaVideoIE
-from .moevideo import MoeVideoIE
-from .mofosex import (
-    MofosexIE,
-    MofosexEmbedIE,
-)
 from .mojvideo import MojvideoIE
 from .monstercat import MonstercatIE
 from .morningstar import MorningstarIE
@@ -1166,7 +1096,6 @@
     MotherlessGalleryIE,
 )
 from .motorsport import MotorsportIE
-from .movieclips import MovieClipsIE
 from .moviepilot import MoviepilotIE
 from .moview import MoviewPlayIE
 from .moviezine import MoviezineIE
@@ -1191,18 +1120,12 @@
     MusicdexArtistIE,
     MusicdexPlaylistIE,
 )
-from .mwave import MwaveIE, MwaveMeetGreetIE
 from .mxplayer import (
     MxplayerIE,
     MxplayerShowIE,
 )
-from .mychannels import MyChannelsIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
-from .myvi import (
-    MyviIE,
-    MyviEmbedIE,
-)
 from .myvideoge import MyVideoGeIE
 from .myvidster import MyVidsterIE
 from .mzaalo import MzaaloIE
@@ -1278,7 +1201,6 @@
     NewgroundsUserIE,
 )
 from .newspicks import NewsPicksIE
-from .newstube import NewstubeIE
 from .newsy import NewsyIE
 from .nextmedia import (
     NextMediaIE,
@@ -1313,7 +1235,6 @@
     NickIE,
     NickBrIE,
     NickDeIE,
-    NickNightIE,
     NickRuIE,
 )
 from .niconico import (
@@ -1346,8 +1267,6 @@
 from .nonktube import NonkTubeIE
 from .noodlemagazine import NoodleMagazineIE
 from .noovo import NoovoIE
-from .normalboots import NormalbootsIE
-from .nosvideo import NosVideoIE
 from .nosnl import NOSNLArticleIE
 from .nova import (
     NovaEmbedIE,
@@ -1422,10 +1341,6 @@
     OnetPlIE,
 )
 from .onionstudios import OnionStudiosIE
-from .ooyala import (
-    OoyalaIE,
-    OoyalaExternalIE,
-)
 from .opencast import (
     OpencastIE,
     OpencastPlaylistIE,
@@ -1454,7 +1369,6 @@
     PalcoMP3ArtistIE,
     PalcoMP3VideoIE,
 )
-from .pandoratv import PandoraTVIE
 from .panopto import (
     PanoptoIE,
     PanoptoListIE,
@@ -1482,7 +1396,6 @@
     PelotonIE,
     PelotonLiveIE
 )
-from .people import PeopleIE
 from .performgroup import PerformGroupIE
 from .periscope import (
     PeriscopeIE,
@@ -1514,13 +1427,10 @@
     PlatziIE,
     PlatziCourseIE,
 )
-from .playfm import PlayFMIE
 from .playplustv import PlayPlusTVIE
-from .plays import PlaysTVIE
 from .playstuff import PlayStuffIE
 from .playsuisse import PlaySuisseIE
 from .playtvak import PlaytvakIE
-from .playvid import PlayvidIE
 from .playwire import PlaywireIE
 from .plutotv import PlutoTVIE
 from .pluralsight import (
@@ -1552,9 +1462,7 @@
 from .popcorntv import PopcornTVIE
 from .porn91 import Porn91IE
 from .pornbox import PornboxIE
-from .porncom import PornComIE
 from .pornflip import PornFlipIE
-from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,
     PornHubUserIE,
@@ -1565,7 +1473,6 @@
 from .pornotube import PornotubeIE
 from .pornovoisines import PornoVoisinesIE
 from .pornoxo import PornoXOIE
-from .pornez import PornezIE
 from .puhutv import (
     PuhuTVIE,
     PuhuTVSerieIE,
@@ -1609,7 +1516,6 @@
 )
 from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
-from .radiobremen import RadioBremenIE
 from .radiofrance import (
     FranceCultureIE,
     RadioFranceIE,
@@ -1661,7 +1567,6 @@
     RCTIPlusTVIE,
 )
 from .rds import RDSIE
-from .recurbate import RecurbateIE
 from .redbee import ParliamentLiveUKIE, RTBFIE
 from .redbulltv import (
     RedBullTVIE,
@@ -1685,7 +1590,6 @@
 from .reuters import ReutersIE
 from .reverbnation import ReverbNationIE
 from .rheinmaintv import RheinMainTVIE
-from .rice import RICEIE
 from .rmcdecouverte import RMCDecouverteIE
 from .rockstargames import RockstarGamesIE
 from .rokfin import (
@@ -1709,11 +1613,7 @@
     RTLLuLiveIE,
     RTLLuRadioIE,
 )
-from .rtl2 import (
-    RTL2IE,
-    RTL2YouIE,
-    RTL2YouSeriesIE,
-)
+from .rtl2 import RTL2IE
 from .rtnews import (
     RTNewsIE,
     RTDocumentryIE,
@@ -1735,10 +1635,8 @@
     RTVEInfantilIE,
     RTVETelevisionIE,
 )
-from .rtvnh import RTVNHIE
 from .rtvs import RTVSIE
 from .rtvslo import RTVSLOIE
-from .ruhd import RUHDIE
 from .rule34video import Rule34VideoIE
 from .rumble import (
     RumbleEmbedIE,
@@ -1820,10 +1718,6 @@
     ShahidIE,
     ShahidShowIE,
 )
-from .shared import (
-    SharedIE,
-    VivoIE,
-)
 from .sharevideos import ShareVideosEmbedIE
 from .sibnet import SibnetEmbedIE
 from .shemaroome import ShemarooMeIE
@@ -1901,7 +1795,6 @@
     SpankBangIE,
     SpankBangPlaylistIE,
 )
-from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE
 from .spike import (
     BellatorIE,
@@ -1951,7 +1844,6 @@
     StoryFireSeriesIE,
 )
 from .streamable import StreamableIE
-from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streamff import StreamFFIE
 from .streetvoice import StreetVoiceIE
@@ -1971,7 +1863,6 @@
     SVTSeriesIE,
 )
 from .swearnet import SwearnetEpisodeIE
-from .swrmediathek import SWRMediathekIE
 from .syvdk import SYVDKIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
@@ -1998,7 +1889,6 @@
     ConanClassicIE,
 )
 from .teamtreehouse import TeamTreeHouseIE
-from .techtalks import TechTalksIE
 from .ted import (
     TedEmbedIE,
     TedPlaylistIE,
@@ -2075,7 +1965,6 @@
     TikTokLiveIE,
     DouyinIE,
 )
-from .tinypic import TinyPicIE
 from .tmz import TMZIE
 from .tnaflix import (
     TNAFlixNetworkEmbedIE,
@@ -2090,10 +1979,6 @@
 from .toggo import (
     ToggoIE,
 )
-from .tokentube import (
-    TokentubeIE,
-    TokentubeChannelIE
-)
 from .tonline import TOnlineIE
 from .toongoggles import ToonGogglesIE
 from .toutv import TouTvIE
@@ -2104,7 +1989,6 @@
     TrillerUserIE,
     TrillerShortIE,
 )
-from .trilulilu import TriluliluIE
 from .trovo import (
     TrovoIE,
     TrovoVodIE,
@@ -2129,7 +2013,6 @@
     TuneInPodcastEpisodeIE,
     TuneInShortenerIE,
 )
-from .tunepk import TunePkIE
 from .turbo import TurboIE
 from .tv2 import (
     TV2IE,
@@ -2171,16 +2054,7 @@
 from .tviplayer import TVIPlayerIE
 from .tvland import TVLandIE
 from .tvn24 import TVN24IE
-from .tvnet import TVNetIE
 from .tvnoe import TVNoeIE
-from .tvnow import (
-    TVNowIE,
-    TVNowFilmIE,
-    TVNowNewIE,
-    TVNowSeasonIE,
-    TVNowAnnualIE,
-    TVNowShowIE,
-)
 from .tvopengr import (
     TVOpenGrWatchIE,
     TVOpenGrEmbedIE,
@@ -2198,7 +2072,6 @@
 )
 from .tvplayer import TVPlayerIE
 from .tweakers import TweakersIE
-from .twentyfourvideo import TwentyFourVideoIE
 from .twentymin import TwentyMinutenIE
 from .twentythreevideo import TwentyThreeVideoIE
 from .twitcasting import (
@@ -2247,7 +2120,6 @@
 from .umg import UMGDeIE
 from .unistra import UnistraIE
 from .unity import UnityIE
-from .unscripted import UnscriptedNewsVideoIE
 from .unsupported import KnownDRMIE, KnownPiracyIE
 from .uol import UOLIE
 from .uplynk import (
@@ -2266,7 +2138,6 @@
 from .utreon import UtreonIE
 from .varzesh3 import Varzesh3IE
 from .vbox7 import Vbox7IE
-from .veehd import VeeHDIE
 from .veo import VeoIE
 from .veoh import (
     VeohIE,
@@ -2288,7 +2159,6 @@
     ViceArticleIE,
     ViceShowIE,
 )
-from .vidbit import VidbitIE
 from .viddler import ViddlerIE
 from .videa import VideaIE
 from .videocampus_sachsen import (
@@ -2339,7 +2209,6 @@
     VimmIE,
     VimmRecordingIE,
 )
-from .vimple import VimpleIE
 from .vine import (
     VineIE,
     VineUserIE,
@@ -2363,10 +2232,8 @@
     VKPlayLiveIE,
 )
 from .vocaroo import VocarooIE
-from .vodlocker import VodlockerIE
 from .vodpl import VODPlIE
 from .vodplatform import VODPlatformIE
-from .voicerepublic import VoiceRepublicIE
 from .voicy import (
     VoicyIE,
     VoicyChannelIE,
@@ -2386,23 +2253,13 @@
     KetnetIE,
     DagelijkseKostIE,
 )
-from .vrak import VrakIE
-from .vrv import (
-    VRVIE,
-    VRVSeriesIE,
-)
-from .vshare import VShareIE
 from .vtm import VTMIE
 from .medialaan import MedialaanIE
 from .vuclip import VuClipIE
-from .vupload import VuploadIE
 from .vvvvid import (
     VVVVIDIE,
     VVVVIDShowIE,
 )
-from .vyborymos import VyboryMosIE
-from .vzaar import VzaarIE
-from .wakanim import WakanimIE
 from .walla import WallaIE
 from .washingtonpost import (
     WashingtonPostIE,
@@ -2414,8 +2271,6 @@
     WASDTVClipIE,
 )
 from .wat import WatIE
-from .watchbox import WatchBoxIE
-from .watchindianporn import WatchIndianPornIE
 from .wdr import (
     WDRIE,
     WDRPageIE,
@@ -2449,7 +2304,6 @@
 from .weyyak import WeyyakIE
 from .whyp import WhypIE
 from .wikimedia import WikimediaIE
-from .willow import WillowIE
 from .wimbledon import WimbledonIE
 from .wimtv import WimTVIE
 from .whowatch import WhoWatchIE
@@ -2483,7 +2337,6 @@
     WykopPostCommentIE,
 )
 from .xanimu import XanimuIE
-from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
 from .xfileshare import XFileShareIE
 from .xhamster import (
@@ -2499,8 +2352,6 @@
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE
-from .xtube import XTubeUserIE, XTubeIE
-from .xuite import XuiteIE
 from .xvideos import (
     XVideosIE,
     XVideosQuickiesIE
@@ -2530,10 +2381,7 @@
     YappyIE,
     YappyProfileIE,
 )
-from .yesjapan import YesJapanIE
-from .yinyuetai import YinYueTaiIE
 from .yle_areena import YleAreenaIE
-from .ynet import YnetIE
 from .youjizz import YouJizzIE
 from .youku import (
     YoukuIE,
diff --git a/yt_dlp/extractor/airmozilla.py b/yt_dlp/extractor/airmozilla.py
deleted file mode 100644
index 669556b98..000000000
--- a/yt_dlp/extractor/airmozilla.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_duration,
-    parse_iso8601,
-)
-
-
-class AirMozillaIE(InfoExtractor):
-    _VALID_URL = r'https?://air\.mozilla\.org/(?P<id>[0-9a-z-]+)/?'
-    _TEST = {
-        'url': 'https://air.mozilla.org/privacy-lab-a-meetup-for-privacy-minded-people-in-san-francisco/',
-        'md5': '8d02f53ee39cf006009180e21df1f3ba',
-        'info_dict': {
-            'id': '6x4q2w',
-            'ext': 'mp4',
-            'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
-            'thumbnail': r're:https?://.*/poster\.jpg',
-            'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
-            'timestamp': 1422487800,
-            'upload_date': '20150128',
-            'location': 'SFO Commons',
-            'duration': 3780,
-            'view_count': int,
-            'categories': ['Main', 'Privacy'],
-        }
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        video_id = self._html_search_regex(r'//vid\.ly/(.*?)/embed', webpage, 'id')
-
-        embed_script = self._download_webpage('https://vid.ly/{0}/embed'.format(video_id), video_id)
-        jwconfig = self._parse_json(self._search_regex(
-            r'initCallback\((.*)\);', embed_script, 'metadata'), video_id)['config']
-
-        info_dict = self._parse_jwplayer_data(jwconfig, video_id)
-        view_count = int_or_none(self._html_search_regex(
-            r'Views since archived: ([0-9]+)',
-            webpage, 'view count', fatal=False))
-        timestamp = parse_iso8601(self._html_search_regex(
-            r'<time datetime="(.*?)"', webpage, 'timestamp', fatal=False))
-        duration = parse_duration(self._search_regex(
-            r'Duration:\s*(\d+\s*hours?\s*\d+\s*minutes?)',
-            webpage, 'duration', fatal=False))
-
-        info_dict.update({
-            'id': video_id,
-            'title': self._og_search_title(webpage),
-            'url': self._og_search_url(webpage),
-            'display_id': display_id,
-            'description': self._og_search_description(webpage),
-            'timestamp': timestamp,
-            'location': self._html_search_regex(r'Location: (.*)', webpage, 'location', default=None),
-            'duration': duration,
-            'view_count': view_count,
-            'categories': re.findall(r'<a href=".*?" class="channel">(.*?)</a>', webpage),
-        })
-
-        return info_dict
diff --git a/yt_dlp/extractor/aol.py b/yt_dlp/extractor/aol.py
index 6949ca974..455f66795 100644
--- a/yt_dlp/extractor/aol.py
+++ b/yt_dlp/extractor/aol.py
@@ -10,6 +10,7 @@
 
 
 class AolIE(YahooIE):  # XXX: Do not subclass from concrete IE
+    _WORKING = False
     IE_NAME = 'aol.com'
     _VALID_URL = r'(?:aol-video:|https?://(?:www\.)?aol\.(?:com|ca|co\.uk|de|jp)/video/(?:[^/]+/)*)(?P<id>\d{9}|[0-9a-f]{24}|[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})'
 
diff --git a/yt_dlp/extractor/atttechchannel.py b/yt_dlp/extractor/atttechchannel.py
deleted file mode 100644
index 6ff4ec0ad..000000000
--- a/yt_dlp/extractor/atttechchannel.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from .common import InfoExtractor
-from ..utils import unified_strdate
-
-
-class ATTTechChannelIE(InfoExtractor):
-    _VALID_URL = r'https?://techchannel\.att\.com/play-video\.cfm/([^/]+/)*(?P<id>.+)'
-    _TEST = {
-        'url': 'http://techchannel.att.com/play-video.cfm/2014/1/27/ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
-        'info_dict': {
-            'id': '11316',
-            'display_id': 'ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
-            'ext': 'flv',
-            'title': 'AT&T Archives : The UNIX System: Making Computers Easier to Use',
-            'description': 'A 1982 film about UNIX is the foundation for software in use around Bell Labs and AT&T.',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'upload_date': '20140127',
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        video_url = self._search_regex(
-            r"url\s*:\s*'(rtmp://[^']+)'",
-            webpage, 'video URL')
-
-        video_id = self._search_regex(
-            r'mediaid\s*=\s*(\d+)',
-            webpage, 'video id', fatal=False)
-
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-        upload_date = unified_strdate(self._search_regex(
-            r'[Rr]elease\s+date:\s*(\d{1,2}/\d{1,2}/\d{4})',
-            webpage, 'upload date', fatal=False), False)
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'url': video_url,
-            'ext': 'flv',
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-        }
diff --git a/yt_dlp/extractor/behindkink.py b/yt_dlp/extractor/behindkink.py
index ca4498150..9d2324f4f 100644
--- a/yt_dlp/extractor/behindkink.py
+++ b/yt_dlp/extractor/behindkink.py
@@ -3,6 +3,7 @@
 
 
 class BehindKinkIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'https?://(?:www\.)?behindkink\.com/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<id>[^/#?_]+)'
     _TEST = {
         'url': 'http://www.behindkink.com/2014/12/05/what-are-you-passionate-about-marley-blaze/',
diff --git a/yt_dlp/extractor/bet.py b/yt_dlp/extractor/bet.py
index 6b867d135..cbf3dd082 100644
--- a/yt_dlp/extractor/bet.py
+++ b/yt_dlp/extractor/bet.py
@@ -1,10 +1,9 @@
 from .mtv import MTVServicesInfoExtractor
 from ..utils import unified_strdate
 
-# TODO Remove - Reason: Outdated Site
-
 
 class BetIE(MTVServicesInfoExtractor):
+    _WORKING = False
     _VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'
     _TESTS = [
         {
diff --git a/yt_dlp/extractor/bfi.py b/yt_dlp/extractor/bfi.py
index 76f0516a4..a6ebfedff 100644
--- a/yt_dlp/extractor/bfi.py
+++ b/yt_dlp/extractor/bfi.py
@@ -5,6 +5,7 @@
 
 
 class BFIPlayerIE(InfoExtractor):
+    _WORKING = False
     IE_NAME = 'bfi:player'
     _VALID_URL = r'https?://player\.bfi\.org\.uk/[^/]+/film/watch-(?P<id>[\w-]+)-online'
     _TEST = {
diff --git a/yt_dlp/extractor/biqle.py b/yt_dlp/extractor/biqle.py
deleted file mode 100644
index 027753503..000000000
--- a/yt_dlp/extractor/biqle.py
+++ /dev/null
@@ -1,110 +0,0 @@
-from .common import InfoExtractor
-from .vk import VKIE
-from ..compat import compat_b64decode
-from ..utils import (
-    int_or_none,
-    js_to_json,
-    traverse_obj,
-    unified_timestamp,
-)
-
-
-class BIQLEIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?biqle\.(?:com|org|ru)/watch/(?P<id>-?\d+_\d+)'
-    _TESTS = [{
-        'url': 'https://biqle.ru/watch/-2000421746_85421746',
-        'md5': 'ae6ef4f04d19ac84e4658046d02c151c',
-        'info_dict': {
-            'id': '-2000421746_85421746',
-            'ext': 'mp4',
-            'title': 'Forsaken By Hope Studio Clip',
-            'description': 'Forsaken By Hope Studio Clip — Смотреть онлайн',
-            'upload_date': '19700101',
-            'thumbnail': r're:https://[^/]+/impf/7vN3ACwSTgChP96OdOfzFjUCzFR6ZglDQgWsIw/KPaACiVJJxM\.jpg\?size=800x450&quality=96&keep_aspect_ratio=1&background=000000&sign=b48ea459c4d33dbcba5e26d63574b1cb&type=video_thumb',
-            'timestamp': 0,
-        },
-    }, {
-        'url': 'http://biqle.org/watch/-44781847_168547604',
-        'md5': '7f24e72af1db0edf7c1aaba513174f97',
-        'info_dict': {
-            'id': '-44781847_168547604',
-            'ext': 'mp4',
-            'title': 'Ребенок в шоке от автоматической мойки',
-            'description': 'Ребенок в шоке от автоматической мойки — Смотреть онлайн',
-            'timestamp': 1396633454,
-            'upload_date': '20140404',
-            'thumbnail': r're:https://[^/]+/c535507/u190034692/video/l_b84df002\.jpg',
-        },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_meta('name', webpage, 'Title', fatal=False)
-        timestamp = unified_timestamp(self._html_search_meta('uploadDate', webpage, 'Upload Date', default=None))
-        description = self._html_search_meta('description', webpage, 'Description', default=None)
-
-        global_embed_url = self._search_regex(
-            r'<script[^<]+?window.globEmbedUrl\s*=\s*\'((?:https?:)?//(?:daxab\.com|dxb\.to|[^/]+/player)/[^\']+)\'',
-            webpage, 'global Embed url')
-        hash = self._search_regex(
-            r'<script id="data-embed-video[^<]+?hash: "([^"]+)"[^<]*</script>', webpage, 'Hash')
-
-        embed_url = global_embed_url + hash
-
-        if VKIE.suitable(embed_url):
-            return self.url_result(embed_url, VKIE.ie_key(), video_id)
-
-        embed_page = self._download_webpage(
-            embed_url, video_id, 'Downloading embed webpage', headers={'Referer': url})
-
-        glob_params = self._parse_json(self._search_regex(
-            r'<script id="globParams">[^<]*window.globParams = ([^;]+);[^<]+</script>',
-            embed_page, 'Global Parameters'), video_id, transform_source=js_to_json)
-        host_name = compat_b64decode(glob_params['server'][::-1]).decode()
-
-        item = self._download_json(
-            f'https://{host_name}/method/video.get/{video_id}', video_id,
-            headers={'Referer': url}, query={
-                'token': glob_params['video']['access_token'],
-                'videos': video_id,
-                'ckey': glob_params['c_key'],
-                'credentials': glob_params['video']['credentials'],
-            })['response']['items'][0]
-
-        formats = []
-        for f_id, f_url in item.get('files', {}).items():
-            if f_id == 'external':
-                return self.url_result(f_url)
-            ext, height = f_id.split('_')
-            height_extra_key = traverse_obj(glob_params, ('video', 'partial', 'quality', height))
-            if height_extra_key:
-                formats.append({
-                    'format_id': f'{height}p',
-                    'url': f'https://{host_name}/{f_url[8:]}&videos={video_id}&extra_key={height_extra_key}',
-                    'height': int_or_none(height),
-                    'ext': ext,
-                })
-
-        thumbnails = []
-        for k, v in item.items():
-            if k.startswith('photo_') and v:
-                width = k.replace('photo_', '')
-                thumbnails.append({
-                    'id': width,
-                    'url': v,
-                    'width': int_or_none(width),
-                })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'comment_count': int_or_none(item.get('comments')),
-            'description': description,
-            'duration': int_or_none(item.get('duration')),
-            'thumbnails': thumbnails,
-            'timestamp': timestamp,
-            'view_count': int_or_none(item.get('views')),
-        }
diff --git a/yt_dlp/extractor/bitwave.py b/yt_dlp/extractor/bitwave.py
deleted file mode 100644
index a82cd263a..000000000
--- a/yt_dlp/extractor/bitwave.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from .common import InfoExtractor
-
-
-class BitwaveReplayIE(InfoExtractor):
-    IE_NAME = 'bitwave:replay'
-    _VALID_URL = r'https?://(?:www\.)?bitwave\.tv/(?P<user>\w+)/replay/(?P<id>\w+)/?$'
-    _TEST = {
-        'url': 'https://bitwave.tv/RhythmicCarnage/replay/z4P6eq5L7WDrM85UCrVr',
-        'only_matching': True
-    }
-
-    def _real_extract(self, url):
-        replay_id = self._match_id(url)
-        replay = self._download_json(
-            'https://api.bitwave.tv/v1/replays/' + replay_id,
-            replay_id
-        )
-
-        return {
-            'id': replay_id,
-            'title': replay['data']['title'],
-            'uploader': replay['data']['name'],
-            'uploader_id': replay['data']['name'],
-            'url': replay['data']['url'],
-            'thumbnails': [
-                {'url': x} for x in replay['data']['thumbnails']
-            ],
-        }
-
-
-class BitwaveStreamIE(InfoExtractor):
-    IE_NAME = 'bitwave:stream'
-    _VALID_URL = r'https?://(?:www\.)?bitwave\.tv/(?P<id>\w+)/?$'
-    _TEST = {
-        'url': 'https://bitwave.tv/doomtube',
-        'only_matching': True
-    }
-
-    def _real_extract(self, url):
-        username = self._match_id(url)
-        channel = self._download_json(
-            'https://api.bitwave.tv/v1/channels/' + username,
-            username)
-
-        formats = self._extract_m3u8_formats(
-            channel['data']['url'], username,
-            'mp4')
-
-        return {
-            'id': username,
-            'title': channel['data']['title'],
-            'uploader': username,
-            'uploader_id': username,
-            'formats': formats,
-            'thumbnail': channel['data']['thumbnail'],
-            'is_live': True,
-            'view_count': channel['data']['viewCount']
-        }
diff --git a/yt_dlp/extractor/bleacherreport.py b/yt_dlp/extractor/bleacherreport.py
index 8d8fabe33..5e5155af2 100644
--- a/yt_dlp/extractor/bleacherreport.py
+++ b/yt_dlp/extractor/bleacherreport.py
@@ -22,7 +22,7 @@ class BleacherReportIE(InfoExtractor):
             'upload_date': '20150615',
             'uploader': 'Team Stream Now ',
         },
-        'add_ie': ['Ooyala'],
+        'skip': 'Video removed',
     }, {
         'url': 'http://bleacherreport.com/articles/2586817-aussie-golfers-get-fright-of-their-lives-after-being-chased-by-angry-kangaroo',
         'md5': '6a5cd403418c7b01719248ca97fb0692',
@@ -70,8 +70,6 @@ def _real_extract(self, url):
             video_type = video['type']
             if video_type in ('cms.bleacherreport.com', 'vid.bleacherreport.com'):
                 info['url'] = 'http://bleacherreport.com/video_embed?id=%s' % video['id']
-            elif video_type == 'ooyala.com':
-                info['url'] = 'ooyala:%s' % video['id']
             elif video_type == 'youtube.com':
                 info['url'] = video['id']
             elif video_type == 'vine.co':
diff --git a/yt_dlp/extractor/br.py b/yt_dlp/extractor/br.py
index 309452d23..6e1c63e2b 100644
--- a/yt_dlp/extractor/br.py
+++ b/yt_dlp/extractor/br.py
@@ -1,18 +1,15 @@
-import json
-
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     ExtractorError,
     int_or_none,
     parse_duration,
-    parse_iso8601,
     xpath_element,
     xpath_text,
 )
 
 
 class BRIE(InfoExtractor):
+    _WORKING = False
     IE_DESC = 'Bayerischer Rundfunk'
     _VALID_URL = r'(?P<base_url>https?://(?:www\.)?br(?:-klassik)?\.de)/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'
 
@@ -167,142 +164,3 @@ def _extract_thumbnails(self, variants, base_url):
         } for variant in variants.findall('variant') if xpath_text(variant, 'url')]
         thumbnails.sort(key=lambda x: x['width'] * x['height'], reverse=True)
         return thumbnails
-
-
-class BRMediathekIE(InfoExtractor):
-    IE_DESC = 'Bayerischer Rundfunk Mediathek'
-    _VALID_URL = r'https?://(?:www\.)?br\.de/mediathek//?video/(?:[^/?&#]+?-)?(?P<id>av:[0-9a-f]{24})'
-
-    _TESTS = [{
-        'url': 'https://www.br.de/mediathek/video/gesundheit-die-sendung-vom-28112017-av:5a1e6a6e8fce6d001871cc8e',
-        'md5': 'fdc3d485835966d1622587d08ba632ec',
-        'info_dict': {
-            'id': 'av:5a1e6a6e8fce6d001871cc8e',
-            'ext': 'mp4',
-            'title': 'Die Sendung vom 28.11.2017',
-            'description': 'md5:6000cdca5912ab2277e5b7339f201ccc',
-            'timestamp': 1511942766,
-            'upload_date': '20171129',
-        }
-    }, {
-        'url': 'https://www.br.de/mediathek//video/av:61b0db581aed360007558c12',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        clip_id = self._match_id(url)
-
-        clip = self._download_json(
-            'https://proxy-base.master.mango.express/graphql',
-            clip_id, data=json.dumps({
-                "query": """{
-  viewer {
-    clip(id: "%s") {
-      title
-      description
-      duration
-      createdAt
-      ageRestriction
-      videoFiles {
-        edges {
-          node {
-            publicLocation
-            fileSize
-            videoProfile {
-              width
-              height
-              bitrate
-              encoding
-            }
-          }
-        }
-      }
-      captionFiles {
-        edges {
-          node {
-            publicLocation
-          }
-        }
-      }
-      teaserImages {
-        edges {
-          node {
-            imageFiles {
-              edges {
-                node {
-                  publicLocation
-                  width
-                  height
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}""" % clip_id}).encode(), headers={
-                'Content-Type': 'application/json',
-            })['data']['viewer']['clip']
-        title = clip['title']
-
-        formats = []
-        for edge in clip.get('videoFiles', {}).get('edges', []):
-            node = edge.get('node', {})
-            n_url = node.get('publicLocation')
-            if not n_url:
-                continue
-            ext = determine_ext(n_url)
-            if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    n_url, clip_id, 'mp4', 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-            else:
-                video_profile = node.get('videoProfile', {})
-                tbr = int_or_none(video_profile.get('bitrate'))
-                format_id = 'http'
-                if tbr:
-                    format_id += '-%d' % tbr
-                formats.append({
-                    'format_id': format_id,
-                    'url': n_url,
-                    'width': int_or_none(video_profile.get('width')),
-                    'height': int_or_none(video_profile.get('height')),
-                    'tbr': tbr,
-                    'filesize': int_or_none(node.get('fileSize')),
-                })
-
-        subtitles = {}
-        for edge in clip.get('captionFiles', {}).get('edges', []):
-            node = edge.get('node', {})
-            n_url = node.get('publicLocation')
-            if not n_url:
-                continue
-            subtitles.setdefault('de', []).append({
-                'url': n_url,
-            })
-
-        thumbnails = []
-        for edge in clip.get('teaserImages', {}).get('edges', []):
-            for image_edge in edge.get('node', {}).get('imageFiles', {}).get('edges', []):
-                node = image_edge.get('node', {})
-                n_url = node.get('publicLocation')
-                if not n_url:
-                    continue
-                thumbnails.append({
-                    'url': n_url,
-                    'width': int_or_none(node.get('width')),
-                    'height': int_or_none(node.get('height')),
-                })
-
-        return {
-            'id': clip_id,
-            'title': title,
-            'description': clip.get('description'),
-            'duration': int_or_none(clip.get('duration')),
-            'timestamp': parse_iso8601(clip.get('createdAt')),
-            'age_limit': int_or_none(clip.get('ageRestriction')),
-            'formats': formats,
-            'subtitles': subtitles,
-            'thumbnails': thumbnails,
-        }
diff --git a/yt_dlp/extractor/breakcom.py b/yt_dlp/extractor/breakcom.py
deleted file mode 100644
index 00cf308c7..000000000
--- a/yt_dlp/extractor/breakcom.py
+++ /dev/null
@@ -1,86 +0,0 @@
-from .common import InfoExtractor
-from .youtube import YoutubeIE
-from ..utils import (
-    int_or_none,
-    url_or_none,
-)
-
-
-class BreakIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?break\.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
-    _TESTS = [{
-        'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
-        'info_dict': {
-            'id': '2468056',
-            'ext': 'mp4',
-            'title': 'When Girls Act Like D-Bags',
-            'age_limit': 13,
-        },
-    }, {
-        # youtube embed
-        'url': 'http://www.break.com/video/someone-forgot-boat-brakes-work',
-        'info_dict': {
-            'id': 'RrrDLdeL2HQ',
-            'ext': 'mp4',
-            'title': 'Whale Watching Boat Crashing Into San Diego Dock',
-            'description': 'md5:afc1b2772f0a8468be51dd80eb021069',
-            'upload_date': '20160331',
-            'uploader': 'Steve Holden',
-            'uploader_id': 'sdholden07',
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://www.break.com/video/ugc/baby-flex-2773063',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id, video_id = self._match_valid_url(url).groups()
-
-        webpage = self._download_webpage(url, display_id)
-
-        youtube_url = YoutubeIE._extract_url(webpage)
-        if youtube_url:
-            return self.url_result(youtube_url, ie=YoutubeIE.ie_key())
-
-        content = self._parse_json(
-            self._search_regex(
-                r'(?s)content["\']\s*:\s*(\[.+?\])\s*[,\n]', webpage,
-                'content'),
-            display_id)
-
-        formats = []
-        for video in content:
-            video_url = url_or_none(video.get('url'))
-            if not video_url:
-                continue
-            bitrate = int_or_none(self._search_regex(
-                r'(\d+)_kbps', video_url, 'tbr', default=None))
-            formats.append({
-                'url': video_url,
-                'format_id': 'http-%d' % bitrate if bitrate else 'http',
-                'tbr': bitrate,
-            })
-
-        title = self._search_regex(
-            (r'title["\']\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
-             r'<h1[^>]*>(?P<value>[^<]+)'), webpage, 'title', group='value')
-
-        def get(key, name):
-            return int_or_none(self._search_regex(
-                r'%s["\']\s*:\s*["\'](\d+)' % key, webpage, name,
-                default=None))
-
-        age_limit = get('ratings', 'age limit')
-        video_id = video_id or get('pid', 'video id') or display_id
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'age_limit': age_limit,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/byutv.py b/yt_dlp/extractor/byutv.py
index 9ed6efe79..ad35427ed 100644
--- a/yt_dlp/extractor/byutv.py
+++ b/yt_dlp/extractor/byutv.py
@@ -8,9 +8,9 @@
 
 
 class BYUtvIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'https?://(?:www\.)?byutv\.org/(?:watch|player)/(?!event/)(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
     _TESTS = [{
-        # ooyalaVOD
         'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
         'info_dict': {
             'id': 'ZvanRocTpW-G5_yZFeltTAMv6jxOU9KH',
@@ -24,7 +24,6 @@ class BYUtvIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
-        'add_ie': ['Ooyala'],
     }, {
         # dvr
         'url': 'https://www.byutv.org/player/8f1dab9b-b243-47c8-b525-3e2d021a3451/byu-softball-pacific-vs-byu-41219---game-2',
@@ -63,19 +62,6 @@ def _real_extract(self, url):
                 'x-byutv-platformkey': 'xsaaw9c7y5',
             })
 
-        ep = video.get('ooyalaVOD')
-        if ep:
-            return {
-                '_type': 'url_transparent',
-                'ie_key': 'Ooyala',
-                'url': 'ooyala:%s' % ep['providerId'],
-                'id': video_id,
-                'display_id': display_id,
-                'title': ep.get('title'),
-                'description': ep.get('description'),
-                'thumbnail': ep.get('imageThumbnail'),
-            }
-
         info = {}
         formats = []
         subtitles = {}
diff --git a/yt_dlp/extractor/camwithher.py b/yt_dlp/extractor/camwithher.py
deleted file mode 100644
index a0b3749ed..000000000
--- a/yt_dlp/extractor/camwithher.py
+++ /dev/null
@@ -1,87 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_duration,
-    unified_strdate,
-)
-
-
-class CamWithHerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?camwithher\.tv/view_video\.php\?.*\bviewkey=(?P<id>\w+)'
-
-    _TESTS = [{
-        'url': 'http://camwithher.tv/view_video.php?viewkey=6e9a24e2c0e842e1f177&page=&viewtype=&category=',
-        'info_dict': {
-            'id': '5644',
-            'ext': 'flv',
-            'title': 'Periscope Tease',
-            'description': 'In the clouds teasing on periscope to my favorite song',
-            'duration': 240,
-            'view_count': int,
-            'comment_count': int,
-            'uploader': 'MileenaK',
-            'upload_date': '20160322',
-            'age_limit': 18,
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://camwithher.tv/view_video.php?viewkey=6dfd8b7c97531a459937',
-        'only_matching': True,
-    }, {
-        'url': 'http://camwithher.tv/view_video.php?page=&viewkey=6e9a24e2c0e842e1f177&viewtype=&category=',
-        'only_matching': True,
-    }, {
-        'url': 'http://camwithher.tv/view_video.php?viewkey=b6c3b5bea9515d1a1fc4&page=&viewtype=&category=mv',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        flv_id = self._html_search_regex(
-            r'<a[^>]+href=["\']/download/\?v=(\d+)', webpage, 'video id')
-
-        # Video URL construction algorithm is reverse-engineered from cwhplayer.swf
-        rtmp_url = 'rtmp://camwithher.tv/clipshare/%s' % (
-            ('mp4:%s.mp4' % flv_id) if int(flv_id) > 2010 else flv_id)
-
-        title = self._html_search_regex(
-            r'<div[^>]+style="float:left"[^>]*>\s*<h2>(.+?)</h2>', webpage, 'title')
-        description = self._html_search_regex(
-            r'>Description:</span>(.+?)</div>', webpage, 'description', default=None)
-
-        runtime = self._search_regex(
-            r'Runtime\s*:\s*(.+?) \|', webpage, 'duration', default=None)
-        if runtime:
-            runtime = re.sub(r'[\s-]', '', runtime)
-        duration = parse_duration(runtime)
-        view_count = int_or_none(self._search_regex(
-            r'Views\s*:\s*(\d+)', webpage, 'view count', default=None))
-        comment_count = int_or_none(self._search_regex(
-            r'Comments\s*:\s*(\d+)', webpage, 'comment count', default=None))
-
-        uploader = self._search_regex(
-            r'Added by\s*:\s*<a[^>]+>([^<]+)</a>', webpage, 'uploader', default=None)
-        upload_date = unified_strdate(self._search_regex(
-            r'Added on\s*:\s*([\d-]+)', webpage, 'upload date', default=None))
-
-        return {
-            'id': flv_id,
-            'url': rtmp_url,
-            'ext': 'flv',
-            'no_resume': True,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'view_count': view_count,
-            'comment_count': comment_count,
-            'uploader': uploader,
-            'upload_date': upload_date,
-            'age_limit': 18
-        }
diff --git a/yt_dlp/extractor/carambatv.py b/yt_dlp/extractor/carambatv.py
deleted file mode 100644
index d6044a319..000000000
--- a/yt_dlp/extractor/carambatv.py
+++ /dev/null
@@ -1,105 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    format_field,
-    float_or_none,
-    int_or_none,
-    try_get,
-)
-
-from .videomore import VideomoreIE
-
-
-class CarambaTVIE(InfoExtractor):
-    _VALID_URL = r'(?:carambatv:|https?://video1\.carambatv\.ru/v/)(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://video1.carambatv.ru/v/191910501',
-        'md5': '2f4a81b7cfd5ab866ee2d7270cb34a2a',
-        'info_dict': {
-            'id': '191910501',
-            'ext': 'mp4',
-            'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 2678.31,
-        },
-    }, {
-        'url': 'carambatv:191910501',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        video = self._download_json(
-            'http://video1.carambatv.ru/v/%s/videoinfo.js' % video_id,
-            video_id)
-
-        title = video['title']
-
-        base_url = video.get('video') or 'http://video1.carambatv.ru/v/%s/' % video_id
-
-        formats = [{
-            'url': base_url + f['fn'],
-            'height': int_or_none(f.get('height')),
-            'format_id': format_field(f, 'height', '%sp'),
-        } for f in video['qualities'] if f.get('fn')]
-
-        thumbnail = video.get('splash')
-        duration = float_or_none(try_get(
-            video, lambda x: x['annotations'][0]['end_time'], compat_str))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
-
-
-class CarambaTVPageIE(InfoExtractor):
-    _VALID_URL = r'https?://carambatv\.ru/(?:[^/]+/)+(?P<id>[^/?#&]+)'
-    _TEST = {
-        'url': 'http://carambatv.ru/movie/bad-comedian/razborka-v-manile/',
-        'md5': 'a49fb0ec2ad66503eeb46aac237d3c86',
-        'info_dict': {
-            'id': '475222',
-            'ext': 'flv',
-            'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
-            'thumbnail': r're:^https?://.*\.jpg',
-            # duration reported by videomore is incorrect
-            'duration': int,
-        },
-        'add_ie': [VideomoreIE.ie_key()],
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        videomore_url = VideomoreIE._extract_url(webpage)
-        if not videomore_url:
-            videomore_id = self._search_regex(
-                r'getVMCode\s*\(\s*["\']?(\d+)', webpage, 'videomore id',
-                default=None)
-            if videomore_id:
-                videomore_url = 'videomore:%s' % videomore_id
-        if videomore_url:
-            title = self._og_search_title(webpage)
-            return {
-                '_type': 'url_transparent',
-                'url': videomore_url,
-                'ie_key': VideomoreIE.ie_key(),
-                'title': title,
-            }
-
-        video_url = self._og_search_property('video:iframe', webpage, default=None)
-
-        if not video_url:
-            video_id = self._search_regex(
-                r'(?:video_id|crmb_vuid)\s*[:=]\s*["\']?(\d+)',
-                webpage, 'video id')
-            video_url = 'carambatv:%s' % video_id
-
-        return self.url_result(video_url, CarambaTVIE.ie_key())
diff --git a/yt_dlp/extractor/channel9.py b/yt_dlp/extractor/channel9.py
deleted file mode 100644
index a88474060..000000000
--- a/yt_dlp/extractor/channel9.py
+++ /dev/null
@@ -1,252 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    int_or_none,
-    parse_iso8601,
-    qualities,
-    unescapeHTML,
-)
-
-
-class Channel9IE(InfoExtractor):
-    IE_DESC = 'Channel 9'
-    IE_NAME = 'channel9'
-    _VALID_URL = r'https?://(?:www\.)?(?:channel9\.msdn\.com|s\.ch9\.ms)/(?P<contentpath>.+?)(?P<rss>/RSS)?/?(?:[?#&]|$)'
-    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>https?://channel9\.msdn\.com/(?:[^/]+/)+)player\b']
-
-    _TESTS = [{
-        'url': 'http://channel9.msdn.com/Events/TechEd/Australia/2013/KOS002',
-        'md5': '32083d4eaf1946db6d454313f44510ca',
-        'info_dict': {
-            'id': '6c413323-383a-49dc-88f9-a22800cab024',
-            'ext': 'wmv',
-            'title': 'Developer Kick-Off Session: Stuff We Love',
-            'description': 'md5:b80bf9355a503c193aff7ec6cd5a7731',
-            'duration': 4576,
-            'thumbnail': r're:https?://.*\.jpg',
-            'timestamp': 1377717420,
-            'upload_date': '20130828',
-            'session_code': 'KOS002',
-            'session_room': 'Arena 1A',
-            'session_speakers': 'count:5',
-        },
-    }, {
-        'url': 'http://channel9.msdn.com/posts/Self-service-BI-with-Power-BI-nuclear-testing',
-        'md5': 'dcf983ee6acd2088e7188c3cf79b46bc',
-        'info_dict': {
-            'id': 'fe8e435f-bb93-4e01-8e97-a28c01887024',
-            'ext': 'wmv',
-            'title': 'Self-service BI with Power BI - nuclear testing',
-            'description': 'md5:2d17fec927fc91e9e17783b3ecc88f54',
-            'duration': 1540,
-            'thumbnail': r're:https?://.*\.jpg',
-            'timestamp': 1386381991,
-            'upload_date': '20131207',
-            'authors': ['Mike Wilmot'],
-        },
-    }, {
-        # low quality mp4 is best
-        'url': 'https://channel9.msdn.com/Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
-        'info_dict': {
-            'id': '33ad69d2-6a4e-4172-83a1-a523013dec76',
-            'ext': 'mp4',
-            'title': 'Ranges for the Standard Library',
-            'description': 'md5:9895e0a9fd80822d2f01c454b8f4a372',
-            'duration': 5646,
-            'thumbnail': r're:https?://.*\.jpg',
-            'upload_date': '20150930',
-            'timestamp': 1443640735,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://channel9.msdn.com/Events/DEVintersection/DEVintersection-2016/RSS',
-        'info_dict': {
-            'id': 'Events/DEVintersection/DEVintersection-2016',
-            'title': 'DEVintersection 2016 Orlando Sessions',
-        },
-        'playlist_mincount': 14,
-    }, {
-        'url': 'https://channel9.msdn.com/Niners/Splendid22/Queue/76acff796e8f411184b008028e0d492b/RSS',
-        'only_matching': True,
-    }, {
-        'url': 'https://channel9.msdn.com/Events/Speakers/scott-hanselman/RSS?UrlSafeName=scott-hanselman',
-        'only_matching': True,
-    }]
-
-    _RSS_URL = 'http://channel9.msdn.com/%s/RSS'
-
-    def _extract_list(self, video_id, rss_url=None):
-        if not rss_url:
-            rss_url = self._RSS_URL % video_id
-        rss = self._download_xml(rss_url, video_id, 'Downloading RSS')
-        entries = [self.url_result(session_url.text, 'Channel9')
-                   for session_url in rss.findall('./channel/item/link')]
-        title_text = rss.find('./channel/title').text
-        return self.playlist_result(entries, video_id, title_text)
-
-    def _real_extract(self, url):
-        content_path, rss = self._match_valid_url(url).groups()
-
-        if rss:
-            return self._extract_list(content_path, url)
-
-        webpage = self._download_webpage(
-            url, content_path, 'Downloading web page')
-
-        episode_data = self._search_regex(
-            r"data-episode='([^']+)'", webpage, 'episode data', default=None)
-        if episode_data:
-            episode_data = self._parse_json(unescapeHTML(
-                episode_data), content_path)
-            content_id = episode_data['contentId']
-            is_session = '/Sessions(' in episode_data['api']
-            content_url = 'https://channel9.msdn.com/odata' + episode_data['api'] + '?$select=Captions,CommentCount,MediaLengthInSeconds,PublishedDate,Rating,RatingCount,Title,VideoMP4High,VideoMP4Low,VideoMP4Medium,VideoPlayerPreviewImage,VideoWMV,VideoWMVHQ,Views,'
-            if is_session:
-                content_url += 'Code,Description,Room,Slides,Speakers,ZipFile&$expand=Speakers'
-            else:
-                content_url += 'Authors,Body&$expand=Authors'
-            content_data = self._download_json(content_url, content_id)
-            title = content_data['Title']
-
-            QUALITIES = (
-                'mp3',
-                'wmv', 'mp4',
-                'wmv-low', 'mp4-low',
-                'wmv-mid', 'mp4-mid',
-                'wmv-high', 'mp4-high',
-            )
-
-            quality_key = qualities(QUALITIES)
-
-            def quality(quality_id, format_url):
-                return (len(QUALITIES) if '_Source.' in format_url
-                        else quality_key(quality_id))
-
-            formats = []
-            urls = set()
-
-            SITE_QUALITIES = {
-                'MP3': 'mp3',
-                'MP4': 'mp4',
-                'Low Quality WMV': 'wmv-low',
-                'Low Quality MP4': 'mp4-low',
-                'Mid Quality WMV': 'wmv-mid',
-                'Mid Quality MP4': 'mp4-mid',
-                'High Quality WMV': 'wmv-high',
-                'High Quality MP4': 'mp4-high',
-            }
-
-            formats_select = self._search_regex(
-                r'(?s)<select[^>]+name=["\']format[^>]+>(.+?)</select', webpage,
-                'formats select', default=None)
-            if formats_select:
-                for mobj in re.finditer(
-                        r'<option\b[^>]+\bvalue=(["\'])(?P<url>(?:(?!\1).)+)\1[^>]*>\s*(?P<format>[^<]+?)\s*<',
-                        formats_select):
-                    format_url = mobj.group('url')
-                    if format_url in urls:
-                        continue
-                    urls.add(format_url)
-                    format_id = mobj.group('format')
-                    quality_id = SITE_QUALITIES.get(format_id, format_id)
-                    formats.append({
-                        'url': format_url,
-                        'format_id': quality_id,
-                        'quality': quality(quality_id, format_url),
-                        'vcodec': 'none' if quality_id == 'mp3' else None,
-                    })
-
-            API_QUALITIES = {
-                'VideoMP4Low': 'mp4-low',
-                'VideoWMV': 'wmv-mid',
-                'VideoMP4Medium': 'mp4-mid',
-                'VideoMP4High': 'mp4-high',
-                'VideoWMVHQ': 'wmv-hq',
-            }
-
-            for format_id, q in API_QUALITIES.items():
-                q_url = content_data.get(format_id)
-                if not q_url or q_url in urls:
-                    continue
-                urls.add(q_url)
-                formats.append({
-                    'url': q_url,
-                    'format_id': q,
-                    'quality': quality(q, q_url),
-                })
-
-            slides = content_data.get('Slides')
-            zip_file = content_data.get('ZipFile')
-
-            if not formats and not slides and not zip_file:
-                self.raise_no_formats(
-                    'None of recording, slides or zip are available for %s' % content_path)
-
-            subtitles = {}
-            for caption in content_data.get('Captions', []):
-                caption_url = caption.get('Url')
-                if not caption_url:
-                    continue
-                subtitles.setdefault(caption.get('Language', 'en'), []).append({
-                    'url': caption_url,
-                    'ext': 'vtt',
-                })
-
-            common = {
-                'id': content_id,
-                'title': title,
-                'description': clean_html(content_data.get('Description') or content_data.get('Body')),
-                'thumbnail': content_data.get('VideoPlayerPreviewImage'),
-                'duration': int_or_none(content_data.get('MediaLengthInSeconds')),
-                'timestamp': parse_iso8601(content_data.get('PublishedDate')),
-                'avg_rating': int_or_none(content_data.get('Rating')),
-                'rating_count': int_or_none(content_data.get('RatingCount')),
-                'view_count': int_or_none(content_data.get('Views')),
-                'comment_count': int_or_none(content_data.get('CommentCount')),
-                'subtitles': subtitles,
-            }
-            if is_session:
-                speakers = []
-                for s in content_data.get('Speakers', []):
-                    speaker_name = s.get('FullName')
-                    if not speaker_name:
-                        continue
-                    speakers.append(speaker_name)
-
-                common.update({
-                    'session_code': content_data.get('Code'),
-                    'session_room': content_data.get('Room'),
-                    'session_speakers': speakers,
-                })
-            else:
-                authors = []
-                for a in content_data.get('Authors', []):
-                    author_name = a.get('DisplayName')
-                    if not author_name:
-                        continue
-                    authors.append(author_name)
-                common['authors'] = authors
-
-            contents = []
-
-            if slides:
-                d = common.copy()
-                d.update({'title': title + '-Slides', 'url': slides})
-                contents.append(d)
-
-            if zip_file:
-                d = common.copy()
-                d.update({'title': title + '-Zip', 'url': zip_file})
-                contents.append(d)
-
-            if formats:
-                d = common.copy()
-                d.update({'title': title, 'formats': formats})
-                contents.append(d)
-            return self.playlist_result(contents)
-        else:
-            return self._extract_list(content_path)
diff --git a/yt_dlp/extractor/chirbit.py b/yt_dlp/extractor/chirbit.py
deleted file mode 100644
index 452711d97..000000000
--- a/yt_dlp/extractor/chirbit.py
+++ /dev/null
@@ -1,88 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_b64decode
-from ..utils import parse_duration
-
-
-class ChirbitIE(InfoExtractor):
-    IE_NAME = 'chirbit'
-    _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?:(?:wp|pl)/|fb_chirbit_player\.swf\?key=)?(?P<id>[\da-zA-Z]+)'
-    _TESTS = [{
-        'url': 'http://chirb.it/be2abG',
-        'info_dict': {
-            'id': 'be2abG',
-            'ext': 'mp3',
-            'title': 'md5:f542ea253f5255240be4da375c6a5d7e',
-            'description': 'md5:f24a4e22a71763e32da5fed59e47c770',
-            'duration': 306,
-            'uploader': 'Gerryaudio',
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        'url': 'https://chirb.it/fb_chirbit_player.swf?key=PrIPv5',
-        'only_matching': True,
-    }, {
-        'url': 'https://chirb.it/wp/MN58c2',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        audio_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'http://chirb.it/%s' % audio_id, audio_id)
-
-        data_fd = self._search_regex(
-            r'data-fd=(["\'])(?P<url>(?:(?!\1).)+)\1',
-            webpage, 'data fd', group='url')
-
-        # Reverse engineered from https://chirb.it/js/chirbit.player.js (look
-        # for soundURL)
-        audio_url = compat_b64decode(data_fd[::-1]).decode('utf-8')
-
-        title = self._search_regex(
-            r'class=["\']chirbit-title["\'][^>]*>([^<]+)', webpage, 'title')
-        description = self._search_regex(
-            r'<h3>Description</h3>\s*<pre[^>]*>([^<]+)</pre>',
-            webpage, 'description', default=None)
-        duration = parse_duration(self._search_regex(
-            r'class=["\']c-length["\'][^>]*>([^<]+)',
-            webpage, 'duration', fatal=False))
-        uploader = self._search_regex(
-            r'id=["\']chirbit-username["\'][^>]*>([^<]+)',
-            webpage, 'uploader', fatal=False)
-
-        return {
-            'id': audio_id,
-            'url': audio_url,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'uploader': uploader,
-        }
-
-
-class ChirbitProfileIE(InfoExtractor):
-    IE_NAME = 'chirbit:profile'
-    _VALID_URL = r'https?://(?:www\.)?chirbit\.com/(?:rss/)?(?P<id>[^/]+)'
-    _TEST = {
-        'url': 'http://chirbit.com/ScarletBeauty',
-        'info_dict': {
-            'id': 'ScarletBeauty',
-        },
-        'playlist_mincount': 3,
-    }
-
-    def _real_extract(self, url):
-        profile_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, profile_id)
-
-        entries = [
-            self.url_result(self._proto_relative_url('//chirb.it/' + video_id))
-            for _, video_id in re.findall(r'<input[^>]+id=([\'"])copy-btn-(?P<id>[0-9a-zA-Z]+)\1', webpage)]
-
-        return self.playlist_result(entries, profile_id)
diff --git a/yt_dlp/extractor/cinchcast.py b/yt_dlp/extractor/cinchcast.py
deleted file mode 100644
index 7a7ea8b22..000000000
--- a/yt_dlp/extractor/cinchcast.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-    xpath_text,
-)
-
-
-class CinchcastIE(InfoExtractor):
-    _VALID_URL = r'https?://player\.cinchcast\.com/.*?(?:assetId|show_id)=(?P<id>[0-9]+)'
-    _EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1']
-
-    _TESTS = [{
-        'url': 'http://player.cinchcast.com/?show_id=5258197&platformId=1&assetType=single',
-        'info_dict': {
-            'id': '5258197',
-            'ext': 'mp3',
-            'title': 'Train Your Brain to Up Your Game with Coach Mandy',
-            'upload_date': '20130816',
-        },
-    }, {
-        # Actual test is run in generic, look for undergroundwellness
-        'url': 'http://player.cinchcast.com/?platformId=1&#038;assetType=single&#038;assetId=7141703',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        doc = self._download_xml(
-            'http://www.blogtalkradio.com/playerasset/mrss?assetType=single&assetId=%s' % video_id,
-            video_id)
-
-        item = doc.find('.//item')
-        title = xpath_text(item, './title', fatal=True)
-        date_str = xpath_text(
-            item, './{http://developer.longtailvideo.com/trac/}date')
-        upload_date = unified_strdate(date_str, day_first=False)
-        # duration is present but wrong
-        formats = [{
-            'format_id': 'main',
-            'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url'],
-        }]
-        backup_url = xpath_text(
-            item, './{http://developer.longtailvideo.com/trac/}backupContent')
-        if backup_url:
-            formats.append({
-                'preference': 2,  # seems to be more reliable
-                'format_id': 'backup',
-                'url': backup_url,
-            })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'upload_date': upload_date,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/clipsyndicate.py b/yt_dlp/extractor/clipsyndicate.py
deleted file mode 100644
index 606444321..000000000
--- a/yt_dlp/extractor/clipsyndicate.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    find_xpath_attr,
-    fix_xml_ampersands
-)
-
-
-class ClipsyndicateIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
-
-    _TESTS = [{
-        'url': 'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe',
-        'md5': '4d7d549451bad625e0ff3d7bd56d776c',
-        'info_dict': {
-            'id': '4629301',
-            'ext': 'mp4',
-            'title': 'Brick Briscoe',
-            'duration': 612,
-            'thumbnail': r're:^https?://.+\.jpg',
-        },
-    }, {
-        'url': 'http://chic.clipsyndicate.com/video/play/5844117/shark_attack',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        js_player = self._download_webpage(
-            'http://eplayer.clipsyndicate.com/embed/player.js?va_id=%s' % video_id,
-            video_id, 'Downlaoding player')
-        # it includes a required token
-        flvars = self._search_regex(r'flvars: "(.*?)"', js_player, 'flvars')
-
-        pdoc = self._download_xml(
-            'http://eplayer.clipsyndicate.com/osmf/playlist?%s' % flvars,
-            video_id, 'Downloading video info',
-            transform_source=fix_xml_ampersands)
-
-        track_doc = pdoc.find('trackList/track')
-
-        def find_param(name):
-            node = find_xpath_attr(track_doc, './/param', 'name', name)
-            if node is not None:
-                return node.attrib['value']
-
-        return {
-            'id': video_id,
-            'title': find_param('title'),
-            'url': track_doc.find('location').text,
-            'thumbnail': find_param('thumbnail'),
-            'duration': int(find_param('duration')),
-        }
diff --git a/yt_dlp/extractor/cloudy.py b/yt_dlp/extractor/cloudy.py
deleted file mode 100644
index 848643e26..000000000
--- a/yt_dlp/extractor/cloudy.py
+++ /dev/null
@@ -1,57 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    str_to_int,
-    unified_strdate,
-)
-
-
-class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec'
-    _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
-    _TESTS = [{
-        'url': 'https://www.cloudy.ec/v/af511e2527aac',
-        'md5': '29832b05028ead1b58be86bf319397ca',
-        'info_dict': {
-            'id': 'af511e2527aac',
-            'ext': 'mp4',
-            'title': 'Funny Cats and Animals Compilation june 2013',
-            'upload_date': '20130913',
-            'view_count': int,
-        }
-    }, {
-        'url': 'http://www.cloudy.ec/embed.php?autoplay=1&id=af511e2527aac',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'https://www.cloudy.ec/embed.php', video_id, query={
-                'id': video_id,
-                'playerPage': 1,
-                'autoplay': 1,
-            })
-
-        info = self._parse_html5_media_entries(url, webpage, video_id)[0]
-
-        webpage = self._download_webpage(
-            'https://www.cloudy.ec/v/%s' % video_id, video_id, fatal=False)
-
-        if webpage:
-            info.update({
-                'title': self._search_regex(
-                    r'<h\d[^>]*>([^<]+)<', webpage, 'title'),
-                'upload_date': unified_strdate(self._search_regex(
-                    r'>Published at (\d{4}-\d{1,2}-\d{1,2})', webpage,
-                    'upload date', fatal=False)),
-                'view_count': str_to_int(self._search_regex(
-                    r'([\d,.]+) views<', webpage, 'view count', fatal=False)),
-            })
-
-        if not info.get('title'):
-            info['title'] = video_id
-
-        info['id'] = video_id
-
-        return info
diff --git a/yt_dlp/extractor/clubic.py b/yt_dlp/extractor/clubic.py
index 403e44aaf..716f25969 100644
--- a/yt_dlp/extractor/clubic.py
+++ b/yt_dlp/extractor/clubic.py
@@ -6,6 +6,7 @@
 
 
 class ClubicIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'https?://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
 
     _TESTS = [{
diff --git a/yt_dlp/extractor/cmt.py b/yt_dlp/extractor/cmt.py
index 8aed7708b..6359102aa 100644
--- a/yt_dlp/extractor/cmt.py
+++ b/yt_dlp/extractor/cmt.py
@@ -4,6 +4,7 @@
 
 
 class CMTIE(MTVIE):  # XXX: Do not subclass from concrete IE
+    _WORKING = False
     IE_NAME = 'cmt.com'
     _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|(?:full-)?episodes|video-clips)/(?P<id>[^/]+)'
 
diff --git a/yt_dlp/extractor/daftsex.py b/yt_dlp/extractor/daftsex.py
deleted file mode 100644
index 92510c767..000000000
--- a/yt_dlp/extractor/daftsex.py
+++ /dev/null
@@ -1,150 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_b64decode
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    js_to_json,
-    parse_count,
-    parse_duration,
-    traverse_obj,
-    try_get,
-    unified_timestamp,
-)
-
-
-class DaftsexIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?daft\.sex/watch/(?P<id>-?\d+_\d+)'
-    _TESTS = [{
-        'url': 'https://daft.sex/watch/-35370899_456246186',
-        'md5': '64c04ef7b4c7b04b308f3b0c78efe7cd',
-        'info_dict': {
-            'id': '-35370899_456246186',
-            'ext': 'mp4',
-            'title': 'just relaxing',
-            'description': 'just relaxing – Watch video Watch video in high quality',
-            'upload_date': '20201113',
-            'timestamp': 1605261911,
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'age_limit': 18,
-            'duration': 15.0,
-            'view_count': int
-        },
-    }, {
-        'url': 'https://daft.sex/watch/-156601359_456242791',
-        'info_dict': {
-            'id': '-156601359_456242791',
-            'ext': 'mp4',
-            'title': 'Skye Blue - Dinner And A Show',
-            'description': 'Skye Blue - Dinner And A Show - Watch video Watch video in high quality',
-            'upload_date': '20200916',
-            'timestamp': 1600250735,
-            'thumbnail': 'https://psv153-1.crazycloud.ru/videos/-156601359/456242791/thumb.jpg?extra=i3D32KaBbBFf9TqDRMAVmQ',
-        },
-        'skip': 'deleted / private'
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        title = self._html_search_meta('name', webpage, 'title')
-        timestamp = unified_timestamp(self._html_search_meta('uploadDate', webpage, 'Upload Date', default=None))
-        description = self._html_search_meta('description', webpage, 'Description', default=None)
-
-        duration = parse_duration(self._search_regex(
-            r'Duration: ((?:[0-9]{2}:){0,2}[0-9]{2})',
-            webpage, 'duration', fatal=False))
-        views = parse_count(self._search_regex(
-            r'Views: ([0-9 ]+)',
-            webpage, 'views', fatal=False))
-
-        player_hash = self._search_regex(
-            r'DaxabPlayer\.Init\({[\s\S]*hash:\s*"([0-9a-zA-Z_\-]+)"[\s\S]*}',
-            webpage, 'player hash')
-        player_color = self._search_regex(
-            r'DaxabPlayer\.Init\({[\s\S]*color:\s*"([0-9a-z]+)"[\s\S]*}',
-            webpage, 'player color', fatal=False) or ''
-
-        embed_page = self._download_webpage(
-            'https://dxb.to/player/%s?color=%s' % (player_hash, player_color),
-            video_id, headers={'Referer': url})
-        video_params = self._parse_json(
-            self._search_regex(
-                r'window\.globParams\s*=\s*({[\S\s]+})\s*;\s*<\/script>',
-                embed_page, 'video parameters'),
-            video_id, transform_source=js_to_json)
-
-        server_domain = 'https://%s' % compat_b64decode(video_params['server'][::-1]).decode('utf-8')
-
-        cdn_files = traverse_obj(video_params, ('video', 'cdn_files')) or {}
-        if cdn_files:
-            formats = []
-            for format_id, format_data in cdn_files.items():
-                ext, height = format_id.split('_')
-                formats.append({
-                    'format_id': format_id,
-                    'url': f'{server_domain}/videos/{video_id.replace("_", "/")}/{height}.mp4?extra={format_data.split(".")[-1]}',
-                    'height': int_or_none(height),
-                    'ext': ext,
-                })
-
-            return {
-                'id': video_id,
-                'title': title,
-                'formats': formats,
-                'description': description,
-                'duration': duration,
-                'thumbnail': try_get(video_params, lambda vi: 'https:' + compat_b64decode(vi['video']['thumb']).decode('utf-8')),
-                'timestamp': timestamp,
-                'view_count': views,
-                'age_limit': 18,
-            }
-
-        items = self._download_json(
-            f'{server_domain}/method/video.get/{video_id}', video_id,
-            headers={'Referer': url}, query={
-                'token': video_params['video']['access_token'],
-                'videos': video_id,
-                'ckey': video_params['c_key'],
-                'credentials': video_params['video']['credentials'],
-            })['response']['items']
-
-        if not items:
-            raise ExtractorError('Video is not available', video_id=video_id, expected=True)
-
-        item = items[0]
-        formats = []
-        for f_id, f_url in item.get('files', {}).items():
-            if f_id == 'external':
-                return self.url_result(f_url)
-            ext, height = f_id.split('_')
-            height_extra_key = traverse_obj(video_params, ('video', 'partial', 'quality', height))
-            if height_extra_key:
-                formats.append({
-                    'format_id': f'{height}p',
-                    'url': f'{server_domain}/{f_url[8:]}&videos={video_id}&extra_key={height_extra_key}',
-                    'height': int_or_none(height),
-                    'ext': ext,
-                })
-
-        thumbnails = []
-        for k, v in item.items():
-            if k.startswith('photo_') and v:
-                width = k.replace('photo_', '')
-                thumbnails.append({
-                    'id': width,
-                    'url': v,
-                    'width': int_or_none(width),
-                })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'comment_count': int_or_none(item.get('comments')),
-            'description': description,
-            'duration': duration,
-            'thumbnails': thumbnails,
-            'timestamp': timestamp,
-            'view_count': views,
-            'age_limit': 18,
-        }
diff --git a/yt_dlp/extractor/defense.py b/yt_dlp/extractor/defense.py
deleted file mode 100644
index 7d73ea862..000000000
--- a/yt_dlp/extractor/defense.py
+++ /dev/null
@@ -1,37 +0,0 @@
-from .common import InfoExtractor
-
-
-class DefenseGouvFrIE(InfoExtractor):
-    IE_NAME = 'defense.gouv.fr'
-    _VALID_URL = r'https?://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'
-
-    _TEST = {
-        'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
-        'md5': '75bba6124da7e63d2d60b5244ec9430c',
-        'info_dict': {
-            'id': '11213',
-            'ext': 'mp4',
-            'title': 'attaque-chimique-syrienne-du-21-aout-2013-1'
-        }
-    }
-
-    def _real_extract(self, url):
-        title = self._match_id(url)
-        webpage = self._download_webpage(url, title)
-
-        video_id = self._search_regex(
-            r"flashvars.pvg_id=\"(\d+)\";",
-            webpage, 'ID')
-
-        json_url = (
-            'http://static.videos.gouv.fr/brightcovehub/export/json/%s' %
-            video_id)
-        info = self._download_json(json_url, title, 'Downloading JSON config')
-        video_url = info['renditions'][0]['url']
-
-        return {
-            'id': video_id,
-            'ext': 'mp4',
-            'url': video_url,
-            'title': title,
-        }
diff --git a/yt_dlp/extractor/dhm.py b/yt_dlp/extractor/dhm.py
index 3d42fc2b0..a5f5f794c 100644
--- a/yt_dlp/extractor/dhm.py
+++ b/yt_dlp/extractor/dhm.py
@@ -3,6 +3,7 @@
 
 
 class DHMIE(InfoExtractor):
+    _WORKING = False
     IE_DESC = 'Filmarchiv - Deutsches Historisches Museum'
     _VALID_URL = r'https?://(?:www\.)?dhm\.de/filmarchiv/(?:[^/]+/)+(?P<id>[^/]+)'
 
diff --git a/yt_dlp/extractor/dotsub.py b/yt_dlp/extractor/dotsub.py
deleted file mode 100644
index 079f83750..000000000
--- a/yt_dlp/extractor/dotsub.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    int_or_none,
-)
-
-
-class DotsubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
-    _TESTS = [{
-        'url': 'https://dotsub.com/view/9c63db2a-fa95-4838-8e6e-13deafe47f09',
-        'md5': '21c7ff600f545358134fea762a6d42b6',
-        'info_dict': {
-            'id': '9c63db2a-fa95-4838-8e6e-13deafe47f09',
-            'ext': 'flv',
-            'title': 'MOTIVATION - "It\'s Possible" Best Inspirational Video Ever',
-            'description': 'md5:41af1e273edbbdfe4e216a78b9d34ac6',
-            'thumbnail': 're:^https?://dotsub.com/media/9c63db2a-fa95-4838-8e6e-13deafe47f09/p',
-            'duration': 198,
-            'uploader': 'liuxt',
-            'timestamp': 1385778501.104,
-            'upload_date': '20131130',
-            'view_count': int,
-        }
-    }, {
-        'url': 'https://dotsub.com/view/747bcf58-bd59-45b7-8c8c-ac312d084ee6',
-        'md5': '2bb4a83896434d5c26be868c609429a3',
-        'info_dict': {
-            'id': '168006778',
-            'ext': 'mp4',
-            'title': 'Apartments and flats in Raipur the white symphony',
-            'description': 'md5:784d0639e6b7d1bc29530878508e38fe',
-            'thumbnail': 're:^https?://dotsub.com/media/747bcf58-bd59-45b7-8c8c-ac312d084ee6/p',
-            'duration': 290,
-            'timestamp': 1476767794.2809999,
-            'upload_date': '20161018',
-            'uploader': 'parthivi001',
-            'uploader_id': 'user52596202',
-            'view_count': int,
-        },
-        'add_ie': ['Vimeo'],
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        info = self._download_json(
-            'https://dotsub.com/api/media/%s/metadata' % video_id, video_id)
-        video_url = info.get('mediaURI')
-
-        if not video_url:
-            webpage = self._download_webpage(url, video_id)
-            video_url = self._search_regex(
-                [r'<source[^>]+src="([^"]+)"', r'"file"\s*:\s*\'([^\']+)'],
-                webpage, 'video url', default=None)
-            info_dict = {
-                'id': video_id,
-                'url': video_url,
-                'ext': 'flv',
-            }
-
-        if not video_url:
-            setup_data = self._parse_json(self._html_search_regex(
-                r'(?s)data-setup=([\'"])(?P<content>(?!\1).+?)\1',
-                webpage, 'setup data', group='content'), video_id)
-            info_dict = {
-                '_type': 'url_transparent',
-                'url': setup_data['src'],
-            }
-
-        info_dict.update({
-            'title': info['title'],
-            'description': info.get('description'),
-            'thumbnail': info.get('screenshotURI'),
-            'duration': int_or_none(info.get('duration'), 1000),
-            'uploader': info.get('user'),
-            'timestamp': float_or_none(info.get('dateCreated'), 1000),
-            'view_count': int_or_none(info.get('numberOfViews')),
-        })
-
-        return info_dict
diff --git a/yt_dlp/extractor/echomsk.py b/yt_dlp/extractor/echomsk.py
deleted file mode 100644
index 850eabbff..000000000
--- a/yt_dlp/extractor/echomsk.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import re
-
-from .common import InfoExtractor
-
-
-class EchoMskIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?echo\.msk\.ru/sounds/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.echo.msk.ru/sounds/1464134.html',
-        'md5': '2e44b3b78daff5b458e4dbc37f191f7c',
-        'info_dict': {
-            'id': '1464134',
-            'ext': 'mp3',
-            'title': 'Особое мнение - 29 декабря 2014, 19:08',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        audio_url = self._search_regex(
-            r'<a rel="mp3" href="([^"]+)">', webpage, 'audio URL')
-
-        title = self._html_search_regex(
-            r'<a href="/programs/[^"]+" target="_blank">([^<]+)</a>',
-            webpage, 'title')
-
-        air_date = self._html_search_regex(
-            r'(?s)<div class="date">(.+?)</div>',
-            webpage, 'date', fatal=False, default=None)
-
-        if air_date:
-            air_date = re.sub(r'(\s)\1+', r'\1', air_date)
-            if air_date:
-                title = '%s - %s' % (title, air_date)
-
-        return {
-            'id': video_id,
-            'url': audio_url,
-            'title': title,
-        }
diff --git a/yt_dlp/extractor/ehow.py b/yt_dlp/extractor/ehow.py
deleted file mode 100644
index 74469ce36..000000000
--- a/yt_dlp/extractor/ehow.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
-
-
-class EHowIE(InfoExtractor):
-    IE_NAME = 'eHow'
-    _VALID_URL = r'https?://(?:www\.)?ehow\.com/[^/_?]*_(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.ehow.com/video_12245069_hardwood-flooring-basics.html',
-        'md5': '9809b4e3f115ae2088440bcb4efbf371',
-        'info_dict': {
-            'id': '12245069',
-            'ext': 'flv',
-            'title': 'Hardwood Flooring Basics',
-            'description': 'Hardwood flooring may be time consuming, but its ultimately a pretty straightforward concept. Learn about hardwood flooring basics with help from a hardware flooring business owner in this free video...',
-            'uploader': 'Erick Nathan',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_url = self._search_regex(
-            r'(?:file|source)=(http[^\'"&]*)', webpage, 'video URL')
-        final_url = compat_urllib_parse_unquote(video_url)
-        uploader = self._html_search_meta('uploader', webpage)
-        title = self._og_search_title(webpage).replace(' | eHow', '')
-
-        return {
-            'id': video_id,
-            'url': final_url,
-            'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage),
-            'uploader': uploader,
-        }
diff --git a/yt_dlp/extractor/elevensports.py b/yt_dlp/extractor/elevensports.py
deleted file mode 100644
index 99c52b3a9..000000000
--- a/yt_dlp/extractor/elevensports.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    parse_iso8601,
-    traverse_obj,
-    url_or_none,
-)
-
-
-class ElevenSportsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?elevensports\.com/view/event/(?P<id>\w+)'
-    _TESTS = [{
-        'url': 'https://elevensports.com/view/event/clf46yr3kenn80jgrqsjmwefk',
-        'md5': 'c0958d9ff90e4503a75544358758921d',
-        'info_dict': {
-            'id': 'clf46yr3kenn80jgrqsjmwefk',
-            'title': 'Cleveland SC vs Lionsbridge FC',
-            'ext': 'mp4',
-            'description': 'md5:03b5238d6549f4ea1fddadf69b5e0b58',
-            'upload_date': '20230323',
-            'timestamp': 1679612400,
-            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
-        },
-        'params': {'skip_download': 'm3u8'}
-    }, {
-        'url': 'https://elevensports.com/view/event/clhpyd53b06160jez74qhgkmf',
-        'md5': 'c0958d9ff90e4503a75544358758921d',
-        'info_dict': {
-            'id': 'clhpyd53b06160jez74qhgkmf',
-            'title': 'AJNLF vs ARRAF',
-            'ext': 'mp4',
-            'description': 'md5:c8c5e75c78f37c6d15cd6c475e43a8c1',
-            'upload_date': '20230521',
-            'timestamp': 1684684800,
-            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
-        },
-        'params': {'skip_download': 'm3u8'}
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        event_id = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['event']['mclsEventId']
-        event_data = self._download_json(
-            f'https://mcls-api.mycujoo.tv/bff/events/v1beta1/{event_id}', video_id,
-            headers={'Authorization': 'Bearer FBVKACGN37JQC5SFA0OVK8KKSIOP153G'})
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
-            event_data['streams'][0]['full_url'], video_id, 'mp4', m3u8_id='hls')
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'subtitles': subtitles,
-            **traverse_obj(event_data, {
-                'title': ('title', {str}),
-                'description': ('description', {str}),
-                'timestamp': ('start_time', {parse_iso8601}),
-                'thumbnail': ('thumbnail_url', {url_or_none}),
-            }),
-        }
diff --git a/yt_dlp/extractor/ellentube.py b/yt_dlp/extractor/ellentube.py
deleted file mode 100644
index 6eb00f9c9..000000000
--- a/yt_dlp/extractor/ellentube.py
+++ /dev/null
@@ -1,130 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    extract_attributes,
-    float_or_none,
-    int_or_none,
-    try_get,
-)
-
-
-class EllenTubeBaseIE(InfoExtractor):
-    def _extract_data_config(self, webpage, video_id):
-        details = self._search_regex(
-            r'(<[^>]+\bdata-component=(["\'])[Dd]etails.+?></div>)', webpage,
-            'details')
-        return self._parse_json(
-            extract_attributes(details)['data-config'], video_id)
-
-    def _extract_video(self, data, video_id):
-        title = data['title']
-
-        formats = []
-        duration = None
-        for entry in data.get('media'):
-            if entry.get('id') == 'm3u8':
-                formats, subtitles = self._extract_m3u8_formats_and_subtitles(
-                    entry['url'], video_id, 'mp4',
-                    entry_protocol='m3u8_native', m3u8_id='hls')
-                duration = int_or_none(entry.get('duration'))
-                break
-
-        def get_insight(kind):
-            return int_or_none(try_get(
-                data, lambda x: x['insight']['%ss' % kind]))
-
-        return {
-            'extractor_key': EllenTubeIE.ie_key(),
-            'id': video_id,
-            'title': title,
-            'description': data.get('description'),
-            'duration': duration,
-            'thumbnail': data.get('thumbnail'),
-            'timestamp': float_or_none(data.get('publishTime'), scale=1000),
-            'view_count': get_insight('view'),
-            'like_count': get_insight('like'),
-            'formats': formats,
-            'subtitles': subtitles,
-        }
-
-
-class EllenTubeIE(EllenTubeBaseIE):
-    _VALID_URL = r'''(?x)
-                        (?:
-                            ellentube:|
-                            https://api-prod\.ellentube\.com/ellenapi/api/item/
-                        )
-                        (?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})
-                    '''
-    _TESTS = [{
-        'url': 'https://api-prod.ellentube.com/ellenapi/api/item/0822171c-3829-43bf-b99f-d77358ae75e3',
-        'md5': '2fabc277131bddafdd120e0fc0f974c9',
-        'info_dict': {
-            'id': '0822171c-3829-43bf-b99f-d77358ae75e3',
-            'ext': 'mp4',
-            'title': 'Ellen Meets Las Vegas Survivors Jesus Campos and Stephen Schuck',
-            'description': 'md5:76e3355e2242a78ad9e3858e5616923f',
-            'thumbnail': r're:^https?://.+?',
-            'duration': 514,
-            'timestamp': 1508505120,
-            'upload_date': '20171020',
-            'view_count': int,
-            'like_count': int,
-        }
-    }, {
-        'url': 'ellentube:734a3353-f697-4e79-9ca9-bfc3002dc1e0',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        data = self._download_json(
-            'https://api-prod.ellentube.com/ellenapi/api/item/%s' % video_id,
-            video_id)
-        return self._extract_video(data, video_id)
-
-
-class EllenTubeVideoIE(EllenTubeBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/video/(?P<id>.+?)\.html'
-    _TEST = {
-        'url': 'https://www.ellentube.com/video/ellen-meets-las-vegas-survivors-jesus-campos-and-stephen-schuck.html',
-        'only_matching': True,
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        video_id = self._extract_data_config(webpage, display_id)['id']
-        return self.url_result(
-            'ellentube:%s' % video_id, ie=EllenTubeIE.ie_key(),
-            video_id=video_id)
-
-
-class EllenTubePlaylistIE(EllenTubeBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/(?:episode|studios)/(?P<id>.+?)\.html'
-    _TESTS = [{
-        'url': 'https://www.ellentube.com/episode/dax-shepard-jordan-fisher-haim.html',
-        'info_dict': {
-            'id': 'dax-shepard-jordan-fisher-haim',
-            'title': "Dax Shepard, 'DWTS' Team Jordan Fisher & Lindsay Arnold, HAIM",
-            'description': 'md5:bfc982194dabb3f4e325e43aa6b2e21c',
-        },
-        'playlist_count': 6,
-    }, {
-        'url': 'https://www.ellentube.com/studios/macey-goes-rving0.html',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        data = self._extract_data_config(webpage, display_id)['data']
-        feed = self._download_json(
-            'https://api-prod.ellentube.com/ellenapi/api/feed/?%s'
-            % data['filter'], display_id)
-        entries = [
-            self._extract_video(elem, elem['id'])
-            for elem in feed if elem.get('type') == 'VIDEO' and elem.get('id')]
-        return self.playlist_result(
-            entries, display_id, data.get('title'),
-            clean_html(data.get('description')))
diff --git a/yt_dlp/extractor/engadget.py b/yt_dlp/extractor/engadget.py
deleted file mode 100644
index e7c5d7bf1..000000000
--- a/yt_dlp/extractor/engadget.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from .common import InfoExtractor
-
-
-class EngadgetIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?engadget\.com/video/(?P<id>[^/?#]+)'
-
-    _TESTS = [{
-        # video with vidible ID
-        'url': 'https://www.engadget.com/video/57a28462134aa15a39f0421a/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        return self.url_result('aol-video:%s' % video_id)
diff --git a/yt_dlp/extractor/escapist.py b/yt_dlp/extractor/escapist.py
deleted file mode 100644
index 85a1cbf40..000000000
--- a/yt_dlp/extractor/escapist.py
+++ /dev/null
@@ -1,108 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-    clean_html,
-    int_or_none,
-    float_or_none,
-)
-
-
-def _decrypt_config(key, string):
-    a = ''
-    i = ''
-    r = ''
-
-    while len(a) < (len(string) / 2):
-        a += key
-
-    a = a[0:int(len(string) / 2)]
-
-    t = 0
-    while t < len(string):
-        i += chr(int(string[t] + string[t + 1], 16))
-        t += 2
-
-    icko = [s for s in i]
-
-    for t, c in enumerate(a):
-        r += chr(ord(c) ^ ord(icko[t]))
-
-    return r
-
-
-class EscapistIE(InfoExtractor):
-    _VALID_URL = r'https?://?(?:(?:www|v1)\.)?escapistmagazine\.com/videos/view/[^/]+/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
-        'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
-        'info_dict': {
-            'id': '6618',
-            'ext': 'mp4',
-            'description': "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
-            'title': "Breaking Down Baldur's Gate",
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 264,
-            'uploader': 'The Escapist',
-        }
-    }, {
-        'url': 'http://www.escapistmagazine.com/videos/view/zero-punctuation/10044-Evolve-One-vs-Multiplayer',
-        'md5': '9e8c437b0dbb0387d3bd3255ca77f6bf',
-        'info_dict': {
-            'id': '10044',
-            'ext': 'mp4',
-            'description': 'This week, Zero Punctuation reviews Evolve.',
-            'title': 'Evolve - One vs Multiplayer',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 304,
-            'uploader': 'The Escapist',
-        }
-    }, {
-        'url': 'http://escapistmagazine.com/videos/view/the-escapist-presents/6618',
-        'only_matching': True,
-    }, {
-        'url': 'https://v1.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        ims_video = self._parse_json(
-            self._search_regex(
-                r'imsVideo\.play\(({.+?})\);', webpage, 'imsVideo'),
-            video_id)
-        video_id = ims_video['videoID']
-        key = ims_video['hash']
-
-        config = self._download_webpage(
-            'http://www.escapistmagazine.com/videos/vidconfig.php',
-            video_id, 'Downloading video config', headers={
-                'Referer': url,
-            }, query={
-                'videoID': video_id,
-                'hash': key,
-            })
-
-        data = self._parse_json(_decrypt_config(key, config), video_id)
-
-        video_data = data['videoData']
-
-        title = clean_html(video_data['title'])
-
-        formats = [{
-            'url': video['src'],
-            'format_id': '%s-%sp' % (determine_ext(video['src']), video['res']),
-            'height': int_or_none(video.get('res')),
-        } for video in data['files']['videos']]
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage) or data.get('poster'),
-            'description': self._og_search_description(webpage),
-            'duration': float_or_none(video_data.get('duration'), 1000),
-            'uploader': video_data.get('publisher'),
-            'series': video_data.get('show'),
-        }
diff --git a/yt_dlp/extractor/esri.py b/yt_dlp/extractor/esri.py
deleted file mode 100644
index 02e7efaf0..000000000
--- a/yt_dlp/extractor/esri.py
+++ /dev/null
@@ -1,70 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    int_or_none,
-    parse_filesize,
-    unified_strdate,
-)
-
-
-class EsriVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.esri\.com/watch/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'https://video.esri.com/watch/1124/arcgis-online-_dash_-developing-applications',
-        'md5': 'd4aaf1408b221f1b38227a9bbaeb95bc',
-        'info_dict': {
-            'id': '1124',
-            'ext': 'mp4',
-            'title': 'ArcGIS Online - Developing Applications',
-            'description': 'Jeremy Bartley demonstrates how to develop applications with ArcGIS Online.',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 185,
-            'upload_date': '20120419',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        formats = []
-        for width, height, content in re.findall(
-                r'(?s)<li><strong>(\d+)x(\d+):</strong>(.+?)</li>', webpage):
-            for video_url, ext, filesize in re.findall(
-                    r'<a[^>]+href="([^"]+)">([^<]+)&nbsp;\(([^<]+)\)</a>', content):
-                formats.append({
-                    'url': compat_urlparse.urljoin(url, video_url),
-                    'ext': ext.lower(),
-                    'format_id': '%s-%s' % (ext.lower(), height),
-                    'width': int(width),
-                    'height': int(height),
-                    'filesize_approx': parse_filesize(filesize),
-                })
-
-        title = self._html_search_meta('title', webpage, 'title')
-        description = self._html_search_meta(
-            'description', webpage, 'description', fatal=False)
-
-        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail', fatal=False)
-        if thumbnail:
-            thumbnail = re.sub(r'_[st]\.jpg$', '_x.jpg', thumbnail)
-
-        duration = int_or_none(self._search_regex(
-            [r'var\s+videoSeconds\s*=\s*(\d+)', r"'duration'\s*:\s*(\d+)"],
-            webpage, 'duration', fatal=False))
-
-        upload_date = unified_strdate(self._html_search_meta(
-            'last-modified', webpage, 'upload date', fatal=False))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'upload_date': upload_date,
-            'formats': formats
-        }
diff --git a/yt_dlp/extractor/expotv.py b/yt_dlp/extractor/expotv.py
deleted file mode 100644
index bda6e3cb2..000000000
--- a/yt_dlp/extractor/expotv.py
+++ /dev/null
@@ -1,74 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    unified_strdate,
-)
-
-
-class ExpoTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?expotv\.com/videos/[^?#]*/(?P<id>[0-9]+)($|[?#])'
-    _TEST = {
-        'url': 'http://www.expotv.com/videos/reviews/3/40/NYX-Butter-lipstick/667916',
-        'md5': 'fe1d728c3a813ff78f595bc8b7a707a8',
-        'info_dict': {
-            'id': '667916',
-            'ext': 'mp4',
-            'title': 'NYX Butter Lipstick Little Susie',
-            'description': 'Goes on like butter, but looks better!',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'Stephanie S.',
-            'upload_date': '20150520',
-            'view_count': int,
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        player_key = self._search_regex(
-            r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
-        config = self._download_json(
-            'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
-            video_id, 'Downloading video configuration')
-
-        formats = []
-        for fcfg in config['sources']:
-            media_url = fcfg.get('file')
-            if not media_url:
-                continue
-            if fcfg.get('type') == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    media_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls'))
-            else:
-                formats.append({
-                    'url': media_url,
-                    'height': int_or_none(fcfg.get('height')),
-                    'format_id': fcfg.get('label'),
-                    'ext': self._search_regex(
-                        r'filename=.*\.([a-z0-9_A-Z]+)&', media_url,
-                        'file extension', default=None) or fcfg.get('type'),
-                })
-
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = config.get('image')
-        view_count = int_or_none(self._search_regex(
-            r'<h5>Plays: ([0-9]+)</h5>', webpage, 'view counts'))
-        uploader = self._search_regex(
-            r'<div class="reviewer">\s*<img alt="([^"]+)"', webpage, 'uploader',
-            fatal=False)
-        upload_date = unified_strdate(self._search_regex(
-            r'<h5>Reviewed on ([0-9/.]+)</h5>', webpage, 'upload date',
-            fatal=False), day_first=False)
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': title,
-            'description': description,
-            'view_count': view_count,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'upload_date': upload_date,
-        }
diff --git a/yt_dlp/extractor/extremetube.py b/yt_dlp/extractor/extremetube.py
deleted file mode 100644
index 2c1969899..000000000
--- a/yt_dlp/extractor/extremetube.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from ..utils import str_to_int
-from .keezmovies import KeezMoviesIE
-
-
-class ExtremeTubeIE(KeezMoviesIE):  # XXX: Do not subclass from concrete IE
-    _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)'
-    _TESTS = [{
-        'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
-        'md5': '92feaafa4b58e82f261e5419f39c60cb',
-        'info_dict': {
-            'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431',
-            'ext': 'mp4',
-            'title': 'Music Video 14 british euro brit european cumshots swallow',
-            'uploader': 'anonim',
-            'view_count': int,
-            'age_limit': 18,
-        }
-    }, {
-        'url': 'http://www.extremetube.com/gay/video/abcde-1234',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.extremetube.com/video/latina-slut-fucked-by-fat-black-dick',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.extremetube.com/video/652431',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        webpage, info = self._extract_info(url)
-
-        if not info['title']:
-            info['title'] = self._search_regex(
-                r'<h1[^>]+title="([^"]+)"[^>]*>', webpage, 'title')
-
-        uploader = self._html_search_regex(
-            r'Uploaded by:\s*</[^>]+>\s*<a[^>]+>(.+?)</a>',
-            webpage, 'uploader', fatal=False)
-        view_count = str_to_int(self._search_regex(
-            r'Views:\s*</[^>]+>\s*<[^>]+>([\d,\.]+)</',
-            webpage, 'view count', fatal=False))
-
-        info.update({
-            'uploader': uploader,
-            'view_count': view_count,
-        })
-
-        return info
diff --git a/yt_dlp/extractor/fourzerostudio.py b/yt_dlp/extractor/fourzerostudio.py
deleted file mode 100644
index c388a3a07..000000000
--- a/yt_dlp/extractor/fourzerostudio.py
+++ /dev/null
@@ -1,106 +0,0 @@
-from .common import InfoExtractor
-from ..utils import traverse_obj, unified_timestamp
-
-
-class FourZeroStudioArchiveIE(InfoExtractor):
-    _VALID_URL = r'https?://0000\.studio/(?P<uploader_id>[^/]+)/broadcasts/(?P<id>[^/]+)/archive'
-    IE_NAME = '0000studio:archive'
-    _TESTS = [{
-        'url': 'https://0000.studio/mumeijiten/broadcasts/1290f433-fce0-4909-a24a-5f7df09665dc/archive',
-        'info_dict': {
-            'id': '1290f433-fce0-4909-a24a-5f7df09665dc',
-            'title': 'noteで『canape』様へのファンレターを執筆します。（数秘術その2）',
-            'timestamp': 1653802534,
-            'release_timestamp': 1653796604,
-            'thumbnails': 'count:1',
-            'comments': 'count:7',
-            'uploader': '『中崎雄心』の執務室。',
-            'uploader_id': 'mumeijiten',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id, uploader_id = self._match_valid_url(url).group('id', 'uploader_id')
-        webpage = self._download_webpage(url, video_id)
-        nuxt_data = self._search_nuxt_data(webpage, video_id, traverse=None)
-
-        pcb = traverse_obj(nuxt_data, ('ssrRefs', lambda _, v: v['__typename'] == 'PublicCreatorBroadcast'), get_all=False)
-        uploader_internal_id = traverse_obj(nuxt_data, (
-            'ssrRefs', lambda _, v: v['__typename'] == 'PublicUser', 'id'), get_all=False)
-
-        formats, subs = self._extract_m3u8_formats_and_subtitles(pcb['archiveUrl'], video_id, ext='mp4')
-
-        return {
-            'id': video_id,
-            'title': pcb.get('title'),
-            'age_limit': 18 if pcb.get('isAdult') else None,
-            'timestamp': unified_timestamp(pcb.get('finishTime')),
-            'release_timestamp': unified_timestamp(pcb.get('createdAt')),
-            'thumbnails': [{
-                'url': pcb['thumbnailUrl'],
-                'ext': 'png',
-            }] if pcb.get('thumbnailUrl') else None,
-            'formats': formats,
-            'subtitles': subs,
-            'comments': [{
-                'author': c.get('username'),
-                'author_id': c.get('postedUserId'),
-                'author_thumbnail': c.get('userThumbnailUrl'),
-                'id': c.get('id'),
-                'text': c.get('body'),
-                'timestamp': unified_timestamp(c.get('createdAt')),
-                'like_count': c.get('likeCount'),
-                'is_favorited': c.get('isLikedByOwner'),
-                'author_is_uploader': c.get('postedUserId') == uploader_internal_id,
-            } for c in traverse_obj(nuxt_data, (
-                'ssrRefs', ..., lambda _, v: v['__typename'] == 'PublicCreatorBroadcastComment')) or []],
-            'uploader_id': uploader_id,
-            'uploader': traverse_obj(nuxt_data, (
-                'ssrRefs', lambda _, v: v['__typename'] == 'PublicUser', 'username'), get_all=False),
-        }
-
-
-class FourZeroStudioClipIE(InfoExtractor):
-    _VALID_URL = r'https?://0000\.studio/(?P<uploader_id>[^/]+)/archive-clip/(?P<id>[^/]+)'
-    IE_NAME = '0000studio:clip'
-    _TESTS = [{
-        'url': 'https://0000.studio/soeji/archive-clip/e46b0278-24cd-40a8-92e1-b8fc2b21f34f',
-        'info_dict': {
-            'id': 'e46b0278-24cd-40a8-92e1-b8fc2b21f34f',
-            'title': 'わたベーさんからイラスト差し入れいただきました。ありがとうございました！',
-            'timestamp': 1652109105,
-            'like_count': 1,
-            'uploader': 'ソエジマケイタ',
-            'uploader_id': 'soeji',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id, uploader_id = self._match_valid_url(url).group('id', 'uploader_id')
-        webpage = self._download_webpage(url, video_id)
-        nuxt_data = self._search_nuxt_data(webpage, video_id, traverse=None)
-
-        clip_info = traverse_obj(nuxt_data, ('ssrRefs', lambda _, v: v['__typename'] == 'PublicCreatorArchivedClip'), get_all=False)
-
-        info = next((
-            m for m in self._parse_html5_media_entries(url, webpage, video_id)
-            if 'mp4' in traverse_obj(m, ('formats', ..., 'ext'))
-        ), None)
-        if not info:
-            self.report_warning('Failed to find a desired media element. Falling back to using NUXT data.')
-            info = {
-                'formats': [{
-                    'ext': 'mp4',
-                    'url': url,
-                } for url in clip_info.get('mediaFiles') or [] if url],
-            }
-        return {
-            **info,
-            'id': video_id,
-            'title': clip_info.get('clipComment'),
-            'timestamp': unified_timestamp(clip_info.get('createdAt')),
-            'like_count': clip_info.get('likeCount'),
-            'uploader_id': uploader_id,
-            'uploader': traverse_obj(nuxt_data, (
-                'ssrRefs', lambda _, v: v['__typename'] == 'PublicUser', 'username'), get_all=False),
-        }
diff --git a/yt_dlp/extractor/foxgay.py b/yt_dlp/extractor/foxgay.py
deleted file mode 100644
index f4f29c65d..000000000
--- a/yt_dlp/extractor/foxgay.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import itertools
-
-from .common import InfoExtractor
-from ..utils import (
-    get_element_by_id,
-    int_or_none,
-    remove_end,
-)
-
-
-class FoxgayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
-    _TEST = {
-        'url': 'http://foxgay.com/videos/fuck-turkish-style-2582.shtml',
-        'md5': '344558ccfea74d33b7adbce22e577f54',
-        'info_dict': {
-            'id': '2582',
-            'ext': 'mp4',
-            'title': 'Fuck Turkish-style',
-            'description': 'md5:6ae2d9486921891efe89231ace13ffdf',
-            'age_limit': 18,
-            'thumbnail': r're:https?://.*\.jpg$',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = remove_end(self._html_extract_title(webpage), ' - Foxgay.com')
-        description = get_element_by_id('inf_tit', webpage)
-
-        # The default user-agent with foxgay cookies leads to pages without videos
-        self.cookiejar.clear('.foxgay.com')
-        # Find the URL for the iFrame which contains the actual video.
-        iframe_url = self._html_search_regex(
-            r'<iframe[^>]+src=([\'"])(?P<url>[^\'"]+)\1', webpage,
-            'video frame', group='url')
-        iframe = self._download_webpage(
-            iframe_url, video_id, headers={'User-Agent': 'curl/7.50.1'},
-            note='Downloading video frame')
-        video_data = self._parse_json(self._search_regex(
-            r'video_data\s*=\s*([^;]+);', iframe, 'video data'), video_id)
-
-        formats = [{
-            'url': source,
-            'height': int_or_none(resolution),
-        } for source, resolution in zip(
-            video_data['sources'], video_data.get('resolutions', itertools.repeat(None)))]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'description': description,
-            'thumbnail': video_data.get('act_vid', {}).get('thumb'),
-            'age_limit': 18,
-        }
diff --git a/yt_dlp/extractor/fusion.py b/yt_dlp/extractor/fusion.py
deleted file mode 100644
index 689422fca..000000000
--- a/yt_dlp/extractor/fusion.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-    int_or_none,
-    mimetype2ext,
-    parse_iso8601,
-)
-
-
-class FusionIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?fusion\.(?:net|tv)/(?:video/|show/.+?\bvideo=)(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://fusion.tv/video/201781/u-s-and-panamanian-forces-work-together-to-stop-a-vessel-smuggling-drugs/',
-        'info_dict': {
-            'id': '3145868',
-            'ext': 'mp4',
-            'title': 'U.S. and Panamanian forces work together to stop a vessel smuggling drugs',
-            'description': 'md5:0cc84a9943c064c0f46b128b41b1b0d7',
-            'duration': 140.0,
-            'timestamp': 1442589635,
-            'uploader': 'UNIVISON',
-            'upload_date': '20150918',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'add_ie': ['Anvato'],
-    }, {
-        'url': 'http://fusion.tv/video/201781',
-        'only_matching': True,
-    }, {
-        'url': 'https://fusion.tv/show/food-exposed-with-nelufar-hedayat/?ancla=full-episodes&video=588644',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        video = self._download_json(
-            'https://platform.fusion.net/wp-json/fusiondotnet/v1/video/' + video_id, video_id)
-
-        info = {
-            'id': video_id,
-            'title': video['title'],
-            'description': video.get('excerpt'),
-            'timestamp': parse_iso8601(video.get('published')),
-            'series': video.get('show'),
-        }
-
-        formats = []
-        src = video.get('src') or {}
-        for f_id, f in src.items():
-            for q_id, q in f.items():
-                q_url = q.get('url')
-                if not q_url:
-                    continue
-                ext = determine_ext(q_url, mimetype2ext(q.get('type')))
-                if ext == 'smil':
-                    formats.extend(self._extract_smil_formats(q_url, video_id, fatal=False))
-                elif f_id == 'm3u8-variant' or (ext == 'm3u8' and q_id == 'Variant'):
-                    formats.extend(self._extract_m3u8_formats(
-                        q_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-                else:
-                    formats.append({
-                        'format_id': '-'.join([f_id, q_id]),
-                        'url': q_url,
-                        'width': int_or_none(q.get('width')),
-                        'height': int_or_none(q.get('height')),
-                        'tbr': int_or_none(self._search_regex(r'_(\d+)\.m(?:p4|3u8)', q_url, 'bitrate')),
-                        'ext': 'mp4' if ext == 'm3u8' else ext,
-                        'protocol': 'm3u8_native' if ext == 'm3u8' else 'https',
-                    })
-        if formats:
-            info['formats'] = formats
-        else:
-            info.update({
-                '_type': 'url',
-                'url': 'anvato:uni:' + video['video_ids']['anvato'],
-                'ie_key': 'Anvato',
-            })
-
-        return info
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 1503e5146..606b4f5d1 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -374,46 +374,6 @@ class GenericIE(InfoExtractor):
             },
             'skip': 'There is a limit of 200 free downloads / month for the test song',
         },
-        # ooyala video
-        {
-            'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
-            'md5': '166dd577b433b4d4ebfee10b0824d8ff',
-            'info_dict': {
-                'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
-                'ext': 'mp4',
-                'title': '2cc213299525360.mov',  # that's what we get
-                'duration': 238.231,
-            },
-            'add_ie': ['Ooyala'],
-        },
-        {
-            # ooyala video embedded with http://player.ooyala.com/iframe.js
-            'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
-            'info_dict': {
-                'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
-                'ext': 'mp4',
-                'title': '"Steve Jobs: Man in the Machine" trailer',
-                'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
-                'duration': 135.427,
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'movie expired',
-        },
-        # ooyala video embedded with http://player.ooyala.com/static/v4/production/latest/core.min.js
-        {
-            'url': 'http://wnep.com/2017/07/22/steampunk-fest-comes-to-honesdale/',
-            'info_dict': {
-                'id': 'lwYWYxYzE6V5uJMjNGyKtwwiw9ZJD7t2',
-                'ext': 'mp4',
-                'title': 'Steampunk Fest Comes to Honesdale',
-                'duration': 43.276,
-            },
-            'params': {
-                'skip_download': True,
-            }
-        },
         # embed.ly video
         {
             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
@@ -506,7 +466,8 @@ class GenericIE(InfoExtractor):
                 'title': 'Ужастики, русский трейлер (2015)',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 153,
-            }
+            },
+            'skip': 'Site dead',
         },
         # XHamster embed
         {
@@ -778,14 +739,16 @@ class GenericIE(InfoExtractor):
             'playlist_mincount': 1,
             'add_ie': ['Youtube'],
         },
-        # Cinchcast embed
+        # Libsyn embed
         {
             'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
             'info_dict': {
-                'id': '7141703',
+                'id': '3793998',
                 'ext': 'mp3',
                 'upload_date': '20141126',
-                'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
+                'title': 'Underground Wellness Radio - Jack Tips: 5 Steps to Permanent Gut Healing',
+                'thumbnail': 'https://assets.libsyn.com/secure/item/3793998/?height=90&width=90',
+                'duration': 3989.0,
             }
         },
         # Cinerama player
@@ -1567,16 +1530,6 @@ class GenericIE(InfoExtractor):
                 'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
             },
         },
-        {
-            # vzaar embed
-            'url': 'http://help.vzaar.com/article/165-embedding-video',
-            'md5': '7e3919d9d2620b89e3e00bec7fe8c9d4',
-            'info_dict': {
-                'id': '8707641',
-                'ext': 'mp4',
-                'title': 'Building A Business Online: Principal Chairs Q & A',
-            },
-        },
         {
             # multiple HTML5 videos on one page
             'url': 'https://www.paragon-software.com/home/rk-free/keyscenarios.html',
diff --git a/yt_dlp/extractor/gfycat.py b/yt_dlp/extractor/gfycat.py
deleted file mode 100644
index edc2e56e4..000000000
--- a/yt_dlp/extractor/gfycat.py
+++ /dev/null
@@ -1,145 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    float_or_none,
-    qualities,
-    ExtractorError,
-)
-
-
-class GfycatIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|giant|thumbs)\.)?gfycat\.com/(?i:ru/|ifr/|gifs/detail/)?(?P<id>[^-/?#\."\']+)'
-    _EMBED_REGEX = [rf'<(?:iframe|source)[^>]+\bsrc=["\'](?P<url>{_VALID_URL})']
-    _TESTS = [{
-        'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
-        'info_dict': {
-            'id': 'DeadlyDecisiveGermanpinscher',
-            'ext': 'mp4',
-            'title': 'Ghost in the Shell',
-            'timestamp': 1410656006,
-            'upload_date': '20140914',
-            'uploader': 'anonymous',
-            'duration': 10.4,
-            'view_count': int,
-            'like_count': int,
-            'categories': list,
-            'age_limit': 0,
-            'uploader_id': 'anonymous',
-            'description': '',
-        }
-    }, {
-        'url': 'http://gfycat.com/ifr/JauntyTimelyAmazontreeboa',
-        'info_dict': {
-            'id': 'JauntyTimelyAmazontreeboa',
-            'ext': 'mp4',
-            'title': 'JauntyTimelyAmazontreeboa',
-            'timestamp': 1411720126,
-            'upload_date': '20140926',
-            'uploader': 'anonymous',
-            'duration': 3.52,
-            'view_count': int,
-            'like_count': int,
-            'categories': list,
-            'age_limit': 0,
-            'uploader_id': 'anonymous',
-            'description': '',
-        }
-    }, {
-        'url': 'https://gfycat.com/alienatedsolidgreathornedowl',
-        'info_dict': {
-            'id': 'alienatedsolidgreathornedowl',
-            'ext': 'mp4',
-            'upload_date': '20211226',
-            'uploader_id': 'reactions',
-            'timestamp': 1640536930,
-            'like_count': int,
-            'description': '',
-            'title': 'Ingrid Michaelson, Zooey Deschanel - Merry Christmas Happy New Year',
-            'categories': list,
-            'age_limit': 0,
-            'duration': 2.9583333333333335,
-            'uploader': 'Reaction GIFs',
-            'view_count': int,
-        }
-    }, {
-        'url': 'https://gfycat.com/ru/RemarkableDrearyAmurstarfish',
-        'only_matching': True
-    }, {
-        'url': 'https://gfycat.com/gifs/detail/UnconsciousLankyIvorygull',
-        'only_matching': True
-    }, {
-        'url': 'https://gfycat.com/acceptablehappygoluckyharborporpoise-baseball',
-        'only_matching': True
-    }, {
-        'url': 'https://thumbs.gfycat.com/acceptablehappygoluckyharborporpoise-size_restricted.gif',
-        'only_matching': True
-    }, {
-        'url': 'https://giant.gfycat.com/acceptablehappygoluckyharborporpoise.mp4',
-        'only_matching': True
-    }, {
-        'url': 'http://gfycat.com/IFR/JauntyTimelyAmazontreeboa',
-        'only_matching': True
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        gfy = self._download_json(
-            'https://api.gfycat.com/v1/gfycats/%s' % video_id,
-            video_id, 'Downloading video info')
-        if 'error' in gfy:
-            raise ExtractorError('Gfycat said: ' + gfy['error'], expected=True)
-        gfy = gfy['gfyItem']
-
-        title = gfy.get('title') or gfy['gfyName']
-        description = gfy.get('description')
-        timestamp = int_or_none(gfy.get('createDate'))
-        uploader = gfy.get('userName') or gfy.get('username')
-        view_count = int_or_none(gfy.get('views'))
-        like_count = int_or_none(gfy.get('likes'))
-        dislike_count = int_or_none(gfy.get('dislikes'))
-        age_limit = 18 if gfy.get('nsfw') == '1' else 0
-
-        width = int_or_none(gfy.get('width'))
-        height = int_or_none(gfy.get('height'))
-        fps = int_or_none(gfy.get('frameRate'))
-        num_frames = int_or_none(gfy.get('numFrames'))
-
-        duration = float_or_none(num_frames, fps) if num_frames and fps else None
-
-        categories = gfy.get('tags') or gfy.get('extraLemmas') or []
-
-        FORMATS = ('gif', 'webm', 'mp4')
-        quality = qualities(FORMATS)
-
-        formats = []
-        for format_id in FORMATS:
-            video_url = gfy.get('%sUrl' % format_id)
-            if not video_url:
-                continue
-            filesize = int_or_none(gfy.get('%sSize' % format_id))
-            formats.append({
-                'url': video_url,
-                'format_id': format_id,
-                'width': width,
-                'height': height,
-                'fps': fps,
-                'filesize': filesize,
-                'quality': quality(format_id),
-            })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-            'uploader': gfy.get('userDisplayName') or uploader,
-            'uploader_id': uploader,
-            'duration': duration,
-            'view_count': view_count,
-            'like_count': like_count,
-            'dislike_count': dislike_count,
-            'categories': categories,
-            'age_limit': age_limit,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/groupon.py b/yt_dlp/extractor/groupon.py
index 362d3ff83..c1cbda35f 100644
--- a/yt_dlp/extractor/groupon.py
+++ b/yt_dlp/extractor/groupon.py
@@ -31,7 +31,6 @@ class GrouponIE(InfoExtractor):
     }
 
     _PROVIDERS = {
-        'ooyala': ('ooyala:%s', 'Ooyala'),
         'youtube': ('%s', 'Youtube'),
     }
 
diff --git a/yt_dlp/extractor/helsinki.py b/yt_dlp/extractor/helsinki.py
deleted file mode 100644
index e518cae1a..000000000
--- a/yt_dlp/extractor/helsinki.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from .common import InfoExtractor
-from ..utils import js_to_json
-
-
-class HelsinkiIE(InfoExtractor):
-    IE_DESC = 'helsinki.fi'
-    _VALID_URL = r'https?://video\.helsinki\.fi/Arkisto/flash\.php\?id=(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://video.helsinki.fi/Arkisto/flash.php?id=20258',
-        'info_dict': {
-            'id': '20258',
-            'ext': 'mp4',
-            'title': 'Tietotekniikkafoorumi-iltapäivä',
-            'description': 'md5:f5c904224d43c133225130fe156a5ee0',
-        },
-        'params': {
-            'skip_download': True,  # RTMP
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        params = self._parse_json(self._html_search_regex(
-            r'(?s)jwplayer\("player"\).setup\((\{.*?\})\);',
-            webpage, 'player code'), video_id, transform_source=js_to_json)
-        formats = [{
-            'url': s['file'],
-            'ext': 'mp4',
-        } for s in params['sources']]
-
-        return {
-            'id': video_id,
-            'title': self._og_search_title(webpage).replace('Video: ', ''),
-            'description': self._og_search_description(webpage),
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/hitbox.py b/yt_dlp/extractor/hitbox.py
deleted file mode 100644
index f0c689883..000000000
--- a/yt_dlp/extractor/hitbox.py
+++ /dev/null
@@ -1,209 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    clean_html,
-    determine_ext,
-    float_or_none,
-    int_or_none,
-    parse_iso8601,
-)
-
-
-class HitboxIE(InfoExtractor):
-    IE_NAME = 'hitbox'
-    _VALID_URL = r'https?://(?:www\.)?(?:hitbox|smashcast)\.tv/(?:[^/]+/)*videos?/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.hitbox.tv/video/203213',
-        'info_dict': {
-            'id': '203213',
-            'title': 'hitbox @ gamescom, Sub Button Hype extended, Giveaway - hitbox News Update with Oxy',
-            'alt_title': 'hitboxlive - Aug 9th #6',
-            'description': '',
-            'ext': 'mp4',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 215.1666,
-            'resolution': 'HD 720p',
-            'uploader': 'hitboxlive',
-            'view_count': int,
-            'timestamp': 1407576133,
-            'upload_date': '20140809',
-            'categories': ['Live Show'],
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://www.smashcast.tv/hitboxlive/videos/203213',
-        'only_matching': True,
-    }]
-
-    def _extract_metadata(self, url, video_id):
-        thumb_base = 'https://edge.sf.hitbox.tv'
-        metadata = self._download_json(
-            '%s/%s' % (url, video_id), video_id, 'Downloading metadata JSON')
-
-        date = 'media_live_since'
-        media_type = 'livestream'
-        if metadata.get('media_type') == 'video':
-            media_type = 'video'
-            date = 'media_date_added'
-
-        video_meta = metadata.get(media_type, [])[0]
-        title = video_meta.get('media_status')
-        alt_title = video_meta.get('media_title')
-        description = clean_html(
-            video_meta.get('media_description')
-            or video_meta.get('media_description_md'))
-        duration = float_or_none(video_meta.get('media_duration'))
-        uploader = video_meta.get('media_user_name')
-        views = int_or_none(video_meta.get('media_views'))
-        timestamp = parse_iso8601(video_meta.get(date), ' ')
-        categories = [video_meta.get('category_name')]
-        thumbs = [{
-            'url': thumb_base + video_meta.get('media_thumbnail'),
-            'width': 320,
-            'height': 180
-        }, {
-            'url': thumb_base + video_meta.get('media_thumbnail_large'),
-            'width': 768,
-            'height': 432
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'alt_title': alt_title,
-            'description': description,
-            'ext': 'mp4',
-            'thumbnails': thumbs,
-            'duration': duration,
-            'uploader': uploader,
-            'view_count': views,
-            'timestamp': timestamp,
-            'categories': categories,
-        }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        player_config = self._download_json(
-            'https://www.smashcast.tv/api/player/config/video/%s' % video_id,
-            video_id, 'Downloading video JSON')
-
-        formats = []
-        for video in player_config['clip']['bitrates']:
-            label = video.get('label')
-            if label == 'Auto':
-                continue
-            video_url = video.get('url')
-            if not video_url:
-                continue
-            bitrate = int_or_none(video.get('bitrate'))
-            if determine_ext(video_url) == 'm3u8':
-                if not video_url.startswith('http'):
-                    continue
-                formats.append({
-                    'url': video_url,
-                    'ext': 'mp4',
-                    'tbr': bitrate,
-                    'format_note': label,
-                    'protocol': 'm3u8_native',
-                })
-            else:
-                formats.append({
-                    'url': video_url,
-                    'tbr': bitrate,
-                    'format_note': label,
-                })
-
-        metadata = self._extract_metadata(
-            'https://www.smashcast.tv/api/media/video', video_id)
-        metadata['formats'] = formats
-
-        return metadata
-
-
-class HitboxLiveIE(HitboxIE):  # XXX: Do not subclass from concrete IE
-    IE_NAME = 'hitbox:live'
-    _VALID_URL = r'https?://(?:www\.)?(?:hitbox|smashcast)\.tv/(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'http://www.hitbox.tv/dimak',
-        'info_dict': {
-            'id': 'dimak',
-            'ext': 'mp4',
-            'description': 'md5:c9f80fa4410bc588d7faa40003fc7d0e',
-            'timestamp': int,
-            'upload_date': compat_str,
-            'title': compat_str,
-            'uploader': 'Dimak',
-        },
-        'params': {
-            # live
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://www.smashcast.tv/dimak',
-        'only_matching': True,
-    }]
-
-    @classmethod
-    def suitable(cls, url):
-        return False if HitboxIE.suitable(url) else super(HitboxLiveIE, cls).suitable(url)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        player_config = self._download_json(
-            'https://www.smashcast.tv/api/player/config/live/%s' % video_id,
-            video_id)
-
-        formats = []
-        cdns = player_config.get('cdns')
-        servers = []
-        for cdn in cdns:
-            # Subscribe URLs are not playable
-            if cdn.get('rtmpSubscribe') is True:
-                continue
-            base_url = cdn.get('netConnectionUrl')
-            host = re.search(r'.+\.([^\.]+\.[^\./]+)/.+', base_url).group(1)
-            if base_url not in servers:
-                servers.append(base_url)
-                for stream in cdn.get('bitrates'):
-                    label = stream.get('label')
-                    if label == 'Auto':
-                        continue
-                    stream_url = stream.get('url')
-                    if not stream_url:
-                        continue
-                    bitrate = int_or_none(stream.get('bitrate'))
-                    if stream.get('provider') == 'hls' or determine_ext(stream_url) == 'm3u8':
-                        if not stream_url.startswith('http'):
-                            continue
-                        formats.append({
-                            'url': stream_url,
-                            'ext': 'mp4',
-                            'tbr': bitrate,
-                            'format_note': label,
-                            'rtmp_live': True,
-                        })
-                    else:
-                        formats.append({
-                            'url': '%s/%s' % (base_url, stream_url),
-                            'ext': 'mp4',
-                            'tbr': bitrate,
-                            'rtmp_live': True,
-                            'format_note': host,
-                            'page_url': url,
-                            'player_url': 'http://www.hitbox.tv/static/player/flowplayer/flowplayer.commercial-3.2.16.swf',
-                        })
-
-        metadata = self._extract_metadata(
-            'https://www.smashcast.tv/api/media/live', video_id)
-        metadata['formats'] = formats
-        metadata['is_live'] = True
-        metadata['title'] = metadata.get('title')
-
-        return metadata
diff --git a/yt_dlp/extractor/howcast.py b/yt_dlp/extractor/howcast.py
deleted file mode 100644
index 59cf80f1a..000000000
--- a/yt_dlp/extractor/howcast.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from .common import InfoExtractor
-from ..utils import parse_iso8601
-
-
-class HowcastIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?howcast\.com/videos/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.howcast.com/videos/390161-How-to-Tie-a-Square-Knot-Properly',
-        'md5': '7d45932269a288149483144f01b99789',
-        'info_dict': {
-            'id': '390161',
-            'ext': 'mp4',
-            'title': 'How to Tie a Square Knot Properly',
-            'description': 'md5:dbe792e5f6f1489027027bf2eba188a3',
-            'timestamp': 1276081287,
-            'upload_date': '20100609',
-            'duration': 56.823,
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'add_ie': ['Ooyala'],
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        embed_code = self._search_regex(
-            r'<iframe[^>]+src="[^"]+\bembed_code=([^\b]+)\b',
-            webpage, 'ooyala embed code')
-
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'Ooyala',
-            'url': 'ooyala:%s' % embed_code,
-            'id': video_id,
-            'timestamp': parse_iso8601(self._html_search_meta(
-                'article:published_time', webpage, 'timestamp')),
-        }
diff --git a/yt_dlp/extractor/howstuffworks.py b/yt_dlp/extractor/howstuffworks.py
deleted file mode 100644
index 238fc0b42..000000000
--- a/yt_dlp/extractor/howstuffworks.py
+++ /dev/null
@@ -1,86 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    find_xpath_attr,
-    int_or_none,
-    js_to_json,
-    unescapeHTML,
-    determine_ext,
-)
-
-
-class HowStuffWorksIE(InfoExtractor):
-    _VALID_URL = r'https?://[\da-z-]+\.(?:howstuffworks|stuff(?:(?:youshould|theydontwantyouto)know|toblowyourmind|momnevertoldyou)|(?:brain|car)stuffshow|fwthinking|geniusstuff)\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
-    _TESTS = [
-        {
-            'url': 'http://www.stufftoblowyourmind.com/videos/optical-illusions-video.htm',
-            'md5': '76646a5acc0c92bf7cd66751ca5db94d',
-            'info_dict': {
-                'id': '855410',
-                'ext': 'mp4',
-                'title': 'Your Trickster Brain: Optical Illusions -- Science on the Web',
-                'description': 'md5:e374ff9561f6833ad076a8cc0a5ab2fb',
-            },
-        },
-        {
-            'url': 'http://shows.howstuffworks.com/more-shows/why-does-balloon-stick-to-hair-video.htm',
-            'only_matching': True,
-        }
-    ]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        clip_js = self._search_regex(
-            r'(?s)var clip = ({.*?});', webpage, 'clip info')
-        clip_info = self._parse_json(
-            clip_js, display_id, transform_source=js_to_json)
-
-        video_id = clip_info['content_id']
-        formats = []
-        m3u8_url = clip_info.get('m3u8')
-        if m3u8_url and determine_ext(m3u8_url) == 'm3u8':
-            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', format_id='hls', fatal=True))
-        flv_url = clip_info.get('flv_url')
-        if flv_url:
-            formats.append({
-                'url': flv_url,
-                'format_id': 'flv',
-            })
-        for video in clip_info.get('mp4', []):
-            formats.append({
-                'url': video['src'],
-                'format_id': 'mp4-%s' % video['bitrate'],
-                'vbr': int_or_none(video['bitrate'].rstrip('k')),
-            })
-
-        if not formats:
-            smil = self._download_xml(
-                'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % video_id,
-                video_id, 'Downloading video SMIL')
-
-            http_base = find_xpath_attr(
-                smil,
-                './{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'),
-                'name',
-                'httpBase').get('content')
-
-            URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=A&g=A'
-
-            for video in smil.findall(
-                    './{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
-                vbr = int_or_none(video.attrib['system-bitrate'], scale=1000)
-                formats.append({
-                    'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
-                    'format_id': '%dk' % vbr,
-                    'vbr': vbr,
-                })
-
-        return {
-            'id': '%s' % video_id,
-            'display_id': display_id,
-            'title': unescapeHTML(clip_info['clip_title']),
-            'description': unescapeHTML(clip_info.get('caption')),
-            'thumbnail': clip_info.get('video_still_url'),
-            'duration': int_or_none(clip_info.get('duration')),
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/keezmovies.py b/yt_dlp/extractor/keezmovies.py
deleted file mode 100644
index b50da420c..000000000
--- a/yt_dlp/extractor/keezmovies.py
+++ /dev/null
@@ -1,125 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..aes import aes_decrypt_text
-from ..compat import compat_urllib_parse_unquote
-from ..utils import (
-    determine_ext,
-    format_field,
-    int_or_none,
-    str_to_int,
-    strip_or_none,
-    url_or_none,
-)
-
-
-class KeezMoviesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/(?:(?P<display_id>[^/]+)-)?(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://www.keezmovies.com/video/arab-wife-want-it-so-bad-i-see-she-thirsty-and-has-tiny-money-18070681',
-        'md5': '2ac69cdb882055f71d82db4311732a1a',
-        'info_dict': {
-            'id': '18070681',
-            'display_id': 'arab-wife-want-it-so-bad-i-see-she-thirsty-and-has-tiny-money',
-            'ext': 'mp4',
-            'title': 'Arab wife want it so bad I see she thirsty and has tiny money.',
-            'thumbnail': None,
-            'view_count': int,
-            'age_limit': 18,
-        }
-    }, {
-        'url': 'http://www.keezmovies.com/video/18070681',
-        'only_matching': True,
-    }]
-
-    def _extract_info(self, url, fatal=True):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-        display_id = (mobj.group('display_id')
-                      if 'display_id' in mobj.groupdict()
-                      else None) or mobj.group('id')
-
-        webpage = self._download_webpage(
-            url, display_id, headers={'Cookie': 'age_verified=1'})
-
-        formats = []
-        format_urls = set()
-
-        title = None
-        thumbnail = None
-        duration = None
-        encrypted = False
-
-        def extract_format(format_url, height=None):
-            format_url = url_or_none(format_url)
-            if not format_url or not format_url.startswith(('http', '//')):
-                return
-            if format_url in format_urls:
-                return
-            format_urls.add(format_url)
-            tbr = int_or_none(self._search_regex(
-                r'[/_](\d+)[kK][/_]', format_url, 'tbr', default=None))
-            if not height:
-                height = int_or_none(self._search_regex(
-                    r'[/_](\d+)[pP][/_]', format_url, 'height', default=None))
-            if encrypted:
-                format_url = aes_decrypt_text(
-                    video_url, title, 32).decode('utf-8')
-            formats.append({
-                'url': format_url,
-                'format_id': format_field(height, None, '%dp'),
-                'height': height,
-                'tbr': tbr,
-            })
-
-        flashvars = self._parse_json(
-            self._search_regex(
-                r'flashvars\s*=\s*({.+?});', webpage,
-                'flashvars', default='{}'),
-            display_id, fatal=False)
-
-        if flashvars:
-            title = flashvars.get('video_title')
-            thumbnail = flashvars.get('image_url')
-            duration = int_or_none(flashvars.get('video_duration'))
-            encrypted = flashvars.get('encrypted') is True
-            for key, value in flashvars.items():
-                mobj = re.search(r'quality_(\d+)[pP]', key)
-                if mobj:
-                    extract_format(value, int(mobj.group(1)))
-            video_url = flashvars.get('video_url')
-            if video_url and determine_ext(video_url, None):
-                extract_format(video_url)
-
-        video_url = self._html_search_regex(
-            r'flashvars\.video_url\s*=\s*(["\'])(?P<url>http.+?)\1',
-            webpage, 'video url', default=None, group='url')
-        if video_url:
-            extract_format(compat_urllib_parse_unquote(video_url))
-
-        if not formats:
-            if 'title="This video is no longer available"' in webpage:
-                self.raise_no_formats(
-                    'Video %s is no longer available' % video_id, expected=True)
-
-        if not title:
-            title = self._html_search_regex(
-                r'<h1[^>]*>([^<]+)', webpage, 'title')
-
-        return webpage, {
-            'id': video_id,
-            'display_id': display_id,
-            'title': strip_or_none(title),
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'age_limit': 18,
-            'formats': formats,
-        }
-
-    def _real_extract(self, url):
-        webpage, info = self._extract_info(url, fatal=False)
-        if not info['formats']:
-            return self.url_result(url, 'Generic')
-        info['view_count'] = str_to_int(self._search_regex(
-            r'<b>([\d,.]+)</b> Views?', webpage, 'view count', fatal=False))
-        return info
diff --git a/yt_dlp/extractor/kinja.py b/yt_dlp/extractor/kinja.py
index df1386fb8..a225d0a0d 100644
--- a/yt_dlp/extractor/kinja.py
+++ b/yt_dlp/extractor/kinja.py
@@ -41,7 +41,6 @@ class KinjaEmbedIE(InfoExtractor):
             kinjavideo|
             mcp|
             megaphone|
-            ooyala|
             soundcloud(?:-playlist)?|
             tumblr-post|
             twitch-stream|
@@ -61,9 +60,6 @@ class KinjaEmbedIE(InfoExtractor):
     }, {
         'url': 'https://kinja.com/ajax/inset/iframe?id=megaphone-PPY1300931075',
         'only_matching': True,
-    }, {
-        'url': 'https://kinja.com/ajax/inset/iframe?id=ooyala-xzMXhleDpopuT0u1ijt_qZj3Va-34pEX%2FZTIxYmJjZDM2NWYzZDViZGRiOWJjYzc5',
-        'only_matching': True,
     }, {
         'url': 'https://kinja.com/ajax/inset/iframe?id=soundcloud-128574047',
         'only_matching': True,
@@ -103,7 +99,6 @@ class KinjaEmbedIE(InfoExtractor):
         'jwplayer-video': _JWPLATFORM_PROVIDER,
         'jwp-video': _JWPLATFORM_PROVIDER,
         'megaphone': ('player.megaphone.fm/', 'Generic'),
-        'ooyala': ('player.ooyala.com/player.js?embedCode=', 'Ooyala'),
         'soundcloud': ('api.soundcloud.com/tracks/', 'Soundcloud'),
         'soundcloud-playlist': ('api.soundcloud.com/playlists/', 'SoundcloudPlaylist'),
         'tumblr-post': ('%s.tumblr.com/post/%s', 'Tumblr'),
@@ -129,8 +124,6 @@ def _real_extract(self, url):
                 video_id, playlist_id = video_id.split('/')
                 result_url = provider[0] % (video_id, playlist_id)
             else:
-                if video_type == 'ooyala':
-                    video_id = video_id.split('/')[0]
                 result_url = provider[0] + video_id
             return self.url_result('http://' + result_url, provider[1])
 
diff --git a/yt_dlp/extractor/laola1tv.py b/yt_dlp/extractor/laola1tv.py
deleted file mode 100644
index 416dd7eb4..000000000
--- a/yt_dlp/extractor/laola1tv.py
+++ /dev/null
@@ -1,261 +0,0 @@
-import json
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    unified_strdate,
-    urlencode_postdata,
-    xpath_element,
-    xpath_text,
-    update_url_query,
-    js_to_json,
-)
-
-
-class Laola1TvEmbedIE(InfoExtractor):
-    IE_NAME = 'laola1tv:embed'
-    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/titanplayer\.php\?.*?\bvideoid=(?P<id>\d+)'
-    _TESTS = [{
-        # flashvars.premium = "false";
-        'url': 'https://www.laola1.tv/titanplayer.php?videoid=708065&type=V&lang=en&portal=int&customer=1024',
-        'info_dict': {
-            'id': '708065',
-            'ext': 'mp4',
-            'title': 'MA Long CHN - FAN Zhendong CHN',
-            'uploader': 'ITTF - International Table Tennis Federation',
-            'upload_date': '20161211',
-        },
-    }]
-
-    def _extract_token_url(self, stream_access_url, video_id, data):
-        return self._download_json(
-            self._proto_relative_url(stream_access_url, 'https:'), video_id,
-            headers={
-                'Content-Type': 'application/json',
-            }, data=json.dumps(data).encode())['data']['stream-access'][0]
-
-    def _extract_formats(self, token_url, video_id):
-        token_doc = self._download_xml(
-            token_url, video_id, 'Downloading token',
-            headers=self.geo_verification_headers())
-
-        token_attrib = xpath_element(token_doc, './/token').attrib
-
-        if token_attrib['status'] != '0':
-            raise ExtractorError(
-                'Token error: %s' % token_attrib['comment'], expected=True)
-
-        formats = self._extract_akamai_formats(
-            '%s?hdnea=%s' % (token_attrib['url'], token_attrib['auth']),
-            video_id)
-        return formats
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        flash_vars = self._search_regex(
-            r'(?s)flashvars\s*=\s*({.+?});', webpage, 'flash vars')
-
-        def get_flashvar(x, *args, **kwargs):
-            flash_var = self._search_regex(
-                r'%s\s*:\s*"([^"]+)"' % x,
-                flash_vars, x, default=None)
-            if not flash_var:
-                flash_var = self._search_regex([
-                    r'flashvars\.%s\s*=\s*"([^"]+)"' % x,
-                    r'%s\s*=\s*"([^"]+)"' % x],
-                    webpage, x, *args, **kwargs)
-            return flash_var
-
-        hd_doc = self._download_xml(
-            'http://www.laola1.tv/server/hd_video.php', video_id, query={
-                'play': get_flashvar('streamid'),
-                'partner': get_flashvar('partnerid'),
-                'portal': get_flashvar('portalid'),
-                'lang': get_flashvar('sprache'),
-                'v5ident': '',
-            })
-
-        _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
-        title = _v('title', fatal=True)
-
-        token_url = None
-        premium = get_flashvar('premium', default=None)
-        if premium:
-            token_url = update_url_query(
-                _v('url', fatal=True), {
-                    'timestamp': get_flashvar('timestamp'),
-                    'auth': get_flashvar('auth'),
-                })
-        else:
-            data_abo = urlencode_postdata(
-                dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(','))))
-            stream_access_url = update_url_query(
-                'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access', {
-                    'videoId': _v('id'),
-                    'target': self._search_regex(r'vs_target = (\d+);', webpage, 'vs target'),
-                    'label': _v('label'),
-                    'area': _v('area'),
-                })
-            token_url = self._extract_token_url(stream_access_url, video_id, data_abo)
-
-        formats = self._extract_formats(token_url, video_id)
-
-        categories_str = _v('meta_sports')
-        categories = categories_str.split(',') if categories_str else []
-        is_live = _v('islive') == 'true'
-
-        return {
-            'id': video_id,
-            'title': title,
-            'upload_date': unified_strdate(_v('time_date')),
-            'uploader': _v('meta_organisation'),
-            'categories': categories,
-            'is_live': is_live,
-            'formats': formats,
-        }
-
-
-class Laola1TvBaseIE(Laola1TvEmbedIE):  # XXX: Do not subclass from concrete IE
-    def _extract_video(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        if 'Dieser Livestream ist bereits beendet.' in webpage:
-            raise ExtractorError('This live stream has already finished.', expected=True)
-
-        conf = self._parse_json(self._search_regex(
-            r'(?s)conf\s*=\s*({.+?});', webpage, 'conf'),
-            display_id,
-            transform_source=lambda s: js_to_json(re.sub(r'shareurl:.+,', '', s)))
-        video_id = conf['videoid']
-
-        config = self._download_json(conf['configUrl'], video_id, query={
-            'videoid': video_id,
-            'partnerid': conf['partnerid'],
-            'language': conf.get('language', ''),
-            'portal': conf.get('portalid', ''),
-        })
-        error = config.get('error')
-        if error:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
-
-        video_data = config['video']
-        title = video_data['title']
-        is_live = video_data.get('isLivestream') and video_data.get('isLive')
-        meta = video_data.get('metaInformation')
-        sports = meta.get('sports')
-        categories = sports.split(',') if sports else []
-
-        token_url = self._extract_token_url(
-            video_data['streamAccess'], video_id,
-            video_data['abo']['required'])
-
-        formats = self._extract_formats(token_url, video_id)
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': video_data.get('description'),
-            'thumbnail': video_data.get('image'),
-            'categories': categories,
-            'formats': formats,
-            'is_live': is_live,
-        }
-
-
-class Laola1TvIE(Laola1TvBaseIE):
-    IE_NAME = 'laola1tv'
-    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/[a-z]+-[a-z]+/[^/]+/(?P<id>[^/?#&]+)'
-
-    _TESTS = [{
-        'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
-        'info_dict': {
-            'id': '227883',
-            'display_id': 'straubing-tigers-koelner-haie',
-            'ext': 'flv',
-            'title': 'Straubing Tigers - Kölner Haie',
-            'upload_date': '20140912',
-            'is_live': False,
-            'categories': ['Eishockey'],
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie',
-        'info_dict': {
-            'id': '464602',
-            'display_id': 'straubing-tigers-koelner-haie',
-            'ext': 'flv',
-            'title': 'Straubing Tigers - Kölner Haie',
-            'upload_date': '20160129',
-            'is_live': False,
-            'categories': ['Eishockey'],
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://www.laola1.tv/de-de/livestream/2016-03-22-belogorie-belgorod-trentino-diatec-lde',
-        'info_dict': {
-            'id': '487850',
-            'display_id': '2016-03-22-belogorie-belgorod-trentino-diatec-lde',
-            'ext': 'flv',
-            'title': 'Belogorie BELGOROD - TRENTINO Diatec',
-            'upload_date': '20160322',
-            'uploader': 'CEV - Europäischer Volleyball Verband',
-            'is_live': True,
-            'categories': ['Volleyball'],
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'skip': 'This live stream has already finished.',
-    }]
-
-    def _real_extract(self, url):
-        return self._extract_video(url)
-
-
-class EHFTVIE(Laola1TvBaseIE):
-    IE_NAME = 'ehftv'
-    _VALID_URL = r'https?://(?:www\.)?ehftv\.com/[a-z]+(?:-[a-z]+)?/[^/]+/(?P<id>[^/?#&]+)'
-
-    _TESTS = [{
-        'url': 'https://www.ehftv.com/int/video/paris-saint-germain-handball-pge-vive-kielce/1166761',
-        'info_dict': {
-            'id': '1166761',
-            'display_id': 'paris-saint-germain-handball-pge-vive-kielce',
-            'ext': 'mp4',
-            'title': 'Paris Saint-Germain Handball - PGE Vive Kielce',
-            'is_live': False,
-            'categories': ['Handball'],
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    def _real_extract(self, url):
-        return self._extract_video(url)
-
-
-class ITTFIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.ittf\.com/video/[^/]+/(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://tv.ittf.com/video/peng-wang-wei-matsudaira-kenta/951802',
-        'only_matching': True,
-    }
-
-    def _real_extract(self, url):
-        return self.url_result(
-            update_url_query('https://www.laola1.tv/titanplayer.php', {
-                'videoid': self._match_id(url),
-                'type': 'V',
-                'lang': 'en',
-                'portal': 'int',
-                'customer': 1024,
-            }), Laola1TvEmbedIE.ie_key())
diff --git a/yt_dlp/extractor/linuxacademy.py b/yt_dlp/extractor/linuxacademy.py
deleted file mode 100644
index 0b1644293..000000000
--- a/yt_dlp/extractor/linuxacademy.py
+++ /dev/null
@@ -1,238 +0,0 @@
-import json
-import random
-
-from .common import InfoExtractor
-from ..compat import compat_b64decode, compat_str
-from ..networking.exceptions import HTTPError
-from ..utils import (
-    clean_html,
-    ExtractorError,
-    js_to_json,
-    parse_duration,
-    try_get,
-    unified_timestamp,
-    urlencode_postdata,
-    urljoin,
-)
-
-
-class LinuxAcademyIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?linuxacademy\.com/cp/
-                        (?:
-                            courses/lesson/course/(?P<chapter_id>\d+)/lesson/(?P<lesson_id>\d+)|
-                            modules/view/id/(?P<course_id>\d+)
-                        )
-                    '''
-    _TESTS = [{
-        'url': 'https://linuxacademy.com/cp/courses/lesson/course/7971/lesson/2/module/675',
-        'info_dict': {
-            'id': '7971-2',
-            'ext': 'mp4',
-            'title': 'What Is Data Science',
-            'description': 'md5:c574a3c20607144fb36cb65bdde76c99',
-            'timestamp': int,  # The timestamp and upload date changes
-            'upload_date': r're:\d+',
-            'duration': 304,
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'skip': 'Requires Linux Academy account credentials',
-    }, {
-        'url': 'https://linuxacademy.com/cp/courses/lesson/course/1498/lesson/2',
-        'only_matching': True,
-    }, {
-        'url': 'https://linuxacademy.com/cp/modules/view/id/154',
-        'info_dict': {
-            'id': '154',
-            'title': 'AWS Certified Cloud Practitioner',
-            'description': 'md5:a68a299ca9bb98d41cca5abc4d4ce22c',
-            'duration': 28835,
-        },
-        'playlist_count': 41,
-        'skip': 'Requires Linux Academy account credentials',
-    }, {
-        'url': 'https://linuxacademy.com/cp/modules/view/id/39',
-        'info_dict': {
-            'id': '39',
-            'title': 'Red Hat Certified Systems Administrator - RHCSA (EX200) Exam Prep  (legacy)',
-            'description': 'md5:0f1d3369e90c3fb14a79813b863c902f',
-            'duration': 89280,
-        },
-        'playlist_count': 73,
-        'skip': 'Requires Linux Academy account credentials',
-    }]
-
-    _AUTHORIZE_URL = 'https://login.linuxacademy.com/authorize'
-    _ORIGIN_URL = 'https://linuxacademy.com'
-    _CLIENT_ID = 'KaWxNn1C2Gc7n83W9OFeXltd8Utb5vvx'
-    _NETRC_MACHINE = 'linuxacademy'
-
-    def _perform_login(self, username, password):
-        def random_string():
-            return ''.join(random.choices(
-                '0123456789ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvwxyz-._~', k=32))
-
-        webpage, urlh = self._download_webpage_handle(
-            self._AUTHORIZE_URL, None, 'Downloading authorize page', query={
-                'client_id': self._CLIENT_ID,
-                'response_type': 'token id_token',
-                'response_mode': 'web_message',
-                'redirect_uri': self._ORIGIN_URL,
-                'scope': 'openid email user_impersonation profile',
-                'audience': self._ORIGIN_URL,
-                'state': random_string(),
-                'nonce': random_string(),
-            })
-
-        login_data = self._parse_json(
-            self._search_regex(
-                r'atob\(\s*(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
-                'login info', group='value'), None,
-            transform_source=lambda x: compat_b64decode(x).decode('utf-8')
-        )['extraParams']
-
-        login_data.update({
-            'client_id': self._CLIENT_ID,
-            'redirect_uri': self._ORIGIN_URL,
-            'tenant': 'lacausers',
-            'connection': 'Username-Password-ACG-Proxy',
-            'username': username,
-            'password': password,
-            'sso': 'true',
-        })
-
-        login_state_url = urlh.url
-
-        try:
-            login_page = self._download_webpage(
-                'https://login.linuxacademy.com/usernamepassword/login', None,
-                'Downloading login page', data=json.dumps(login_data).encode(),
-                headers={
-                    'Content-Type': 'application/json',
-                    'Origin': 'https://login.linuxacademy.com',
-                    'Referer': login_state_url,
-                })
-        except ExtractorError as e:
-            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
-                error = self._parse_json(e.cause.response.read(), None)
-                message = error.get('description') or error['code']
-                raise ExtractorError(
-                    '%s said: %s' % (self.IE_NAME, message), expected=True)
-            raise
-
-        callback_page, urlh = self._download_webpage_handle(
-            'https://login.linuxacademy.com/login/callback', None,
-            'Downloading callback page',
-            data=urlencode_postdata(self._hidden_inputs(login_page)),
-            headers={
-                'Content-Type': 'application/x-www-form-urlencoded',
-                'Origin': 'https://login.linuxacademy.com',
-                'Referer': login_state_url,
-            })
-
-        access_token = self._search_regex(
-            r'access_token=([^=&]+)', urlh.url,
-            'access token', default=None)
-        if not access_token:
-            access_token = self._parse_json(
-                self._search_regex(
-                    r'authorizationResponse\s*=\s*({.+?})\s*;', callback_page,
-                    'authorization response'), None,
-                transform_source=js_to_json)['response']['access_token']
-
-        self._download_webpage(
-            'https://linuxacademy.com/cp/login/tokenValidateLogin/token/%s'
-            % access_token, None, 'Downloading token validation page')
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        chapter_id, lecture_id, course_id = mobj.group('chapter_id', 'lesson_id', 'course_id')
-        item_id = course_id if course_id else '%s-%s' % (chapter_id, lecture_id)
-
-        webpage = self._download_webpage(url, item_id)
-
-        # course path
-        if course_id:
-            module = self._parse_json(
-                self._search_regex(
-                    r'window\.module\s*=\s*({(?:(?!};)[^"]|"([^"]|\\")*")+})\s*;', webpage, 'module'),
-                item_id)
-            entries = []
-            chapter_number = None
-            chapter = None
-            chapter_id = None
-            for item in module['items']:
-                if not isinstance(item, dict):
-                    continue
-
-                def type_field(key):
-                    return (try_get(item, lambda x: x['type'][key], compat_str) or '').lower()
-                type_fields = (type_field('name'), type_field('slug'))
-                # Move to next module section
-                if 'section' in type_fields:
-                    chapter = item.get('course_name')
-                    chapter_id = item.get('course_module')
-                    chapter_number = 1 if not chapter_number else chapter_number + 1
-                    continue
-                # Skip non-lessons
-                if 'lesson' not in type_fields:
-                    continue
-                lesson_url = urljoin(url, item.get('url'))
-                if not lesson_url:
-                    continue
-                title = item.get('title') or item.get('lesson_name')
-                description = item.get('md_desc') or clean_html(item.get('description')) or clean_html(item.get('text'))
-                entries.append({
-                    '_type': 'url_transparent',
-                    'url': lesson_url,
-                    'ie_key': LinuxAcademyIE.ie_key(),
-                    'title': title,
-                    'description': description,
-                    'timestamp': unified_timestamp(item.get('date')) or unified_timestamp(item.get('created_on')),
-                    'duration': parse_duration(item.get('duration')),
-                    'chapter': chapter,
-                    'chapter_id': chapter_id,
-                    'chapter_number': chapter_number,
-                })
-            return {
-                '_type': 'playlist',
-                'entries': entries,
-                'id': course_id,
-                'title': module.get('title'),
-                'description': module.get('md_desc') or clean_html(module.get('desc')),
-                'duration': parse_duration(module.get('duration')),
-            }
-
-        # single video path
-        m3u8_url = self._parse_json(
-            self._search_regex(
-                r'player\.playlist\s*=\s*(\[.+?\])\s*;', webpage, 'playlist'),
-            item_id)[0]['file']
-        formats = self._extract_m3u8_formats(
-            m3u8_url, item_id, 'mp4', entry_protocol='m3u8_native',
-            m3u8_id='hls')
-        info = {
-            'id': item_id,
-            'formats': formats,
-        }
-        lesson = self._parse_json(
-            self._search_regex(
-                (r'window\.lesson\s*=\s*({.+?})\s*;',
-                 r'player\.lesson\s*=\s*({.+?})\s*;'),
-                webpage, 'lesson', default='{}'), item_id, fatal=False)
-        if lesson:
-            info.update({
-                'title': lesson.get('lesson_name'),
-                'description': lesson.get('md_desc') or clean_html(lesson.get('desc')),
-                'timestamp': unified_timestamp(lesson.get('date')) or unified_timestamp(lesson.get('created_on')),
-                'duration': parse_duration(lesson.get('duration')),
-            })
-        if not info.get('title'):
-            info['title'] = self._search_regex(
-                (r'>Lecture\s*:\s*(?P<value>[^<]+)',
-                 r'lessonName\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1'), webpage,
-                'title', group='value')
-        return info
diff --git a/yt_dlp/extractor/m6.py b/yt_dlp/extractor/m6.py
deleted file mode 100644
index 9dcc60164..000000000
--- a/yt_dlp/extractor/m6.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from .common import InfoExtractor
-
-
-class M6IE(InfoExtractor):
-    IE_NAME = 'm6'
-    _VALID_URL = r'https?://(?:www\.)?m6\.fr/[^/]+/videos/(?P<id>\d+)-[^\.]+\.html'
-
-    _TEST = {
-        'url': 'http://www.m6.fr/emission-les_reines_du_shopping/videos/11323908-emeline_est_la_reine_du_shopping_sur_le_theme_ma_fete_d_8217_anniversaire.html',
-        'md5': '242994a87de2c316891428e0176bcb77',
-        'info_dict': {
-            'id': '11323908',
-            'ext': 'mp4',
-            'title': 'Emeline est la Reine du Shopping sur le thème « Ma fête d’anniversaire ! »',
-            'description': 'md5:1212ae8fb4b7baa4dc3886c5676007c2',
-            'duration': 100,
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        return self.url_result('6play:%s' % video_id, 'SixPlay', video_id)
diff --git a/yt_dlp/extractor/meta.py b/yt_dlp/extractor/meta.py
deleted file mode 100644
index 7c11e6017..000000000
--- a/yt_dlp/extractor/meta.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from .common import InfoExtractor
-from .pladform import PladformIE
-from ..utils import (
-    unescapeHTML,
-    int_or_none,
-    ExtractorError,
-)
-
-
-class METAIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.meta\.ua/(?:iframe/)?(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://video.meta.ua/5502115.video',
-        'md5': '71b6f3ee274bef16f1ab410f7f56b476',
-        'info_dict': {
-            'id': '5502115',
-            'ext': 'mp4',
-            'title': 'Sony Xperia Z camera test [HQ]',
-            'description': 'Xperia Z shoots video in FullHD HDR.',
-            'uploader_id': 'nomobile',
-            'uploader': 'CHЁZA.TV',
-            'upload_date': '20130211',
-        },
-        'add_ie': ['Youtube'],
-    }, {
-        'url': 'http://video.meta.ua/iframe/5502115',
-        'only_matching': True,
-    }, {
-        # pladform embed
-        'url': 'http://video.meta.ua/7121015.video',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        st_html5 = self._search_regex(
-            r"st_html5\s*=\s*'#([^']+)'", webpage, 'uppod html5 st', default=None)
-
-        if st_html5:
-            # uppod st decryption algorithm is reverse engineered from function un(s) at uppod.js
-            json_str = ''
-            for i in range(0, len(st_html5), 3):
-                json_str += '&#x0%s;' % st_html5[i:i + 3]
-            uppod_data = self._parse_json(unescapeHTML(json_str), video_id)
-            error = uppod_data.get('customnotfound')
-            if error:
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
-
-            video_url = uppod_data['file']
-            info = {
-                'id': video_id,
-                'url': video_url,
-                'title': uppod_data.get('comment') or self._og_search_title(webpage),
-                'description': self._og_search_description(webpage, default=None),
-                'thumbnail': uppod_data.get('poster') or self._og_search_thumbnail(webpage),
-                'duration': int_or_none(self._og_search_property(
-                    'video:duration', webpage, default=None)),
-            }
-            if 'youtube.com/' in video_url:
-                info.update({
-                    '_type': 'url_transparent',
-                    'ie_key': 'Youtube',
-                })
-            return info
-
-        pladform_url = PladformIE._extract_url(webpage)
-        if pladform_url:
-            return self.url_result(pladform_url)
diff --git a/yt_dlp/extractor/metacafe.py b/yt_dlp/extractor/metacafe.py
deleted file mode 100644
index d7f5def0e..000000000
--- a/yt_dlp/extractor/metacafe.py
+++ /dev/null
@@ -1,281 +0,0 @@
-import json
-import re
-import urllib.parse
-
-from .common import InfoExtractor
-from ..compat import compat_parse_qs, compat_urllib_parse_unquote
-from ..utils import (
-    ExtractorError,
-    determine_ext,
-    get_element_by_attribute,
-    int_or_none,
-    mimetype2ext,
-)
-
-
-class MetacafeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?metacafe\.com/watch/(?P<id>[^/]+)/(?P<display_id>[^/?#]+)'
-    _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
-    _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
-    IE_NAME = 'metacafe'
-    _TESTS = [
-        # Youtube video
-        {
-            'add_ie': ['Youtube'],
-            'url': 'http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/',
-            'info_dict': {
-                'id': '_aUehQsCQtM',
-                'ext': 'mp4',
-                'upload_date': '20090102',
-                'title': 'The Electric Company | "Short I" | PBS KIDS GO!',
-                'description': 'md5:2439a8ef6d5a70e380c22f5ad323e5a8',
-                'uploader': 'PBS',
-                'uploader_id': 'PBS'
-            }
-        },
-        # Normal metacafe video
-        {
-            'url': 'http://www.metacafe.com/watch/11121940/news_stuff_you_wont_do_with_your_playstation_4/',
-            'md5': '6e0bca200eaad2552e6915ed6fd4d9ad',
-            'info_dict': {
-                'id': '11121940',
-                'ext': 'mp4',
-                'title': 'News: Stuff You Won\'t Do with Your PlayStation 4',
-                'uploader': 'ign',
-                'description': 'Sony released a massive FAQ on the PlayStation Blog detailing the PS4\'s capabilities and limitations.',
-            },
-            'skip': 'Page is temporarily unavailable.',
-        },
-        # metacafe video with family filter
-        {
-            'url': 'http://www.metacafe.com/watch/2155630/adult_art_by_david_hart_156/',
-            'md5': 'b06082c5079bbdcde677a6291fbdf376',
-            'info_dict': {
-                'id': '2155630',
-                'ext': 'mp4',
-                'title': 'Adult Art By David Hart 156',
-                'uploader': '63346',
-                'description': 'md5:9afac8fc885252201ad14563694040fc',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        # AnyClip video
-        {
-            'url': 'http://www.metacafe.com/watch/an-dVVXnuY7Jh77J/the_andromeda_strain_1971_stop_the_bomb_part_3/',
-            'info_dict': {
-                'id': 'an-dVVXnuY7Jh77J',
-                'ext': 'mp4',
-                'title': 'The Andromeda Strain (1971): Stop the Bomb Part 3',
-                'uploader': 'AnyClip',
-                'description': 'md5:cbef0460d31e3807f6feb4e7a5952e5b',
-            },
-        },
-        # age-restricted video
-        {
-            'url': 'http://www.metacafe.com/watch/5186653/bbc_internal_christmas_tape_79_uncensored_outtakes_etc/',
-            'md5': '98dde7c1a35d02178e8ab7560fe8bd09',
-            'info_dict': {
-                'id': '5186653',
-                'ext': 'mp4',
-                'title': 'BBC INTERNAL Christmas Tape \'79 - UNCENSORED Outtakes, Etc.',
-                'uploader': 'Dwayne Pipe',
-                'description': 'md5:950bf4c581e2c059911fa3ffbe377e4b',
-                'age_limit': 18,
-            },
-        },
-        # cbs video
-        {
-            'url': 'http://www.metacafe.com/watch/cb-8VD4r_Zws8VP/open_this_is_face_the_nation_february_9/',
-            'info_dict': {
-                'id': '8VD4r_Zws8VP',
-                'ext': 'flv',
-                'title': 'Open: This is Face the Nation, February 9',
-                'description': 'md5:8a9ceec26d1f7ed6eab610834cc1a476',
-                'duration': 96,
-                'uploader': 'CBSI-NEW',
-                'upload_date': '20140209',
-                'timestamp': 1391959800,
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
-        },
-        # Movieclips.com video
-        {
-            'url': 'http://www.metacafe.com/watch/mv-Wy7ZU/my_week_with_marilyn_do_you_love_me/',
-            'info_dict': {
-                'id': 'mv-Wy7ZU',
-                'ext': 'mp4',
-                'title': 'My Week with Marilyn - Do You Love Me?',
-                'description': 'From the movie My Week with Marilyn - Colin (Eddie Redmayne) professes his love to Marilyn (Michelle Williams) and gets her to promise to return to set and finish the movie.',
-                'uploader': 'movie_trailers',
-                'duration': 176,
-            },
-            'params': {
-                'skip_download': 'requires rtmpdump',
-            }
-        }
-    ]
-
-    def report_disclaimer(self):
-        self.to_screen('Retrieving disclaimer')
-
-    def _real_extract(self, url):
-        # Extract id and simplified title from URL
-        video_id, display_id = self._match_valid_url(url).groups()
-
-        # the video may come from an external site
-        m_external = re.match(r'^(\w{2})-(.*)$', video_id)
-        if m_external is not None:
-            prefix, ext_id = m_external.groups()
-            # Check if video comes from YouTube
-            if prefix == 'yt':
-                return self.url_result('http://www.youtube.com/watch?v=%s' % ext_id, 'Youtube')
-            # CBS videos use theplatform.com
-            if prefix == 'cb':
-                return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
-
-        headers = {
-            # Disable family filter
-            'Cookie': 'user=%s; ' % urllib.parse.quote(json.dumps({'ffilter': False}))
-        }
-
-        # AnyClip videos require the flashversion cookie so that we get the link
-        # to the mp4 file
-        if video_id.startswith('an-'):
-            headers['Cookie'] += 'flashVersion=0; '
-
-        # Retrieve video webpage to extract further information
-        webpage = self._download_webpage(url, video_id, headers=headers)
-
-        error = get_element_by_attribute(
-            'class', 'notfound-page-title', webpage)
-        if error:
-            raise ExtractorError(error, expected=True)
-
-        video_title = self._html_search_meta(
-            ['og:title', 'twitter:title'], webpage, 'title', default=None) or self._search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
-
-        # Extract URL, uploader and title from webpage
-        self.report_extraction(video_id)
-        video_url = None
-        mobj = re.search(r'(?m)&(?:media|video)URL=([^&]+)', webpage)
-        if mobj is not None:
-            mediaURL = compat_urllib_parse_unquote(mobj.group(1))
-            video_ext = determine_ext(mediaURL)
-
-            # Extract gdaKey if available
-            mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
-            if mobj is None:
-                video_url = mediaURL
-            else:
-                gdaKey = mobj.group(1)
-                video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
-        if video_url is None:
-            mobj = re.search(r'<video src="([^"]+)"', webpage)
-            if mobj:
-                video_url = mobj.group(1)
-                video_ext = 'mp4'
-        if video_url is None:
-            flashvars = self._search_regex(
-                r' name="flashvars" value="(.*?)"', webpage, 'flashvars',
-                default=None)
-            if flashvars:
-                vardict = compat_parse_qs(flashvars)
-                if 'mediaData' not in vardict:
-                    raise ExtractorError('Unable to extract media URL')
-                mobj = re.search(
-                    r'"mediaURL":"(?P<mediaURL>http.*?)",(.*?)"key":"(?P<key>.*?)"', vardict['mediaData'][0])
-                if mobj is None:
-                    raise ExtractorError('Unable to extract media URL')
-                mediaURL = mobj.group('mediaURL').replace('\\/', '/')
-                video_url = '%s?__gda__=%s' % (mediaURL, mobj.group('key'))
-                video_ext = determine_ext(video_url)
-        if video_url is None:
-            player_url = self._search_regex(
-                r"swfobject\.embedSWF\('([^']+)'",
-                webpage, 'config URL', default=None)
-            if player_url:
-                config_url = self._search_regex(
-                    r'config=(.+)$', player_url, 'config URL')
-                config_doc = self._download_xml(
-                    config_url, video_id,
-                    note='Downloading video config')
-                smil_url = config_doc.find('.//properties').attrib['smil_file']
-                smil_doc = self._download_xml(
-                    smil_url, video_id,
-                    note='Downloading SMIL document')
-                base_url = smil_doc.find('./head/meta').attrib['base']
-                video_url = []
-                for vn in smil_doc.findall('.//video'):
-                    br = int(vn.attrib['system-bitrate'])
-                    play_path = vn.attrib['src']
-                    video_url.append({
-                        'format_id': 'smil-%d' % br,
-                        'url': base_url,
-                        'play_path': play_path,
-                        'page_url': url,
-                        'player_url': player_url,
-                        'ext': play_path.partition(':')[0],
-                    })
-        if video_url is None:
-            flashvars = self._parse_json(self._search_regex(
-                r'flashvars\s*=\s*({.*});', webpage, 'flashvars',
-                default=None), video_id, fatal=False)
-            if flashvars:
-                video_url = []
-                for source in flashvars.get('sources'):
-                    source_url = source.get('src')
-                    if not source_url:
-                        continue
-                    ext = mimetype2ext(source.get('type')) or determine_ext(source_url)
-                    if ext == 'm3u8':
-                        video_url.extend(self._extract_m3u8_formats(
-                            source_url, video_id, 'mp4',
-                            'm3u8_native', m3u8_id='hls', fatal=False))
-                    else:
-                        video_url.append({
-                            'url': source_url,
-                            'ext': ext,
-                        })
-
-        if video_url is None:
-            raise ExtractorError('Unsupported video type')
-
-        description = self._html_search_meta(
-            ['og:description', 'twitter:description', 'description'],
-            webpage, 'title', fatal=False)
-        thumbnail = self._html_search_meta(
-            ['og:image', 'twitter:image'], webpage, 'title', fatal=False)
-        video_uploader = self._html_search_regex(
-            r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
-            webpage, 'uploader nickname', fatal=False)
-        duration = int_or_none(
-            self._html_search_meta('video:duration', webpage, default=None))
-        age_limit = (
-            18
-            if re.search(r'(?:"contentRating":|"rating",)"restricted"', webpage)
-            else 0)
-
-        if isinstance(video_url, list):
-            formats = video_url
-        else:
-            formats = [{
-                'url': video_url,
-                'ext': video_ext,
-            }]
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'description': description,
-            'uploader': video_uploader,
-            'title': video_title,
-            'thumbnail': thumbnail,
-            'age_limit': age_limit,
-            'formats': formats,
-            'duration': duration,
-        }
diff --git a/yt_dlp/extractor/mgoon.py b/yt_dlp/extractor/mgoon.py
deleted file mode 100644
index 2388a7192..000000000
--- a/yt_dlp/extractor/mgoon.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    qualities,
-    unified_strdate,
-)
-
-
-class MgoonIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?
-    (?:(:?m\.)?mgoon\.com/(?:ch/(?:.+)/v|play/view)|
-        video\.mgoon\.com)/(?P<id>[0-9]+)'''
-    _API_URL = 'http://mpos.mgoon.com/player/video?id={0:}'
-    _TESTS = [
-        {
-            'url': 'http://m.mgoon.com/ch/hi6618/v/5582148',
-            'md5': 'dd46bb66ab35cf6d51cc812fd82da79d',
-            'info_dict': {
-                'id': '5582148',
-                'uploader_id': 'hi6618',
-                'duration': 240.419,
-                'upload_date': '20131220',
-                'ext': 'mp4',
-                'title': 'md5:543aa4c27a4931d371c3f433e8cebebc',
-                'thumbnail': r're:^https?://.*\.jpg$',
-            }
-        },
-        {
-            'url': 'http://www.mgoon.com/play/view/5582148',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://video.mgoon.com/5582148',
-            'only_matching': True,
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-
-        data = self._download_json(self._API_URL.format(video_id), video_id)
-
-        if data.get('errorInfo', {}).get('code') != 'NONE':
-            raise ExtractorError('%s encountered an error: %s' % (
-                self.IE_NAME, data['errorInfo']['message']), expected=True)
-
-        v_info = data['videoInfo']
-        title = v_info.get('v_title')
-        thumbnail = v_info.get('v_thumbnail')
-        duration = v_info.get('v_duration')
-        upload_date = unified_strdate(v_info.get('v_reg_date'))
-        uploader_id = data.get('userInfo', {}).get('u_alias')
-        if duration:
-            duration /= 1000.0
-
-        age_limit = None
-        if data.get('accessInfo', {}).get('code') == 'VIDEO_STATUS_ADULT':
-            age_limit = 18
-
-        formats = []
-        get_quality = qualities(['360p', '480p', '720p', '1080p'])
-        for fmt in data['videoFiles']:
-            formats.append({
-                'format_id': fmt['label'],
-                'quality': get_quality(fmt['label']),
-                'url': fmt['url'],
-                'ext': fmt['format'],
-
-            })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'upload_date': upload_date,
-            'uploader_id': uploader_id,
-            'age_limit': age_limit,
-        }
diff --git a/yt_dlp/extractor/miomio.py b/yt_dlp/extractor/miomio.py
deleted file mode 100644
index 8df8cba19..000000000
--- a/yt_dlp/extractor/miomio.py
+++ /dev/null
@@ -1,134 +0,0 @@
-import random
-
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..networking import Request
-from ..utils import ExtractorError, int_or_none, xpath_text
-
-
-class MioMioIE(InfoExtractor):
-    IE_NAME = 'miomio.tv'
-    _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
-    _TESTS = [{
-        # "type=video" in flashvars
-        'url': 'http://www.miomio.tv/watch/cc88912/',
-        'info_dict': {
-            'id': '88912',
-            'ext': 'flv',
-            'title': '【SKY】字幕 铠武昭和VS平成 假面骑士大战FEAT战队 魔星字幕组 字幕',
-            'duration': 5923,
-        },
-        'skip': 'Unable to load videos',
-    }, {
-        'url': 'http://www.miomio.tv/watch/cc184024/',
-        'info_dict': {
-            'id': '43729',
-            'title': '《动漫同人插画绘制》',
-        },
-        'playlist_mincount': 86,
-        'skip': 'Unable to load videos',
-    }, {
-        'url': 'http://www.miomio.tv/watch/cc173113/',
-        'info_dict': {
-            'id': '173113',
-            'title': 'The New Macbook 2015 上手试玩与简评'
-        },
-        'playlist_mincount': 2,
-        'skip': 'Unable to load videos',
-    }, {
-        # new 'h5' player
-        'url': 'http://www.miomio.tv/watch/cc273997/',
-        'md5': '0b27a4b4495055d826813f8c3a6b2070',
-        'info_dict': {
-            'id': '273997',
-            'ext': 'mp4',
-            'title': 'マツコの知らない世界【劇的進化SP！ビニール傘＆冷凍食品2016】 1_2 - 16 05 31',
-        },
-        'skip': 'Unable to load videos',
-    }]
-
-    def _extract_mioplayer(self, webpage, video_id, title, http_headers):
-        xml_config = self._search_regex(
-            r'flashvars="type=(?:sina|video)&amp;(.+?)&amp;',
-            webpage, 'xml config')
-
-        # skipping the following page causes lags and eventually connection drop-outs
-        self._request_webpage(
-            'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
-            video_id)
-
-        vid_config_request = Request(
-            'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
-            headers=http_headers)
-
-        # the following xml contains the actual configuration information on the video file(s)
-        vid_config = self._download_xml(vid_config_request, video_id)
-
-        if not int_or_none(xpath_text(vid_config, 'timelength')):
-            raise ExtractorError('Unable to load videos!', expected=True)
-
-        entries = []
-        for f in vid_config.findall('./durl'):
-            segment_url = xpath_text(f, 'url', 'video url')
-            if not segment_url:
-                continue
-            order = xpath_text(f, 'order', 'order')
-            segment_id = video_id
-            segment_title = title
-            if order:
-                segment_id += '-%s' % order
-                segment_title += ' part %s' % order
-            entries.append({
-                'id': segment_id,
-                'url': segment_url,
-                'title': segment_title,
-                'duration': int_or_none(xpath_text(f, 'length', 'duration'), 1000),
-                'http_headers': http_headers,
-            })
-
-        return entries
-
-    def _download_chinese_webpage(self, *args, **kwargs):
-        # Requests with English locales return garbage
-        headers = {
-            'Accept-Language': 'zh-TW,en-US;q=0.7,en;q=0.3',
-        }
-        kwargs.setdefault('headers', {}).update(headers)
-        return self._download_webpage(*args, **kwargs)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_chinese_webpage(
-            url, video_id)
-
-        title = self._html_search_meta(
-            'description', webpage, 'title', fatal=True)
-
-        mioplayer_path = self._search_regex(
-            r'src="(/mioplayer(?:_h5)?/[^"]+)"', webpage, 'ref_path')
-
-        if '_h5' in mioplayer_path:
-            player_url = compat_urlparse.urljoin(url, mioplayer_path)
-            player_webpage = self._download_chinese_webpage(
-                player_url, video_id,
-                note='Downloading player webpage', headers={'Referer': url})
-            entries = self._parse_html5_media_entries(player_url, player_webpage, video_id)
-            http_headers = {'Referer': player_url}
-        else:
-            http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path}
-            entries = self._extract_mioplayer(webpage, video_id, title, http_headers)
-
-        if len(entries) == 1:
-            segment = entries[0]
-            segment['id'] = video_id
-            segment['title'] = title
-            segment['http_headers'] = http_headers
-            return segment
-
-        return {
-            '_type': 'multi_video',
-            'id': video_id,
-            'entries': entries,
-            'title': title,
-            'http_headers': http_headers,
-        }
diff --git a/yt_dlp/extractor/mnet.py b/yt_dlp/extractor/mnet.py
deleted file mode 100644
index 98bab2e10..000000000
--- a/yt_dlp/extractor/mnet.py
+++ /dev/null
@@ -1,85 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_duration,
-    parse_iso8601,
-)
-
-
-class MnetIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mnet\.(?:com|interest\.me)/tv/vod/(?:.*?\bclip_id=)?(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.mnet.com/tv/vod/171008',
-        'info_dict': {
-            'id': '171008',
-            'title': 'SS_이해인@히든박스',
-            'description': 'md5:b9efa592c3918b615ba69fe9f8a05c55',
-            'duration': 88,
-            'upload_date': '20151231',
-            'timestamp': 1451564040,
-            'age_limit': 0,
-            'thumbnails': 'mincount:5',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'ext': 'flv',
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://mnet.interest.me/tv/vod/172790',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.mnet.com/tv/vod/vod_view.asp?clip_id=172790&tabMenu=',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        # TODO: extract rtmp formats
-        # no stype -> rtmp url
-        # stype=H -> m3u8 url
-        # stype=M -> mpd url
-        info = self._download_json(
-            'http://content.api.mnet.com/player/vodConfig',
-            video_id, 'Downloading vod config JSON', query={
-                'id': video_id,
-                'ctype': 'CLIP',
-                'stype': 'H',
-            })['data']['info']
-
-        title = info['title']
-
-        cdn_data = self._download_json(
-            info['cdn'], video_id, 'Downloading vod cdn JSON')['data'][0]
-        m3u8_url = cdn_data['url']
-        token = cdn_data.get('token')
-        if token and token != '-':
-            m3u8_url += '?' + token
-        formats = self._extract_wowza_formats(
-            m3u8_url, video_id, skip_protocols=['rtmp', 'rtsp', 'f4m'])
-
-        description = info.get('ment')
-        duration = parse_duration(info.get('time'))
-        timestamp = parse_iso8601(info.get('date'), delimiter=' ')
-        age_limit = info.get('adult')
-        if age_limit is not None:
-            age_limit = 0 if age_limit == 'N' else 18
-        thumbnails = [{
-            'id': thumb_format,
-            'url': thumb['url'],
-            'width': int_or_none(thumb.get('width')),
-            'height': int_or_none(thumb.get('height')),
-        } for thumb_format, thumb in info.get('cover', {}).items() if thumb.get('url')]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'timestamp': timestamp,
-            'age_limit': age_limit,
-            'thumbnails': thumbnails,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/moevideo.py b/yt_dlp/extractor/moevideo.py
deleted file mode 100644
index fda08cae9..000000000
--- a/yt_dlp/extractor/moevideo.py
+++ /dev/null
@@ -1,74 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    int_or_none,
-)
-
-
-class MoeVideoIE(InfoExtractor):
-    IE_DESC = 'LetitBit video services: moevideo.net, playreplay.net and videochart.net'
-    _VALID_URL = r'''(?x)
-        https?://(?P<host>(?:www\.)?
-        (?:(?:moevideo|playreplay|videochart)\.net|thesame\.tv))/
-        (?:video|framevideo|embed)/(?P<id>[0-9a-z]+\.[0-9A-Za-z]+)'''
-    _API_URL = 'http://api.letitbit.net/'
-    _API_KEY = 'tVL0gjqo5'
-    _TESTS = [
-        {
-            'url': 'http://moevideo.net/video/00297.0036103fe3d513ef27915216fd29',
-            'md5': '129f5ae1f6585d0e9bb4f38e774ffb3a',
-            'info_dict': {
-                'id': '00297.0036103fe3d513ef27915216fd29',
-                'ext': 'flv',
-                'title': 'Sink cut out machine',
-                'description': 'md5:f29ff97b663aefa760bf7ca63c8ca8a8',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'width': 540,
-                'height': 360,
-                'duration': 179,
-                'filesize': 17822500,
-            },
-            'skip': 'Video has been removed',
-        },
-        {
-            'url': 'http://playreplay.net/video/77107.7f325710a627383d40540d8e991a',
-            'md5': '74f0a014d5b661f0f0e2361300d1620e',
-            'info_dict': {
-                'id': '77107.7f325710a627383d40540d8e991a',
-                'ext': 'flv',
-                'title': 'Operacion Condor.',
-                'description': 'md5:7e68cb2fcda66833d5081c542491a9a3',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'width': 480,
-                'height': 296,
-                'duration': 6027,
-                'filesize': 588257923,
-            },
-            'skip': 'Video has been removed',
-        },
-    ]
-
-    def _real_extract(self, url):
-        host, video_id = self._match_valid_url(url).groups()
-
-        webpage = self._download_webpage(
-            'http://%s/video/%s' % (host, video_id),
-            video_id, 'Downloading webpage')
-
-        title = self._og_search_title(webpage)
-
-        embed_webpage = self._download_webpage(
-            'http://%s/embed/%s' % (host, video_id),
-            video_id, 'Downloading embed webpage')
-        video = self._parse_json(self._search_regex(
-            r'mvplayer\("#player"\s*,\s*({.+})',
-            embed_webpage, 'mvplayer'), video_id)['video']
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': video.get('poster') or self._og_search_thumbnail(webpage),
-            'description': clean_html(self._og_search_description(webpage)),
-            'duration': int_or_none(self._og_search_property('video:duration', webpage)),
-            'url': video['ourUrl'],
-        }
diff --git a/yt_dlp/extractor/mofosex.py b/yt_dlp/extractor/mofosex.py
deleted file mode 100644
index 9cb6980c1..000000000
--- a/yt_dlp/extractor/mofosex.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    str_to_int,
-    unified_strdate,
-)
-from .keezmovies import KeezMoviesIE
-
-
-class MofosexIE(KeezMoviesIE):  # XXX: Do not subclass from concrete IE
-    _VALID_URL = r'https?://(?:www\.)?mofosex\.com/videos/(?P<id>\d+)/(?P<display_id>[^/?#&.]+)\.html'
-    _TESTS = [{
-        'url': 'http://www.mofosex.com/videos/318131/amateur-teen-playing-and-masturbating-318131.html',
-        'md5': '558fcdafbb63a87c019218d6e49daf8a',
-        'info_dict': {
-            'id': '318131',
-            'display_id': 'amateur-teen-playing-and-masturbating-318131',
-            'ext': 'mp4',
-            'title': 'amateur teen playing and masturbating',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'upload_date': '20121114',
-            'view_count': int,
-            'like_count': int,
-            'dislike_count': int,
-            'age_limit': 18,
-        }
-    }, {
-        # This video is no longer available
-        'url': 'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        webpage, info = self._extract_info(url)
-
-        view_count = str_to_int(self._search_regex(
-            r'VIEWS:</span>\s*([\d,.]+)', webpage, 'view count', fatal=False))
-        like_count = int_or_none(self._search_regex(
-            r'id=["\']amountLikes["\'][^>]*>(\d+)', webpage,
-            'like count', fatal=False))
-        dislike_count = int_or_none(self._search_regex(
-            r'id=["\']amountDislikes["\'][^>]*>(\d+)', webpage,
-            'like count', fatal=False))
-        upload_date = unified_strdate(self._html_search_regex(
-            r'Added:</span>([^<]+)', webpage, 'upload date', fatal=False))
-
-        info.update({
-            'view_count': view_count,
-            'like_count': like_count,
-            'dislike_count': dislike_count,
-            'upload_date': upload_date,
-            'thumbnail': self._og_search_thumbnail(webpage),
-        })
-
-        return info
-
-
-class MofosexEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mofosex\.com/embed/?\?.*?\bvideoid=(?P<id>\d+)'
-    _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?mofosex\.com/embed/?\?.*?\bvideoid=\d+)']
-    _TESTS = [{
-        'url': 'https://www.mofosex.com/embed/?videoid=318131&referrer=KM',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        return self.url_result(
-            'http://www.mofosex.com/videos/{0}/{0}.html'.format(video_id),
-            ie=MofosexIE.ie_key(), video_id=video_id)
diff --git a/yt_dlp/extractor/movieclips.py b/yt_dlp/extractor/movieclips.py
deleted file mode 100644
index f7f2921fd..000000000
--- a/yt_dlp/extractor/movieclips.py
+++ /dev/null
@@ -1,47 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    smuggle_url,
-    float_or_none,
-    parse_iso8601,
-    update_url_query,
-)
-
-
-class MovieClipsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?movieclips\.com/videos/.+-(?P<id>\d+)(?:\?|$)'
-    _TEST = {
-        'url': 'http://www.movieclips.com/videos/warcraft-trailer-1-561180739597',
-        'md5': '42b5a0352d4933a7bd54f2104f481244',
-        'info_dict': {
-            'id': 'pKIGmG83AqD9',
-            'ext': 'mp4',
-            'title': 'Warcraft Trailer 1',
-            'description': 'Watch Trailer 1 from Warcraft (2016). Legendary’s WARCRAFT is a 3D epic adventure of world-colliding conflict based.',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1446843055,
-            'upload_date': '20151106',
-            'uploader': 'Movieclips',
-        },
-        'add_ie': ['ThePlatform'],
-        'skip': 'redirects to YouTube',
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video = next(v for v in self._parse_json(self._search_regex(
-            r'var\s+__REACT_ENGINE__\s*=\s*({.+});',
-            webpage, 'react engine'), video_id)['playlist']['videos'] if v['id'] == video_id)
-
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
-            'url': smuggle_url(update_url_query(
-                video['contentUrl'], {'mbr': 'true'}), {'force_smil_url': True}),
-            'title': self._og_search_title(webpage),
-            'description': self._html_search_meta('description', webpage),
-            'duration': float_or_none(video.get('duration')),
-            'timestamp': parse_iso8601(video.get('dateCreated')),
-            'thumbnail': video.get('defaultImage'),
-            'uploader': video.get('provider'),
-        }
diff --git a/yt_dlp/extractor/msn.py b/yt_dlp/extractor/msn.py
index f91c53eba..77d1806a3 100644
--- a/yt_dlp/extractor/msn.py
+++ b/yt_dlp/extractor/msn.py
@@ -11,6 +11,7 @@
 
 
 class MSNIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'https?://(?:(?:www|preview)\.)?msn\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/[a-z]{2}-(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         'url': 'https://www.msn.com/en-in/money/video/7-ways-to-get-rid-of-chest-congestion/vi-BBPxU6d',
diff --git a/yt_dlp/extractor/mwave.py b/yt_dlp/extractor/mwave.py
deleted file mode 100644
index efbfd9d43..000000000
--- a/yt_dlp/extractor/mwave.py
+++ /dev/null
@@ -1,87 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    int_or_none,
-    parse_duration,
-)
-
-
-class MwaveIE(InfoExtractor):
-    _VALID_URL = r'https?://mwave\.interest\.me/(?:[^/]+/)?mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
-    _URL_TEMPLATE = 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=%s'
-    _TESTS = [{
-        'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168859',
-        # md5 is unstable
-        'info_dict': {
-            'id': '168859',
-            'ext': 'flv',
-            'title': '[M COUNTDOWN] SISTAR - SHAKE IT',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'M COUNTDOWN',
-            'duration': 206,
-            'view_count': int,
-        }
-    }, {
-        'url': 'http://mwave.interest.me/en/mnettv/videodetail.m?searchVideoDetailVO.clip_id=176199',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        vod_info = self._download_json(
-            'http://mwave.interest.me/onair/vod_info.m?vodtype=CL&sectorid=&endinfo=Y&id=%s' % video_id,
-            video_id, 'Download vod JSON')
-
-        formats = []
-        for num, cdn_info in enumerate(vod_info['cdn']):
-            stream_url = cdn_info.get('url')
-            if not stream_url:
-                continue
-            stream_name = cdn_info.get('name') or compat_str(num)
-            f4m_stream = self._download_json(
-                stream_url, video_id,
-                'Download %s stream JSON' % stream_name)
-            f4m_url = f4m_stream.get('fileurl')
-            if not f4m_url:
-                continue
-            formats.extend(
-                self._extract_f4m_formats(f4m_url + '&hdcore=3.0.3', video_id, f4m_id=stream_name))
-
-        return {
-            'id': video_id,
-            'title': vod_info['title'],
-            'thumbnail': vod_info.get('cover'),
-            'uploader': vod_info.get('program_title'),
-            'duration': parse_duration(vod_info.get('time')),
-            'view_count': int_or_none(vod_info.get('hit')),
-            'formats': formats,
-        }
-
-
-class MwaveMeetGreetIE(InfoExtractor):
-    _VALID_URL = r'https?://mwave\.interest\.me/(?:[^/]+/)?meetgreet/view/(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://mwave.interest.me/meetgreet/view/256',
-        'info_dict': {
-            'id': '173294',
-            'ext': 'flv',
-            'title': '[MEET&GREET] Park BoRam',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'Mwave',
-            'duration': 3634,
-            'view_count': int,
-        }
-    }, {
-        'url': 'http://mwave.interest.me/en/meetgreet/view/256',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        clip_id = self._html_search_regex(
-            r'<iframe[^>]+src="/mnettv/ifr_clip\.m\?searchVideoDetailVO\.clip_id=(\d+)',
-            webpage, 'clip ID')
-        clip_url = MwaveIE._URL_TEMPLATE % clip_id
-        return self.url_result(clip_url, 'Mwave', clip_id)
diff --git a/yt_dlp/extractor/mychannels.py b/yt_dlp/extractor/mychannels.py
deleted file mode 100644
index 8a70c1f7b..000000000
--- a/yt_dlp/extractor/mychannels.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from .common import InfoExtractor
-
-
-class MyChannelsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mychannels\.com/.*(?P<id_type>video|production)_id=(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'https://mychannels.com/missholland/miss-holland?production_id=3416',
-        'md5': 'b8993daad4262dd68d89d651c0c52c45',
-        'info_dict': {
-            'id': 'wUUDZZep6vQD',
-            'ext': 'mp4',
-            'title': 'Miss Holland joins VOTE LEAVE',
-            'description': 'Miss Holland | #13 Not a potato',
-            'uploader': 'Miss Holland',
-        }
-    }
-
-    def _real_extract(self, url):
-        id_type, url_id = self._match_valid_url(url).groups()
-        webpage = self._download_webpage(url, url_id)
-        video_data = self._html_search_regex(r'<div([^>]+data-%s-id="%s"[^>]+)>' % (id_type, url_id), webpage, 'video data')
-
-        def extract_data_val(attr, fatal=False):
-            return self._html_search_regex(r'data-%s\s*=\s*"([^"]+)"' % attr, video_data, attr, fatal=fatal)
-        minoto_id = extract_data_val('minoto-id') or self._search_regex(r'/id/([a-zA-Z0-9]+)', extract_data_val('video-src', True), 'minoto id')
-
-        return {
-            '_type': 'url_transparent',
-            'url': 'minoto:%s' % minoto_id,
-            'id': url_id,
-            'title': extract_data_val('title', True),
-            'description': extract_data_val('description'),
-            'thumbnail': extract_data_val('image'),
-            'uploader': extract_data_val('channel'),
-        }
diff --git a/yt_dlp/extractor/myvi.py b/yt_dlp/extractor/myvi.py
deleted file mode 100644
index df7200be2..000000000
--- a/yt_dlp/extractor/myvi.py
+++ /dev/null
@@ -1,100 +0,0 @@
-from .common import InfoExtractor
-from .vimple import SprutoBaseIE
-
-
-class MyviIE(SprutoBaseIE):
-    _VALID_URL = r'''(?x)
-                        (?:
-                            https?://
-                                (?:www\.)?
-                                myvi\.
-                                (?:
-                                    (?:ru/player|tv)/
-                                    (?:
-                                        (?:
-                                            embed/html|
-                                            flash|
-                                            api/Video/Get
-                                        )/|
-                                        content/preloader\.swf\?.*\bid=
-                                    )|
-                                    ru/watch/
-                                )|
-                            myvi:
-                        )
-                        (?P<id>[\da-zA-Z_-]+)
-                    '''
-    _EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/(?:embed/html|flash)/[^"]+)\1']
-    _TESTS = [{
-        'url': 'http://myvi.ru/player/embed/html/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
-        'md5': '571bbdfba9f9ed229dc6d34cc0f335bf',
-        'info_dict': {
-            'id': 'f16b2bbd-cde8-481c-a981-7cd48605df43',
-            'ext': 'mp4',
-            'title': 'хозяин жизни',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 25,
-        },
-    }, {
-        'url': 'http://myvi.ru/player/content/preloader.swf?id=oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wOYf1WFpPfc_bWTKGVf_Zafr0',
-        'only_matching': True,
-    }, {
-        'url': 'http://myvi.ru/player/api/Video/Get/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
-        'only_matching': True,
-    }, {
-        'url': 'http://myvi.tv/embed/html/oTGTNWdyz4Zwy_u1nraolwZ1odenTd9WkTnRfIL9y8VOgHYqOHApE575x4_xxS9Vn0?ap=0',
-        'only_matching': True,
-    }, {
-        'url': 'http://myvi.ru/player/flash/ocp2qZrHI-eZnHKQBK4cZV60hslH8LALnk0uBfKsB-Q4WnY26SeGoYPi8HWHxu0O30',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.myvi.ru/watch/YwbqszQynUaHPn_s82sx0Q2',
-        'only_matching': True,
-    }, {
-        'url': 'myvi:YwbqszQynUaHPn_s82sx0Q2',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        spruto = self._download_json(
-            'http://myvi.ru/player/api/Video/Get/%s?sig' % video_id, video_id)['sprutoData']
-
-        return self._extract_spruto(spruto, video_id)
-
-
-class MyviEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?myvi\.tv/(?:[^?]+\?.*?\bv=|embed/)(?P<id>[\da-z]+)'
-    _TESTS = [{
-        'url': 'https://www.myvi.tv/embed/ccdqic3wgkqwpb36x9sxg43t4r',
-        'info_dict': {
-            'id': 'b3ea0663-3234-469d-873e-7fecf36b31d1',
-            'ext': 'mp4',
-            'title': 'Твоя (original song).mp4',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 277,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://www.myvi.tv/idmi6o?v=ccdqic3wgkqwpb36x9sxg43t4r#watch',
-        'only_matching': True,
-    }]
-
-    @classmethod
-    def suitable(cls, url):
-        return False if MyviIE.suitable(url) else super(MyviEmbedIE, cls).suitable(url)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'https://www.myvi.tv/embed/%s' % video_id, video_id)
-
-        myvi_id = self._search_regex(
-            r'CreatePlayer\s*\(\s*["\'].*?\bv=([\da-zA-Z_]+)',
-            webpage, 'video id')
-
-        return self.url_result('myvi:%s' % myvi_id, ie=MyviIE.ie_key())
diff --git a/yt_dlp/extractor/newstube.py b/yt_dlp/extractor/newstube.py
deleted file mode 100644
index 820eb4ba7..000000000
--- a/yt_dlp/extractor/newstube.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import base64
-import hashlib
-
-from .common import InfoExtractor
-from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
-from ..utils import (
-    int_or_none,
-    parse_codecs,
-    parse_duration,
-)
-
-
-class NewstubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?newstube\.ru/media/(?P<id>.+)'
-    _TEST = {
-        'url': 'http://www.newstube.ru/media/telekanal-cnn-peremestil-gorod-slavyansk-v-krym',
-        'md5': '9d10320ad473444352f72f746ccb8b8c',
-        'info_dict': {
-            'id': '728e0ef2-e187-4012-bac0-5a081fdcb1f6',
-            'ext': 'mp4',
-            'title': 'Телеканал CNN переместил город Славянск в Крым',
-            'description': 'md5:419a8c9f03442bc0b0a794d689360335',
-            'duration': 31.05,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        page = self._download_webpage(url, video_id)
-        title = self._html_search_meta(['og:title', 'twitter:title'], page, fatal=True)
-
-        video_guid = self._html_search_regex(
-            r'<meta\s+property="og:video(?::(?:(?:secure_)?url|iframe))?"\s+content="https?://(?:www\.)?newstube\.ru/embed/(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
-            page, 'video GUID')
-
-        enc_data = base64.b64decode(self._download_webpage(
-            'https://www.newstube.ru/embed/api/player/getsources2',
-            video_guid, query={
-                'guid': video_guid,
-                'ff': 3,
-            }))
-        key = hashlib.pbkdf2_hmac(
-            'sha1', video_guid.replace('-', '').encode(), enc_data[:16], 1)[:16]
-        dec_data = unpad_pkcs7(aes_cbc_decrypt_bytes(enc_data[32:], key, enc_data[16:32]))
-        sources = self._parse_json(dec_data, video_guid)
-
-        formats = []
-        for source in sources:
-            source_url = source.get('Src')
-            if not source_url:
-                continue
-            height = int_or_none(source.get('Height'))
-            f = {
-                'format_id': 'http' + ('-%dp' % height if height else ''),
-                'url': source_url,
-                'width': int_or_none(source.get('Width')),
-                'height': height,
-            }
-            source_type = source.get('Type')
-            if source_type:
-                f.update(parse_codecs(self._search_regex(
-                    r'codecs="([^"]+)"', source_type, 'codecs', fatal=False)))
-            formats.append(f)
-
-        self._check_formats(formats, video_guid)
-
-        return {
-            'id': video_guid,
-            'title': title,
-            'description': self._html_search_meta(['description', 'og:description'], page),
-            'thumbnail': self._html_search_meta(['og:image:secure_url', 'og:image', 'twitter:image'], page),
-            'duration': parse_duration(self._html_search_meta('duration', page)),
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/nick.py b/yt_dlp/extractor/nick.py
index de22cb8d6..165d8ce9d 100644
--- a/yt_dlp/extractor/nick.py
+++ b/yt_dlp/extractor/nick.py
@@ -188,26 +188,6 @@ def _get_feed_url(self, uri, url=None):
         return self._remove_template_parameter(config['feedWithQueryParams'])
 
 
-class NickNightIE(NickDeIE):  # XXX: Do not subclass from concrete IE
-    IE_NAME = 'nicknight'
-    _VALID_URL = r'https?://(?:www\.)(?P<host>nicknight\.(?:de|at|tv))/(?:playlist|shows)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'http://www.nicknight.at/shows/977-awkward/videos/85987-nimmer-beste-freunde',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.nicknight.at/shows/977-awkward',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.nicknight.at/shows/1900-faking-it',
-        'only_matching': True,
-    }]
-
-    def _extract_mrss_url(self, webpage, *args):
-        return self._search_regex(
-            r'mrss\s*:\s*(["\'])(?P<url>http.+?)\1', webpage,
-            'mrss url', group='url')
-
-
 class NickRuIE(MTVServicesInfoExtractor):
     IE_NAME = 'nickelodeonru'
     _VALID_URL = r'https?://(?:www\.)nickelodeon\.(?:ru|fr|es|pt|ro|hu|com\.tr)/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
diff --git a/yt_dlp/extractor/normalboots.py b/yt_dlp/extractor/normalboots.py
deleted file mode 100644
index 07babcd2c..000000000
--- a/yt_dlp/extractor/normalboots.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from .common import InfoExtractor
-from .jwplatform import JWPlatformIE
-
-from ..utils import (
-    unified_strdate,
-)
-
-
-class NormalbootsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
-    _TEST = {
-        'url': 'http://normalboots.com/video/home-alone-games-jontron/',
-        'info_dict': {
-            'id': 'home-alone-games-jontron',
-            'ext': 'mp4',
-            'title': 'Home Alone Games - JonTron - NormalBoots',
-            'description': 'Jon is late for Christmas. Typical. Thanks to: Paul Ritchey for Co-Writing/Filming: http://www.youtube.com/user/ContinueShow Michael Azzi for Christmas Intro Animation: http://michafrar.tumblr.com/ Jerrod Waters for Christmas Intro Music: http://www.youtube.com/user/xXJerryTerryXx Casey Ormond for ‘Tense Battle Theme’:\xa0http://www.youtube.com/Kiamet/',
-            'uploader': 'JonTron',
-            'upload_date': '20140125',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-        'add_ie': ['JWPlatform'],
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        video_uploader = self._html_search_regex(
-            r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
-            webpage, 'uploader', fatal=False)
-        video_upload_date = unified_strdate(self._html_search_regex(
-            r'<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
-            webpage, 'date', fatal=False))
-
-        jwplatform_url = JWPlatformIE._extract_url(webpage)
-
-        return {
-            '_type': 'url_transparent',
-            'id': video_id,
-            'url': jwplatform_url,
-            'ie_key': JWPlatformIE.ie_key(),
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader': video_uploader,
-            'upload_date': video_upload_date,
-        }
diff --git a/yt_dlp/extractor/nosvideo.py b/yt_dlp/extractor/nosvideo.py
deleted file mode 100644
index 7e9688c0b..000000000
--- a/yt_dlp/extractor/nosvideo.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..networking import Request
-from ..utils import (
-    ExtractorError,
-    urlencode_postdata,
-    xpath_text,
-    xpath_with_ns,
-)
-
-_x = lambda p: xpath_with_ns(p, {'xspf': 'http://xspf.org/ns/0/'})
-
-
-class NosVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nosvideo\.com/' + \
-                 r'(?:embed/|\?v=)(?P<id>[A-Za-z0-9]{12})/?'
-    _PLAYLIST_URL = 'http://nosvideo.com/xml/{xml_id:s}.xml'
-    _FILE_DELETED_REGEX = r'<b>File Not Found</b>'
-    _TEST = {
-        'url': 'http://nosvideo.com/?v=mu8fle7g7rpq',
-        'md5': '6124ed47130d8be3eacae635b071e6b6',
-        'info_dict': {
-            'id': 'mu8fle7g7rpq',
-            'ext': 'mp4',
-            'title': 'big_buck_bunny_480p_surround-fix.avi.mp4',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        fields = {
-            'id': video_id,
-            'op': 'download1',
-            'method_free': 'Continue to Video',
-        }
-        req = Request(url, urlencode_postdata(fields))
-        req.headers['Content-type'] = 'application/x-www-form-urlencoded'
-        webpage = self._download_webpage(req, video_id,
-                                         'Downloading download page')
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id,
-                                 expected=True)
-
-        xml_id = self._search_regex(r'php\|([^\|]+)\|', webpage, 'XML ID')
-        playlist_url = self._PLAYLIST_URL.format(xml_id=xml_id)
-        playlist = self._download_xml(playlist_url, video_id)
-
-        track = playlist.find(_x('.//xspf:track'))
-        if track is None:
-            raise ExtractorError(
-                'XML playlist is missing the \'track\' element',
-                expected=True)
-        title = xpath_text(track, _x('./xspf:title'), 'title')
-        url = xpath_text(track, _x('./xspf:file'), 'URL', fatal=True)
-        thumbnail = xpath_text(track, _x('./xspf:image'), 'thumbnail')
-        if title is not None:
-            title = title.strip()
-
-        formats = [{
-            'format_id': 'sd',
-            'url': url,
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/nrl.py b/yt_dlp/extractor/nrl.py
index 798d03417..1e8cf0b75 100644
--- a/yt_dlp/extractor/nrl.py
+++ b/yt_dlp/extractor/nrl.py
@@ -2,6 +2,7 @@
 
 
 class NRLTVIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'https?://(?:www\.)?nrl\.com/tv(/[^/]+)*/(?P<id>[^/?&#]+)'
     _TEST = {
         'url': 'https://www.nrl.com/tv/news/match-highlights-titans-v-knights-862805/',
diff --git a/yt_dlp/extractor/ooyala.py b/yt_dlp/extractor/ooyala.py
deleted file mode 100644
index 65afccdb1..000000000
--- a/yt_dlp/extractor/ooyala.py
+++ /dev/null
@@ -1,230 +0,0 @@
-import base64
-import re
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_b64decode,
-    compat_str,
-)
-from ..utils import (
-    determine_ext,
-    float_or_none,
-    int_or_none,
-    smuggle_url,
-    try_get,
-    unsmuggle_url,
-)
-
-
-class OoyalaBaseIE(InfoExtractor):
-    _PLAYER_BASE = 'http://player.ooyala.com/'
-    _CONTENT_TREE_BASE = _PLAYER_BASE + 'player_api/v1/content_tree/'
-    _AUTHORIZATION_URL_TEMPLATE = _PLAYER_BASE + 'sas/player_api/v2/authorization/embed_code/%s/%s'
-
-    def _extract(self, content_tree_url, video_id, domain=None, supportedformats=None, embed_token=None):
-        content_tree = self._download_json(content_tree_url, video_id)['content_tree']
-        metadata = content_tree[list(content_tree)[0]]
-        embed_code = metadata['embed_code']
-        pcode = metadata.get('asset_pcode') or embed_code
-        title = metadata['title']
-
-        auth_data = self._download_json(
-            self._AUTHORIZATION_URL_TEMPLATE % (pcode, embed_code),
-            video_id, headers=self.geo_verification_headers(), query={
-                'domain': domain or 'player.ooyala.com',
-                'supportedFormats': supportedformats or 'mp4,rtmp,m3u8,hds,dash,smooth',
-                'embedToken': embed_token,
-            })['authorization_data'][embed_code]
-
-        urls = []
-        formats = []
-        streams = auth_data.get('streams') or [{
-            'delivery_type': 'hls',
-            'url': {
-                'data': base64.b64encode(('http://player.ooyala.com/hls/player/all/%s.m3u8' % embed_code).encode()).decode(),
-            }
-        }]
-        for stream in streams:
-            url_data = try_get(stream, lambda x: x['url']['data'], compat_str)
-            if not url_data:
-                continue
-            s_url = compat_b64decode(url_data).decode('utf-8')
-            if not s_url or s_url in urls:
-                continue
-            urls.append(s_url)
-            ext = determine_ext(s_url, None)
-            delivery_type = stream.get('delivery_type')
-            if delivery_type == 'hls' or ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    re.sub(r'/ip(?:ad|hone)/', '/all/', s_url), embed_code, 'mp4', 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-            elif delivery_type == 'hds' or ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    s_url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
-            elif delivery_type == 'dash' or ext == 'mpd':
-                formats.extend(self._extract_mpd_formats(
-                    s_url, embed_code, mpd_id='dash', fatal=False))
-            elif delivery_type == 'smooth':
-                self._extract_ism_formats(
-                    s_url, embed_code, ism_id='mss', fatal=False)
-            elif ext == 'smil':
-                formats.extend(self._extract_smil_formats(
-                    s_url, embed_code, fatal=False))
-            else:
-                formats.append({
-                    'url': s_url,
-                    'ext': ext or delivery_type,
-                    'vcodec': stream.get('video_codec'),
-                    'format_id': delivery_type,
-                    'width': int_or_none(stream.get('width')),
-                    'height': int_or_none(stream.get('height')),
-                    'abr': int_or_none(stream.get('audio_bitrate')),
-                    'vbr': int_or_none(stream.get('video_bitrate')),
-                    'fps': float_or_none(stream.get('framerate')),
-                })
-        if not formats and not auth_data.get('authorized'):
-            self.raise_no_formats('%s said: %s' % (
-                self.IE_NAME, auth_data['message']), expected=True)
-
-        subtitles = {}
-        for lang, sub in metadata.get('closed_captions_vtt', {}).get('captions', {}).items():
-            sub_url = sub.get('url')
-            if not sub_url:
-                continue
-            subtitles[lang] = [{
-                'url': sub_url,
-            }]
-
-        return {
-            'id': embed_code,
-            'title': title,
-            'description': metadata.get('description'),
-            'thumbnail': metadata.get('thumbnail_image') or metadata.get('promo_image'),
-            'duration': float_or_none(metadata.get('duration'), 1000),
-            'subtitles': subtitles,
-            'formats': formats,
-        }
-
-
-class OoyalaIE(OoyalaBaseIE):
-    _VALID_URL = r'(?:ooyala:|https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=)(?P<id>.+?)(&|$)'
-
-    _TESTS = [
-        {
-            # From http://it.slashdot.org/story/13/04/25/178216/recovering-data-from-broken-hard-drives-and-ssds-video
-            'url': 'http://player.ooyala.com/player.js?embedCode=pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
-            'info_dict': {
-                'id': 'pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
-                'ext': 'mp4',
-                'title': 'Explaining Data Recovery from Hard Drives and SSDs',
-                'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
-                'duration': 853.386,
-            },
-            # The video in the original webpage now uses PlayWire
-            'skip': 'Ooyala said: movie expired',
-        }, {
-            # Only available for ipad
-            'url': 'http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
-            'info_dict': {
-                'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
-                'ext': 'mp4',
-                'title': 'Simulation Overview - Levels of Simulation',
-                'duration': 194.948,
-            },
-        },
-        {
-            # Information available only through SAS api
-            # From http://community.plm.automation.siemens.com/t5/News-NX-Manufacturing/Tool-Path-Divide/ba-p/4187
-            'url': 'http://player.ooyala.com/player.js?embedCode=FiOG81ZTrvckcchQxmalf4aQj590qTEx',
-            'md5': 'a84001441b35ea492bc03736e59e7935',
-            'info_dict': {
-                'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx',
-                'ext': 'mp4',
-                'title': 'Divide Tool Path.mp4',
-                'duration': 204.405,
-            }
-        },
-        {
-            # empty stream['url']['data']
-            'url': 'http://player.ooyala.com/player.js?embedCode=w2bnZtYjE6axZ_dw1Cd0hQtXd_ige2Is',
-            'only_matching': True,
-        }
-    ]
-
-    def _extract_from_webpage(self, url, webpage):
-        mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage)
-                or re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage)
-                or re.search(r'OO\.Player\.create\.apply\(\s*OO\.Player\s*,\s*op\(\s*\[\s*[\'"][^\'"]*[\'"]\s*,\s*[\'"](?P<ec>.{32})[\'"]', webpage)
-                or re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage)
-                or re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
-        if mobj is not None:
-            embed_token = self._search_regex(
-                r'embedToken[\'"]?\s*:\s*[\'"]([^\'"]+)',
-                webpage, 'ooyala embed token', default=None)
-            yield self._build_url_result(smuggle_url(
-                mobj.group('ec'), {
-                    'domain': url,
-                    'embed_token': embed_token,
-                }))
-            return
-
-        # Look for multiple Ooyala embeds on SBN network websites
-        mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
-        if mobj is not None:
-            for v in self._parse_json(mobj.group(1), self._generic_id(url), fatal=False) or []:
-                yield self._build_url_result(smuggle_url(v['provider_video_id'], {'domain': url}))
-
-    @staticmethod
-    def _url_for_embed_code(embed_code):
-        return 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code
-
-    @classmethod
-    def _build_url_result(cls, embed_code):
-        return cls.url_result(cls._url_for_embed_code(embed_code),
-                              ie=cls.ie_key())
-
-    def _real_extract(self, url):
-        url, smuggled_data = unsmuggle_url(url, {})
-        embed_code = self._match_id(url)
-        domain = smuggled_data.get('domain')
-        supportedformats = smuggled_data.get('supportedformats')
-        embed_token = smuggled_data.get('embed_token')
-        content_tree_url = self._CONTENT_TREE_BASE + 'embed_code/%s/%s' % (embed_code, embed_code)
-        return self._extract(content_tree_url, embed_code, domain, supportedformats, embed_token)
-
-
-class OoyalaExternalIE(OoyalaBaseIE):
-    _VALID_URL = r'''(?x)
-                    (?:
-                        ooyalaexternal:|
-                        https?://.+?\.ooyala\.com/.*?\bexternalId=
-                    )
-                    (?P<partner_id>[^:]+)
-                    :
-                    (?P<id>.+)
-                    (?:
-                        :|
-                        .*?&pcode=
-                    )
-                    (?P<pcode>.+?)
-                    (?:&|$)
-                    '''
-
-    _TEST = {
-        'url': 'https://player.ooyala.com/player.js?externalId=espn:10365079&pcode=1kNG061cgaoolOncv54OAO1ceO-I&adSetCode=91cDU6NuXTGKz3OdjOxFdAgJVtQcKJnI&callback=handleEvents&hasModuleParams=1&height=968&playerBrandingId=7af3bd04449c444c964f347f11873075&targetReplaceId=videoPlayer&width=1656&wmode=opaque&allowScriptAccess=always',
-        'info_dict': {
-            'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
-            'ext': 'mp4',
-            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
-            'duration': 1302.0,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
-
-    def _real_extract(self, url):
-        partner_id, video_id, pcode = self._match_valid_url(url).groups()
-        content_tree_url = self._CONTENT_TREE_BASE + 'external_id/%s/%s:%s' % (pcode, partner_id, video_id)
-        return self._extract(content_tree_url, video_id)
diff --git a/yt_dlp/extractor/pandoratv.py b/yt_dlp/extractor/pandoratv.py
deleted file mode 100644
index ccc78da57..000000000
--- a/yt_dlp/extractor/pandoratv.py
+++ /dev/null
@@ -1,128 +0,0 @@
-from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-)
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    parse_duration,
-    parse_qs,
-    str_to_int,
-    urlencode_postdata,
-)
-
-
-class PandoraTVIE(InfoExtractor):
-    IE_NAME = 'pandora.tv'
-    IE_DESC = '판도라TV'
-    _VALID_URL = r'''(?x)
-                        https?://
-                            (?:
-                                (?:www\.)?pandora\.tv/view/(?P<user_id>[^/]+)/(?P<id>\d+)|  # new format
-                                (?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?|        # old format
-                                m\.pandora\.tv/?\?                                          # mobile
-                            )
-                    '''
-    _TESTS = [{
-        'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',
-        'info_dict': {
-            'id': '53294230',
-            'ext': 'flv',
-            'title': '頭を撫でてくれる？',
-            'description': '頭を撫でてくれる？',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 39,
-            'upload_date': '20151218',
-            'uploader': 'カワイイ動物まとめ',
-            'uploader_id': 'mikakim',
-            'view_count': int,
-            'like_count': int,
-        }
-    }, {
-        'url': 'http://channel.pandora.tv/channel/video.ptv?ch_userid=gogoucc&prgid=54721744',
-        'info_dict': {
-            'id': '54721744',
-            'ext': 'flv',
-            'title': '[HD] JAPAN COUNTDOWN 170423',
-            'description': '[HD] JAPAN COUNTDOWN 170423',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 1704.9,
-            'upload_date': '20170423',
-            'uploader': 'GOGO_UCC',
-            'uploader_id': 'gogoucc',
-            'view_count': int,
-            'like_count': int,
-        },
-        'params': {
-            # Test metadata only
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://www.pandora.tv/view/mikakim/53294230#36797454_new',
-        'only_matching': True,
-    }, {
-        'url': 'http://m.pandora.tv/?c=view&ch_userid=mikakim&prgid=54600346',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        user_id = mobj.group('user_id')
-        video_id = mobj.group('id')
-
-        if not user_id or not video_id:
-            qs = parse_qs(url)
-            video_id = qs.get('prgid', [None])[0]
-            user_id = qs.get('ch_userid', [None])[0]
-            if any(not f for f in (video_id, user_id,)):
-                raise ExtractorError('Invalid URL', expected=True)
-
-        data = self._download_json(
-            'http://m.pandora.tv/?c=view&m=viewJsonApi&ch_userid=%s&prgid=%s'
-            % (user_id, video_id), video_id)
-
-        info = data['data']['rows']['vod_play_info']['result']
-
-        formats = []
-        for format_id, format_url in info.items():
-            if not format_url:
-                continue
-            height = self._search_regex(
-                r'^v(\d+)[Uu]rl$', format_id, 'height', default=None)
-            if not height:
-                continue
-
-            play_url = self._download_json(
-                'http://m.pandora.tv/?c=api&m=play_url', video_id,
-                data=urlencode_postdata({
-                    'prgid': video_id,
-                    'runtime': info.get('runtime'),
-                    'vod_url': format_url,
-                }),
-                headers={
-                    'Origin': url,
-                    'Content-Type': 'application/x-www-form-urlencoded',
-                })
-            format_url = play_url.get('url')
-            if not format_url:
-                continue
-
-            formats.append({
-                'format_id': '%sp' % height,
-                'url': format_url,
-                'height': int(height),
-            })
-
-        return {
-            'id': video_id,
-            'title': info['subject'],
-            'description': info.get('body'),
-            'thumbnail': info.get('thumbnail') or info.get('poster'),
-            'duration': float_or_none(info.get('runtime'), 1000) or parse_duration(info.get('time')),
-            'upload_date': info['fid'].split('/')[-1][:8] if isinstance(info.get('fid'), compat_str) else None,
-            'uploader': info.get('nickname'),
-            'uploader_id': info.get('upload_userid'),
-            'view_count': str_to_int(info.get('hit')),
-            'like_count': str_to_int(info.get('likecnt')),
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/people.py b/yt_dlp/extractor/people.py
deleted file mode 100644
index c5143c3ed..000000000
--- a/yt_dlp/extractor/people.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from .common import InfoExtractor
-
-
-class PeopleIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?people\.com/people/videos/0,,(?P<id>\d+),00\.html'
-
-    _TEST = {
-        'url': 'http://www.people.com/people/videos/0,,20995451,00.html',
-        'info_dict': {
-            'id': 'ref:20995451',
-            'ext': 'mp4',
-            'title': 'Astronaut Love Triangle Victim Speaks Out: “The Crime in 2007 Hasn’t Defined Us”',
-            'description': 'Colleen Shipman speaks to PEOPLE for the first time about life after the attack',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 246.318,
-            'timestamp': 1458720585,
-            'upload_date': '20160323',
-            'uploader_id': '416418724',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'add_ie': ['BrightcoveNew'],
-    }
-
-    def _real_extract(self, url):
-        return self.url_result(
-            'http://players.brightcove.net/416418724/default_default/index.html?videoId=ref:%s'
-            % self._match_id(url), 'BrightcoveNew')
diff --git a/yt_dlp/extractor/playfm.py b/yt_dlp/extractor/playfm.py
deleted file mode 100644
index e895ba480..000000000
--- a/yt_dlp/extractor/playfm.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    parse_iso8601,
-)
-
-
-class PlayFMIE(InfoExtractor):
-    IE_NAME = 'play.fm'
-    _VALID_URL = r'https?://(?:www\.)?play\.fm/(?P<slug>(?:[^/]+/)+(?P<id>[^/]+))/?(?:$|[?#])'
-
-    _TEST = {
-        'url': 'https://www.play.fm/dan-drastic/sven-tasnadi-leipzig-electronic-music-batofar-paris-fr-2014-07-12',
-        'md5': 'c505f8307825a245d0c7ad1850001f22',
-        'info_dict': {
-            'id': '71276',
-            'ext': 'mp3',
-            'title': 'Sven Tasnadi - LEIPZIG ELECTRONIC MUSIC @ Batofar (Paris,FR) - 2014-07-12',
-            'description': '',
-            'duration': 5627,
-            'timestamp': 1406033781,
-            'upload_date': '20140722',
-            'uploader': 'Dan Drastic',
-            'uploader_id': '71170',
-            'view_count': int,
-            'comment_count': int,
-        },
-    }
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-        slug = mobj.group('slug')
-
-        recordings = self._download_json(
-            'http://v2api.play.fm/recordings/slug/%s' % slug, video_id)
-
-        error = recordings.get('error')
-        if isinstance(error, dict):
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error.get('message')),
-                expected=True)
-
-        audio_url = recordings['audio']
-        video_id = compat_str(recordings.get('id') or video_id)
-        title = recordings['title']
-        description = recordings.get('description')
-        duration = int_or_none(recordings.get('recordingDuration'))
-        timestamp = parse_iso8601(recordings.get('created_at'))
-        uploader = recordings.get('page', {}).get('title')
-        uploader_id = compat_str(recordings.get('page', {}).get('id'))
-        view_count = int_or_none(recordings.get('playCount'))
-        comment_count = int_or_none(recordings.get('commentCount'))
-        categories = [tag['name'] for tag in recordings.get('tags', []) if tag.get('name')]
-
-        return {
-            'id': video_id,
-            'url': audio_url,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'timestamp': timestamp,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'view_count': view_count,
-            'comment_count': comment_count,
-            'categories': categories,
-        }
diff --git a/yt_dlp/extractor/plays.py b/yt_dlp/extractor/plays.py
deleted file mode 100644
index 9371f7b23..000000000
--- a/yt_dlp/extractor/plays.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class PlaysTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?plays\.tv/(?:video|embeds)/(?P<id>[0-9a-f]{18})'
-    _TESTS = [{
-        'url': 'https://plays.tv/video/56af17f56c95335490/when-you-outplay-the-azir-wall',
-        'md5': 'dfeac1198506652b5257a62762cec7bc',
-        'info_dict': {
-            'id': '56af17f56c95335490',
-            'ext': 'mp4',
-            'title': 'Bjergsen - When you outplay the Azir wall',
-            'description': 'Posted by Bjergsen',
-        }
-    }, {
-        'url': 'https://plays.tv/embeds/56af17f56c95335490',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(
-            'https://plays.tv/video/%s' % video_id, video_id)
-
-        info = self._search_json_ld(webpage, video_id,)
-
-        mpd_url, sources = re.search(
-            r'(?s)<video[^>]+data-mpd="([^"]+)"[^>]*>(.+?)</video>',
-            webpage).groups()
-        formats = self._extract_mpd_formats(
-            self._proto_relative_url(mpd_url), video_id, mpd_id='DASH')
-        for format_id, height, format_url in re.findall(r'<source\s+res="((\d+)h?)"\s+src="([^"]+)"', sources):
-            formats.append({
-                'url': self._proto_relative_url(format_url),
-                'format_id': 'http-' + format_id,
-                'height': int_or_none(height),
-            })
-
-        info.update({
-            'id': video_id,
-            'description': self._og_search_description(webpage),
-            'thumbnail': info.get('thumbnail') or self._og_search_thumbnail(webpage),
-            'formats': formats,
-        })
-
-        return info
diff --git a/yt_dlp/extractor/playvid.py b/yt_dlp/extractor/playvid.py
deleted file mode 100644
index 1e0989d0a..000000000
--- a/yt_dlp/extractor/playvid.py
+++ /dev/null
@@ -1,90 +0,0 @@
-import re
-import urllib.parse
-
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
-from ..utils import ExtractorError, clean_html
-
-
-class PlayvidIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
-    _TESTS = [{
-        'url': 'http://www.playvid.com/watch/RnmBNgtrrJu',
-        'md5': 'ffa2f6b2119af359f544388d8c01eb6c',
-        'info_dict': {
-            'id': 'RnmBNgtrrJu',
-            'ext': 'mp4',
-            'title': 'md5:9256d01c6317e3f703848b5906880dc8',
-            'duration': 82,
-            'age_limit': 18,
-        },
-        'skip': 'Video removed due to ToS',
-    }, {
-        'url': 'http://www.playvid.com/watch/hwb0GpNkzgH',
-        'md5': '39d49df503ad7b8f23a4432cbf046477',
-        'info_dict': {
-            'id': 'hwb0GpNkzgH',
-            'ext': 'mp4',
-            'title': 'Ellen Euro Cutie Blond Takes a Sexy Survey Get Facial in The Park',
-            'age_limit': 18,
-            'thumbnail': r're:^https?://.*\.jpg$',
-        },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        m_error = re.search(
-            r'<div class="block-error">\s*<div class="heading">\s*<div>(?P<msg>.+?)</div>\s*</div>', webpage)
-        if m_error:
-            raise ExtractorError(clean_html(m_error.group('msg')), expected=True)
-
-        video_title = None
-        duration = None
-        video_thumbnail = None
-        formats = []
-
-        # most of the information is stored in the flashvars
-        flashvars = self._html_search_regex(
-            r'flashvars="(.+?)"', webpage, 'flashvars')
-
-        infos = compat_urllib_parse_unquote(flashvars).split(r'&')
-        for info in infos:
-            videovars_match = re.match(r'^video_vars\[(.+?)\]=(.+?)$', info)
-            if videovars_match:
-                key = videovars_match.group(1)
-                val = videovars_match.group(2)
-
-                if key == 'title':
-                    video_title = urllib.parse.unquote_plus(val)
-                if key == 'duration':
-                    try:
-                        duration = int(val)
-                    except ValueError:
-                        pass
-                if key == 'big_thumb':
-                    video_thumbnail = val
-
-                videourl_match = re.match(
-                    r'^video_urls\]\[(?P<resolution>[0-9]+)p', key)
-                if videourl_match:
-                    height = int(videourl_match.group('resolution'))
-                    formats.append({
-                        'height': height,
-                        'url': val,
-                    })
-
-        # Extract title - should be in the flashvars; if not, look elsewhere
-        if video_title is None:
-            video_title = self._html_extract_title(webpage)
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': video_title,
-            'thumbnail': video_thumbnail,
-            'duration': duration,
-            'description': None,
-            'age_limit': 18
-        }
diff --git a/yt_dlp/extractor/porncom.py b/yt_dlp/extractor/porncom.py
deleted file mode 100644
index c8ef240d7..000000000
--- a/yt_dlp/extractor/porncom.py
+++ /dev/null
@@ -1,99 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    int_or_none,
-    js_to_json,
-    parse_filesize,
-    str_to_int,
-)
-
-
-class PornComIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[a-zA-Z]+\.)?porn\.com/videos/(?:(?P<display_id>[^/]+)-)?(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://www.porn.com/videos/teen-grabs-a-dildo-and-fucks-her-pussy-live-on-1hottie-i-rec-2603339',
-        'md5': '3f30ce76267533cd12ba999263156de7',
-        'info_dict': {
-            'id': '2603339',
-            'display_id': 'teen-grabs-a-dildo-and-fucks-her-pussy-live-on-1hottie-i-rec',
-            'ext': 'mp4',
-            'title': 'Teen grabs a dildo and fucks her pussy live on 1hottie, I rec',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 551,
-            'view_count': int,
-            'age_limit': 18,
-            'categories': list,
-            'tags': list,
-        },
-    }, {
-        'url': 'http://se.porn.com/videos/marsha-may-rides-seth-on-top-of-his-thick-cock-2658067',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
-
-        webpage = self._download_webpage(url, display_id)
-
-        config = self._parse_json(
-            self._search_regex(
-                (r'=\s*({.+?})\s*;\s*v1ar\b',
-                 r'=\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*='),
-                webpage, 'config', default='{}'),
-            display_id, transform_source=js_to_json, fatal=False)
-
-        if config:
-            title = config['title']
-            formats = [{
-                'url': stream['url'],
-                'format_id': stream.get('id'),
-                'height': int_or_none(self._search_regex(
-                    r'^(\d+)[pP]', stream.get('id') or '', 'height', default=None))
-            } for stream in config['streams'] if stream.get('url')]
-            thumbnail = (compat_urlparse.urljoin(
-                config['thumbCDN'], config['poster'])
-                if config.get('thumbCDN') and config.get('poster') else None)
-            duration = int_or_none(config.get('length'))
-        else:
-            title = self._search_regex(
-                (r'<title>([^<]+)</title>', r'<h1[^>]*>([^<]+)</h1>'),
-                webpage, 'title')
-            formats = [{
-                'url': compat_urlparse.urljoin(url, format_url),
-                'format_id': '%sp' % height,
-                'height': int(height),
-                'filesize_approx': parse_filesize(filesize),
-            } for format_url, height, filesize in re.findall(
-                r'<a[^>]+href="(/download/[^"]+)">[^<]*?(\d+)p<span[^>]*>(\d+\s*[a-zA-Z]+)<',
-                webpage)]
-            thumbnail = None
-            duration = None
-
-        view_count = str_to_int(self._search_regex(
-            (r'Views:\s*</span>\s*<span>\s*([\d,.]+)',
-             r'class=["\']views["\'][^>]*><p>([\d,.]+)'), webpage,
-            'view count', fatal=False))
-
-        def extract_list(kind):
-            s = self._search_regex(
-                (r'(?s)%s:\s*</span>\s*<span>(.+?)</span>' % kind.capitalize(),
-                 r'(?s)<p[^>]*>%s:(.+?)</p>' % kind.capitalize()),
-                webpage, kind, fatal=False)
-            return re.findall(r'<a[^>]+>([^<]+)</a>', s or '')
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'view_count': view_count,
-            'formats': formats,
-            'age_limit': 18,
-            'categories': extract_list('categories'),
-            'tags': extract_list('tags'),
-        }
diff --git a/yt_dlp/extractor/pornez.py b/yt_dlp/extractor/pornez.py
deleted file mode 100644
index bc45f865e..000000000
--- a/yt_dlp/extractor/pornez.py
+++ /dev/null
@@ -1,60 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    int_or_none,
-    get_element_by_class,
-    urljoin,
-)
-
-
-class PornezIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornez\.net/(?:video(?P<id>\w+)|watch)/'
-    _TESTS = [{
-        'url': 'https://pornez.net/video344819/mistresst-funny_penis_names-wmv/',
-        'info_dict': {
-            'id': '344819',
-            'ext': 'mp4',
-            'title': 'mistresst funny_penis_names wmv',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'age_limit': 18,
-        },
-        'params': {'skip_download': 'm3u8'},
-    }, {
-        'url': 'https://pornez.net/watch/leana+lovings+stiff+for+stepdaughter/',
-        'info_dict': {
-            'id': '156161',
-            'ext': 'mp4',
-            'title': 'Watch leana lovings stiff for stepdaughter porn video.',
-            'age_limit': 18,
-        },
-        'params': {'skip_download': 'm3u8'},
-    }, {
-        'url': 'https://pornez.net/videovzs27fj/tutor4k-e14-blue-wave-1080p-nbq-tutor4k-e14-blue-wave/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        if not video_id:
-            video_id = self._search_regex(
-                r'<link[^>]+\bhref=["\']https?://pornez.net/\?p=(\w+)["\']', webpage, 'id')
-
-        iframe_src = self._html_search_regex(r'<iframe[^>]+src="([^"]+)"', webpage, 'iframe')
-        iframe = self._download_webpage(urljoin('https://pornez.net', iframe_src), video_id)
-
-        entries = self._parse_html5_media_entries(iframe_src, iframe, video_id)[0]
-        for fmt in entries['formats']:
-            height = self._search_regex(r'_(\d+)\.m3u8', fmt['url'], 'height')
-            fmt['format_id'] = '%sp' % height
-            fmt['height'] = int_or_none(height)
-
-        entries.update({
-            'id': video_id,
-            'title': (clean_html(get_element_by_class('video-title', webpage))
-                      or self._html_search_meta(
-                      ['twitter:title', 'og:title', 'description'], webpage, 'title', default=None)),
-            'thumbnail': self._html_search_meta(['thumbnailUrl'], webpage, 'thumb', default=None),
-            'age_limit': 18,
-        })
-        return entries
diff --git a/yt_dlp/extractor/pornhd.py b/yt_dlp/extractor/pornhd.py
deleted file mode 100644
index c8a1ec80b..000000000
--- a/yt_dlp/extractor/pornhd.py
+++ /dev/null
@@ -1,116 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-    int_or_none,
-    js_to_json,
-    merge_dicts,
-    urljoin,
-)
-
-
-class PornHdIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
-    _TESTS = [{
-        'url': 'http://www.pornhd.com/videos/9864/selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
-        'md5': '87f1540746c1d32ec7a2305c12b96b25',
-        'info_dict': {
-            'id': '9864',
-            'display_id': 'selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
-            'ext': 'mp4',
-            'title': 'Restroom selfie masturbation',
-            'description': 'md5:3748420395e03e31ac96857a8f125b2b',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'view_count': int,
-            'like_count': int,
-            'age_limit': 18,
-        },
-        'skip': 'HTTP Error 404: Not Found',
-    }, {
-        'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
-        'md5': '1b7b3a40b9d65a8e5b25f7ab9ee6d6de',
-        'info_dict': {
-            'id': '1962',
-            'display_id': 'sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
-            'ext': 'mp4',
-            'title': 'md5:98c6f8b2d9c229d0f0fde47f61a1a759',
-            'description': 'md5:8ff0523848ac2b8f9b065ba781ccf294',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'view_count': int,
-            'like_count': int,
-            'age_limit': 18,
-        },
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-
-        webpage = self._download_webpage(url, display_id or video_id)
-
-        title = self._html_search_regex(
-            [r'<span[^>]+class=["\']video-name["\'][^>]*>([^<]+)',
-             r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
-
-        sources = self._parse_json(js_to_json(self._search_regex(
-            r"(?s)sources'?\s*[:=]\s*(\{.+?\})",
-            webpage, 'sources', default='{}')), video_id)
-
-        info = {}
-        if not sources:
-            entries = self._parse_html5_media_entries(url, webpage, video_id)
-            if entries:
-                info = entries[0]
-
-        if not sources and not info:
-            message = self._html_search_regex(
-                r'(?s)<(div|p)[^>]+class="no-video"[^>]*>(?P<value>.+?)</\1',
-                webpage, 'error message', group='value')
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
-
-        formats = []
-        for format_id, video_url in sources.items():
-            video_url = urljoin(url, video_url)
-            if not video_url:
-                continue
-            height = int_or_none(self._search_regex(
-                r'^(\d+)[pP]', format_id, 'height', default=None))
-            formats.append({
-                'url': video_url,
-                'ext': determine_ext(video_url, 'mp4'),
-                'format_id': format_id,
-                'height': height,
-            })
-        if formats:
-            info['formats'] = formats
-
-        description = self._html_search_regex(
-            (r'(?s)<section[^>]+class=["\']video-description[^>]+>(?P<value>.+?)</section>',
-             r'<(div|p)[^>]+class="description"[^>]*>(?P<value>[^<]+)</\1'),
-            webpage, 'description', fatal=False,
-            group='value') or self._html_search_meta(
-            'description', webpage, default=None) or self._og_search_description(webpage)
-        view_count = int_or_none(self._html_search_regex(
-            r'(\d+) views\s*<', webpage, 'view count', fatal=False))
-        thumbnail = self._search_regex(
-            r"poster'?\s*:\s*([\"'])(?P<url>(?:(?!\1).)+)\1", webpage,
-            'thumbnail', default=None, group='url')
-
-        like_count = int_or_none(self._search_regex(
-            (r'(\d+)</span>\s*likes',
-             r'(\d+)\s*</11[^>]+>(?:&nbsp;|\s)*\blikes',
-             r'class=["\']save-count["\'][^>]*>\s*(\d+)'),
-            webpage, 'like count', fatal=False))
-
-        return merge_dicts(info, {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'view_count': view_count,
-            'like_count': like_count,
-            'formats': formats,
-            'age_limit': 18,
-        })
diff --git a/yt_dlp/extractor/radiobremen.py b/yt_dlp/extractor/radiobremen.py
deleted file mode 100644
index 99ba050d0..000000000
--- a/yt_dlp/extractor/radiobremen.py
+++ /dev/null
@@ -1,59 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import parse_duration
-
-
-class RadioBremenIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?radiobremen\.de/mediathek/(?:index\.html)?\?id=(?P<id>[0-9]+)'
-    IE_NAME = 'radiobremen'
-
-    _TEST = {
-        'url': 'http://www.radiobremen.de/mediathek/?id=141876',
-        'info_dict': {
-            'id': '141876',
-            'ext': 'mp4',
-            'duration': 178,
-            'width': 512,
-            'title': 'Druck auf Patrick Öztürk',
-            'thumbnail': r're:https?://.*\.jpg$',
-            'description': 'Gegen den SPD-Bürgerschaftsabgeordneten Patrick Öztürk wird wegen Beihilfe zum gewerbsmäßigen Betrug ermittelt. Am Donnerstagabend sollte er dem Vorstand des SPD-Unterbezirks Bremerhaven dazu Rede und Antwort stehen.',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        meta_url = 'http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s' % video_id
-        meta_doc = self._download_webpage(
-            meta_url, video_id, 'Downloading metadata')
-        title = self._html_search_regex(
-            r'<h1.*>(?P<title>.+)</h1>', meta_doc, 'title')
-        description = self._html_search_regex(
-            r'<p>(?P<description>.*)</p>', meta_doc, 'description', fatal=False)
-        duration = parse_duration(self._html_search_regex(
-            r'L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>',
-            meta_doc, 'duration', fatal=False))
-
-        page_doc = self._download_webpage(
-            url, video_id, 'Downloading video information')
-        mobj = re.search(
-            r"ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)",
-            page_doc)
-        video_url = (
-            "http://dl-ondemand.radiobremen.de/mediabase/%s/%s_%s_%s.mp4" %
-            (video_id, video_id, mobj.group("secret"), mobj.group('width')))
-
-        formats = [{
-            'url': video_url,
-            'ext': 'mp4',
-            'width': int(mobj.group('width')),
-        }]
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'formats': formats,
-            'thumbnail': mobj.group('thumbnail'),
-        }
diff --git a/yt_dlp/extractor/recurbate.py b/yt_dlp/extractor/recurbate.py
deleted file mode 100644
index d7294cb14..000000000
--- a/yt_dlp/extractor/recurbate.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from .common import InfoExtractor
-from ..networking.exceptions import HTTPError
-from ..utils import ExtractorError, merge_dicts
-
-
-class RecurbateIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?recurbate\.com/play\.php\?video=(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://recurbate.com/play.php?video=39161415',
-        'md5': 'dd2b4ec57aa3e3572cb5cf0997fca99f',
-        'info_dict': {
-            'id': '39161415',
-            'ext': 'mp4',
-            'description': 'md5:db48d09e4d93fc715f47fd3d6b7edd51',
-            'title': 'Performer zsnicole33 show on 2022-10-25 20:23, Chaturbate Archive – Recurbate',
-            'age_limit': 18,
-        },
-        'skip': 'Website require membership.',
-    }]
-
-    def _real_extract(self, url):
-        SUBSCRIPTION_MISSING_MESSAGE = 'This video is only available for registered users; Set your authenticated browser user agent via the --user-agent parameter.'
-        video_id = self._match_id(url)
-        try:
-            webpage = self._download_webpage(url, video_id)
-        except ExtractorError as e:
-            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
-                self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
-            raise
-        token = self._html_search_regex(r'data-token="([^"]+)"', webpage, 'token')
-        video_url = f'https://recurbate.com/api/get.php?video={video_id}&token={token}'
-
-        video_webpage = self._download_webpage(video_url, video_id)
-        if video_webpage == 'shall_subscribe':
-            self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
-        entries = self._parse_html5_media_entries(video_url, video_webpage, video_id)
-        return merge_dicts({
-            'id': video_id,
-            'title': self._html_extract_title(webpage, 'title'),
-            'description': self._og_search_description(webpage),
-            'age_limit': self._rta_search(webpage),
-        }, entries[0])
diff --git a/yt_dlp/extractor/rice.py b/yt_dlp/extractor/rice.py
deleted file mode 100644
index 3dd4d31de..000000000
--- a/yt_dlp/extractor/rice.py
+++ /dev/null
@@ -1,112 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_parse_qs
-from ..utils import (
-    xpath_text,
-    xpath_element,
-    int_or_none,
-    parse_iso8601,
-    ExtractorError,
-)
-
-
-class RICEIE(InfoExtractor):
-    _VALID_URL = r'https?://mediahub\.rice\.edu/app/[Pp]ortal/video\.aspx\?(?P<query>.+)'
-    _TEST = {
-        'url': 'https://mediahub.rice.edu/app/Portal/video.aspx?PortalID=25ffd62c-3d01-4b29-8c70-7c94270efb3e&DestinationID=66bc9434-03bd-4725-b47e-c659d8d809db&ContentID=YEWIvbhb40aqdjMD1ALSqw',
-        'md5': '9b83b4a2eead4912dc3b7fac7c449b6a',
-        'info_dict': {
-            'id': 'YEWIvbhb40aqdjMD1ALSqw',
-            'ext': 'mp4',
-            'title': 'Active Learning in Archeology',
-            'upload_date': '20140616',
-            'timestamp': 1402926346,
-        }
-    }
-    _NS = 'http://schemas.datacontract.org/2004/07/ensembleVideo.Data.Service.Contracts.Models.Player.Config'
-
-    def _real_extract(self, url):
-        qs = compat_parse_qs(self._match_valid_url(url).group('query'))
-        if not qs.get('PortalID') or not qs.get('DestinationID') or not qs.get('ContentID'):
-            raise ExtractorError('Invalid URL', expected=True)
-
-        portal_id = qs['PortalID'][0]
-        playlist_id = qs['DestinationID'][0]
-        content_id = qs['ContentID'][0]
-
-        content_data = self._download_xml('https://mediahub.rice.edu/api/portal/GetContentTitle', content_id, query={
-            'portalId': portal_id,
-            'playlistId': playlist_id,
-            'contentId': content_id
-        })
-        metadata = xpath_element(content_data, './/metaData', fatal=True)
-        title = xpath_text(metadata, 'primaryTitle', fatal=True)
-        encodings = xpath_element(content_data, './/encodings', fatal=True)
-        player_data = self._download_xml('https://mediahub.rice.edu/api/player/GetPlayerConfig', content_id, query={
-            'temporaryLinkId': xpath_text(encodings, 'temporaryLinkId', fatal=True),
-            'contentId': content_id,
-        })
-
-        common_fmt = {}
-        dimensions = xpath_text(encodings, 'dimensions')
-        if dimensions:
-            wh = dimensions.split('x')
-            if len(wh) == 2:
-                common_fmt.update({
-                    'width': int_or_none(wh[0]),
-                    'height': int_or_none(wh[1]),
-                })
-
-        formats = []
-        rtsp_path = xpath_text(player_data, self._xpath_ns('RtspPath', self._NS))
-        if rtsp_path:
-            fmt = {
-                'url': rtsp_path,
-                'format_id': 'rtsp',
-            }
-            fmt.update(common_fmt)
-            formats.append(fmt)
-        for source in player_data.findall(self._xpath_ns('.//Source', self._NS)):
-            video_url = xpath_text(source, self._xpath_ns('File', self._NS))
-            if not video_url:
-                continue
-            if '.m3u8' in video_url:
-                formats.extend(self._extract_m3u8_formats(video_url, content_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-            else:
-                fmt = {
-                    'url': video_url,
-                    'format_id': video_url.split(':')[0],
-                }
-                fmt.update(common_fmt)
-                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', video_url)
-                if rtmp:
-                    fmt.update({
-                        'url': rtmp.group('url'),
-                        'play_path': rtmp.group('playpath'),
-                        'app': rtmp.group('app'),
-                        'ext': 'flv',
-                    })
-                formats.append(fmt)
-
-        thumbnails = []
-        for content_asset in content_data.findall('.//contentAssets'):
-            asset_type = xpath_text(content_asset, 'type')
-            if asset_type == 'image':
-                image_url = xpath_text(content_asset, 'httpPath')
-                if not image_url:
-                    continue
-                thumbnails.append({
-                    'id': xpath_text(content_asset, 'ID'),
-                    'url': image_url,
-                })
-
-        return {
-            'id': content_id,
-            'title': title,
-            'description': xpath_text(metadata, 'abstract'),
-            'duration': int_or_none(xpath_text(metadata, 'duration')),
-            'timestamp': parse_iso8601(xpath_text(metadata, 'dateUpdated')),
-            'thumbnails': thumbnails,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/rtl2.py b/yt_dlp/extractor/rtl2.py
index 056cf87d2..07e1aa3ce 100644
--- a/yt_dlp/extractor/rtl2.py
+++ b/yt_dlp/extractor/rtl2.py
@@ -1,16 +1,7 @@
 import re
 
 from .common import InfoExtractor
-from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
-from ..compat import (
-    compat_b64decode,
-    compat_str,
-)
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    strip_or_none,
-)
+from ..utils import int_or_none
 
 
 class RTL2IE(InfoExtractor):
@@ -102,92 +93,3 @@ def _real_extract(self, url):
             'duration': int_or_none(video_info.get('duration')),
             'formats': formats,
         }
-
-
-class RTL2YouBaseIE(InfoExtractor):
-    _BACKWERK_BASE_URL = 'https://p-you-backwerk.rtl2apps.de/'
-
-
-class RTL2YouIE(RTL2YouBaseIE):
-    IE_NAME = 'rtl2:you'
-    _VALID_URL = r'http?://you\.rtl2\.de/(?:video/\d+/|youplayer/index\.html\?.*?\bvid=)(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://you.rtl2.de/video/3002/15740/MJUNIK%20%E2%80%93%20Home%20of%20YOU/307-hirn-wo-bist-du',
-        'info_dict': {
-            'id': '15740',
-            'ext': 'mp4',
-            'title': 'MJUNIK – Home of YOU - #307 Hirn, wo bist du?!',
-            'description': 'md5:ddaa95c61b372b12b66e115b2772fe01',
-            'age_limit': 12,
-        },
-    }, {
-        'url': 'http://you.rtl2.de/youplayer/index.html?vid=15712',
-        'only_matching': True,
-    }]
-    _AES_KEY = b'\xe9W\xe4.<*\xb8\x1a\xd2\xb6\x92\xf3C\xd3\xefL\x1b\x03*\xbbbH\xc0\x03\xffo\xc2\xf2(\xaa\xaa!'
-    _GEO_COUNTRIES = ['DE']
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        stream_data = self._download_json(
-            self._BACKWERK_BASE_URL + 'stream/video/' + video_id, video_id)
-
-        data, iv = compat_b64decode(stream_data['streamUrl']).decode().split(':')
-        stream_url = unpad_pkcs7(aes_cbc_decrypt_bytes(
-            compat_b64decode(data), self._AES_KEY, compat_b64decode(iv)))
-        if b'rtl2_you_video_not_found' in stream_url:
-            raise ExtractorError('video not found', expected=True)
-
-        formats = self._extract_m3u8_formats(stream_url.decode(), video_id, 'mp4', 'm3u8_native')
-
-        video_data = self._download_json(
-            self._BACKWERK_BASE_URL + 'video/' + video_id, video_id)
-
-        series = video_data.get('formatTitle')
-        title = episode = video_data.get('title') or series
-        if series and series != title:
-            title = '%s - %s' % (series, title)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'description': strip_or_none(video_data.get('description')),
-            'thumbnail': video_data.get('image'),
-            'duration': int_or_none(stream_data.get('duration') or video_data.get('duration'), 1000),
-            'series': series,
-            'episode': episode,
-            'age_limit': int_or_none(video_data.get('minimumAge')),
-        }
-
-
-class RTL2YouSeriesIE(RTL2YouBaseIE):
-    IE_NAME = 'rtl2:you:series'
-    _VALID_URL = r'http?://you\.rtl2\.de/videos/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://you.rtl2.de/videos/115/dragon-ball',
-        'info_dict': {
-            'id': '115',
-        },
-        'playlist_mincount': 5,
-    }
-
-    def _real_extract(self, url):
-        series_id = self._match_id(url)
-        stream_data = self._download_json(
-            self._BACKWERK_BASE_URL + 'videos',
-            series_id, query={
-                'formatId': series_id,
-                'limit': 1000000000,
-            })
-
-        entries = []
-        for video in stream_data.get('videos', []):
-            video_id = compat_str(video['videoId'])
-            if not video_id:
-                continue
-            entries.append(self.url_result(
-                'http://you.rtl2.de/video/%s/%s' % (series_id, video_id),
-                'RTL2You', video_id))
-        return self.playlist_result(entries, series_id)
diff --git a/yt_dlp/extractor/rtvnh.py b/yt_dlp/extractor/rtvnh.py
deleted file mode 100644
index 7c6174494..000000000
--- a/yt_dlp/extractor/rtvnh.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from .common import InfoExtractor
-from ..utils import ExtractorError
-
-
-class RTVNHIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.rtvnh.nl/video/131946',
-        'md5': 'cdbec9f44550763c8afc96050fa747dc',
-        'info_dict': {
-            'id': '131946',
-            'ext': 'mp4',
-            'title': 'Grote zoektocht in zee bij Zandvoort naar vermiste vrouw',
-            'thumbnail': r're:^https?:.*\.jpg$'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        meta = self._parse_json(self._download_webpage(
-            'http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
-
-        status = meta.get('status')
-        if status != 200:
-            raise ExtractorError(
-                '%s returned error code %d' % (self.IE_NAME, status), expected=True)
-
-        formats = []
-        rtmp_formats = self._extract_smil_formats(
-            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
-        formats.extend(rtmp_formats)
-
-        for rtmp_format in rtmp_formats:
-            rtmp_url = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
-            rtsp_format = rtmp_format.copy()
-            del rtsp_format['play_path']
-            del rtsp_format['ext']
-            rtsp_format.update({
-                'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
-                'url': rtmp_url.replace('rtmp://', 'rtsp://'),
-                'protocol': 'rtsp',
-            })
-            formats.append(rtsp_format)
-            http_base_url = rtmp_url.replace('rtmp://', 'http://')
-            formats.extend(self._extract_m3u8_formats(
-                http_base_url + '/playlist.m3u8', video_id, 'mp4',
-                'm3u8_native', m3u8_id='hls', fatal=False))
-            formats.extend(self._extract_f4m_formats(
-                http_base_url + '/manifest.f4m',
-                video_id, f4m_id='hds', fatal=False))
-
-        return {
-            'id': video_id,
-            'title': meta['title'].strip(),
-            'thumbnail': meta.get('image'),
-            'formats': formats
-        }
diff --git a/yt_dlp/extractor/ruhd.py b/yt_dlp/extractor/ruhd.py
deleted file mode 100644
index abaa3f9ea..000000000
--- a/yt_dlp/extractor/ruhd.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from .common import InfoExtractor
-
-
-class RUHDIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ruhd\.ru/play\.php\?vid=(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.ruhd.ru/play.php?vid=207',
-        'md5': 'd1a9ec4edf8598e3fbd92bb16072ba83',
-        'info_dict': {
-            'id': '207',
-            'ext': 'divx',
-            'title': 'КОТ бааааам',
-            'description': 'классный кот)',
-            'thumbnail': r're:^http://.*\.jpg$',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        video_url = self._html_search_regex(
-            r'<param name="src" value="([^"]+)"', webpage, 'video url')
-        title = self._html_search_regex(
-            r'<title>([^<]+)&nbsp;&nbsp; RUHD\.ru - Видео Высокого качества №1 в России!</title>',
-            webpage, 'title')
-        description = self._html_search_regex(
-            r'(?s)<div id="longdesc">(.+?)<span id="showlink">',
-            webpage, 'description', fatal=False)
-        thumbnail = self._html_search_regex(
-            r'<param name="previewImage" value="([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
-        if thumbnail:
-            thumbnail = 'http://www.ruhd.ru' + thumbnail
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-        }
diff --git a/yt_dlp/extractor/scte.py b/yt_dlp/extractor/scte.py
index d839ffcde..9c2ca8c51 100644
--- a/yt_dlp/extractor/scte.py
+++ b/yt_dlp/extractor/scte.py
@@ -46,6 +46,7 @@ def is_logged(webpage):
 
 
 class SCTEIE(SCTEBaseIE):
+    _WORKING = False
     _VALID_URL = r'https?://learning\.scte\.org/mod/scorm/view\.php?.*?\bid=(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://learning.scte.org/mod/scorm/view.php?id=31484',
@@ -93,6 +94,7 @@ def _real_extract(self, url):
 
 
 class SCTECourseIE(SCTEBaseIE):
+    _WORKING = False
     _VALID_URL = r'https?://learning\.scte\.org/(?:mod/sub)?course/view\.php?.*?\bid=(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://learning.scte.org/mod/subcourse/view.php?id=31491',
diff --git a/yt_dlp/extractor/shared.py b/yt_dlp/extractor/shared.py
deleted file mode 100644
index 9a237b320..000000000
--- a/yt_dlp/extractor/shared.py
+++ /dev/null
@@ -1,138 +0,0 @@
-import urllib.parse
-
-from .common import InfoExtractor
-from ..compat import compat_b64decode
-from ..utils import (
-    KNOWN_EXTENSIONS,
-    ExtractorError,
-    determine_ext,
-    int_or_none,
-    js_to_json,
-    parse_filesize,
-    rot47,
-    url_or_none,
-    urlencode_postdata,
-)
-
-
-class SharedBaseIE(InfoExtractor):
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage, urlh = self._download_webpage_handle(url, video_id)
-
-        if self._FILE_NOT_FOUND in webpage:
-            raise ExtractorError(
-                'Video %s does not exist' % video_id, expected=True)
-
-        video_url = self._extract_video_url(webpage, video_id, url)
-
-        title = self._extract_title(webpage)
-        filesize = int_or_none(self._extract_filesize(webpage))
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'ext': 'mp4',
-            'filesize': filesize,
-            'title': title,
-        }
-
-    def _extract_title(self, webpage):
-        return compat_b64decode(self._html_search_meta(
-            'full:title', webpage, 'title')).decode('utf-8')
-
-    def _extract_filesize(self, webpage):
-        return self._html_search_meta(
-            'full:size', webpage, 'file size', fatal=False)
-
-
-class SharedIE(SharedBaseIE):
-    IE_DESC = 'shared.sx'
-    _VALID_URL = r'https?://shared\.sx/(?P<id>[\da-z]{10})'
-    _FILE_NOT_FOUND = '>File does not exist<'
-
-    _TEST = {
-        'url': 'http://shared.sx/0060718775',
-        'md5': '106fefed92a8a2adb8c98e6a0652f49b',
-        'info_dict': {
-            'id': '0060718775',
-            'ext': 'mp4',
-            'title': 'Bmp4',
-            'filesize': 1720110,
-        },
-    }
-
-    def _extract_video_url(self, webpage, video_id, url):
-        download_form = self._hidden_inputs(webpage)
-
-        video_page = self._download_webpage(
-            url, video_id, 'Downloading video page',
-            data=urlencode_postdata(download_form),
-            headers={
-                'Content-Type': 'application/x-www-form-urlencoded',
-                'Referer': url,
-            })
-
-        video_url = self._html_search_regex(
-            r'data-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
-            video_page, 'video URL', group='url')
-
-        return video_url
-
-
-class VivoIE(SharedBaseIE):
-    IE_DESC = 'vivo.sx'
-    _VALID_URL = r'https?://vivo\.s[xt]/(?P<id>[\da-z]{10})'
-    _FILE_NOT_FOUND = '>The file you have requested does not exists or has been removed'
-
-    _TESTS = [{
-        'url': 'http://vivo.sx/d7ddda0e78',
-        'md5': '15b3af41be0b4fe01f4df075c2678b2c',
-        'info_dict': {
-            'id': 'd7ddda0e78',
-            'ext': 'mp4',
-            'title': 'Chicken',
-            'filesize': 515659,
-        },
-    }, {
-        'url': 'http://vivo.st/d7ddda0e78',
-        'only_matching': True,
-    }]
-
-    def _extract_title(self, webpage):
-        title = self._html_search_regex(
-            r'data-name\s*=\s*(["\'])(?P<title>(?:(?!\1).)+)\1', webpage,
-            'title', default=None, group='title')
-        if title:
-            ext = determine_ext(title)
-            if ext.lower() in KNOWN_EXTENSIONS:
-                title = title.rpartition('.' + ext)[0]
-            return title
-        return self._og_search_title(webpage)
-
-    def _extract_filesize(self, webpage):
-        return parse_filesize(self._search_regex(
-            r'data-type=["\']video["\'][^>]*>Watch.*?<strong>\s*\((.+?)\)',
-            webpage, 'filesize', fatal=False))
-
-    def _extract_video_url(self, webpage, video_id, url):
-        def decode_url_old(encoded_url):
-            return compat_b64decode(encoded_url).decode('utf-8')
-
-        stream_url = self._search_regex(
-            r'data-stream\s*=\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
-            'stream url', default=None, group='url')
-        if stream_url:
-            stream_url = url_or_none(decode_url_old(stream_url))
-        if stream_url:
-            return stream_url
-
-        def decode_url(encoded_url):
-            return rot47(urllib.parse.unquote_plus(encoded_url))
-
-        return decode_url(self._parse_json(
-            self._search_regex(
-                r'(?s)InitializeStream\s*\(\s*({.+?})\s*\)\s*;', webpage,
-                'stream'),
-            video_id, transform_source=js_to_json)['source'])
diff --git a/yt_dlp/extractor/sky.py b/yt_dlp/extractor/sky.py
index 0a8b6cc76..574ac219c 100644
--- a/yt_dlp/extractor/sky.py
+++ b/yt_dlp/extractor/sky.py
@@ -3,9 +3,7 @@
 from .common import InfoExtractor
 from ..utils import (
     extract_attributes,
-    smuggle_url,
     strip_or_none,
-    urljoin,
 )
 
 
@@ -13,29 +11,10 @@ class SkyBaseIE(InfoExtractor):
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
     _SDC_EL_REGEX = r'(?s)(<div[^>]+data-(?:component-name|fn)="sdc-(?:articl|sit)e-video"[^>]*>)'
 
-    def _process_ooyala_element(self, webpage, sdc_el, url):
+    def _process_video_element(self, webpage, sdc_el, url):
         sdc = extract_attributes(sdc_el)
         provider = sdc.get('data-provider')
-        if provider == 'ooyala':
-            video_id = sdc['data-sdc-video-id']
-            video_url = 'ooyala:%s' % video_id
-            ie_key = 'Ooyala'
-            ooyala_el = self._search_regex(
-                r'(<div[^>]+class="[^"]*\bsdc-article-video__media-ooyala\b[^"]*"[^>]+data-video-id="%s"[^>]*>)' % video_id,
-                webpage, 'video data', fatal=False)
-            if ooyala_el:
-                ooyala_attrs = extract_attributes(ooyala_el) or {}
-                if ooyala_attrs.get('data-token-required') == 'true':
-                    token_fetch_url = (self._parse_json(ooyala_attrs.get(
-                        'data-token-fetch-options', '{}'),
-                        video_id, fatal=False) or {}).get('url')
-                    if token_fetch_url:
-                        embed_token = self._download_json(urljoin(
-                            url, token_fetch_url), video_id, fatal=False)
-                        if embed_token:
-                            video_url = smuggle_url(
-                                video_url, {'embed_token': embed_token})
-        elif provider == 'brightcove':
+        if provider == 'brightcove':
             video_id = sdc['data-video-id']
             account_id = sdc.get('data-account-id') or '6058004172001'
             player_id = sdc.get('data-player-id') or 'RC9PQUaJ6'
@@ -52,7 +31,7 @@ def _process_ooyala_element(self, webpage, sdc_el, url):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        info = self._process_ooyala_element(webpage, self._search_regex(
+        info = self._process_video_element(webpage, self._search_regex(
             self._SDC_EL_REGEX, webpage, 'sdc element'), url)
         info.update({
             'title': self._og_search_title(webpage),
@@ -73,7 +52,7 @@ class SkySportsIE(SkyBaseIE):
             'title': 'Bale: It\'s our time to shine',
             'description': 'md5:e88bda94ae15f7720c5cb467e777bb6d',
         },
-        'add_ie': ['Ooyala'],
+        'add_ie': ['BrightcoveNew'],
     }, {
         'url': 'https://www.skysports.com/watch/video/sports/f1/12160544/abu-dhabi-gp-the-notebook',
         'only_matching': True,
@@ -122,7 +101,7 @@ def _real_extract(self, url):
         article_id = self._match_id(url)
         webpage = self._download_webpage(url, article_id)
 
-        entries = [self._process_ooyala_element(webpage, sdc_el, url)
+        entries = [self._process_video_element(webpage, sdc_el, url)
                    for sdc_el in re.findall(self._SDC_EL_REGEX, webpage)]
 
         return self.playlist_result(
@@ -149,7 +128,7 @@ def _real_extract(self, url):
 
         entries = []
         for sdc_el in re.findall(self._SDC_EL_REGEX, webpage):
-            entries.append(self._process_ooyala_element(webpage, sdc_el, url))
+            entries.append(self._process_video_element(webpage, sdc_el, url))
 
         return self.playlist_result(
             entries, article_id, self._og_search_title(webpage),
diff --git a/yt_dlp/extractor/spankwire.py b/yt_dlp/extractor/spankwire.py
deleted file mode 100644
index 334b29773..000000000
--- a/yt_dlp/extractor/spankwire.py
+++ /dev/null
@@ -1,174 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    int_or_none,
-    merge_dicts,
-    str_or_none,
-    str_to_int,
-    url_or_none,
-)
-
-
-class SpankwireIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?spankwire\.com/
-                        (?:
-                            [^/]+/video|
-                            EmbedPlayer\.aspx/?\?.*?\bArticleId=
-                        )
-                        (?P<id>\d+)
-                    '''
-    _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?spankwire\.com/EmbedPlayer\.aspx/?\?.*?\bArticleId=\d+)']
-    _TESTS = [{
-        # download URL pattern: */<height>P_<tbr>K_<video_id>.mp4
-        'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
-        'md5': '5aa0e4feef20aad82cbcae3aed7ab7cd',
-        'info_dict': {
-            'id': '103545',
-            'ext': 'mp4',
-            'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
-            'description': 'Crazy Bitch X rated music video.',
-            'duration': 222,
-            'uploader': 'oreusz',
-            'uploader_id': '124697',
-            'timestamp': 1178587885,
-            'upload_date': '20070508',
-            'average_rating': float,
-            'view_count': int,
-            'comment_count': int,
-            'age_limit': 18,
-            'categories': list,
-            'tags': list,
-        },
-    }, {
-        # download URL pattern: */mp4_<format_id>_<video_id>.mp4
-        'url': 'http://www.spankwire.com/Titcums-Compiloation-I/video1921551/',
-        'md5': '09b3c20833308b736ae8902db2f8d7e6',
-        'info_dict': {
-            'id': '1921551',
-            'ext': 'mp4',
-            'title': 'Titcums Compiloation I',
-            'description': 'cum on tits',
-            'uploader': 'dannyh78999',
-            'uploader_id': '3056053',
-            'upload_date': '20150822',
-            'age_limit': 18,
-        },
-        'params': {
-            'proxy': '127.0.0.1:8118'
-        },
-        'skip': 'removed',
-    }, {
-        'url': 'https://www.spankwire.com/EmbedPlayer.aspx/?ArticleId=156156&autostart=true',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        video = self._download_json(
-            'https://www.spankwire.com/api/video/%s.json' % video_id, video_id)
-
-        title = video['title']
-
-        formats = []
-        videos = video.get('videos')
-        if isinstance(videos, dict):
-            for format_id, format_url in videos.items():
-                video_url = url_or_none(format_url)
-                if not format_url:
-                    continue
-                height = int_or_none(self._search_regex(
-                    r'(\d+)[pP]', format_id, 'height', default=None))
-                m = re.search(
-                    r'/(?P<height>\d+)[pP]_(?P<tbr>\d+)[kK]', video_url)
-                if m:
-                    tbr = int(m.group('tbr'))
-                    height = height or int(m.group('height'))
-                else:
-                    tbr = None
-                formats.append({
-                    'url': video_url,
-                    'format_id': '%dp' % height if height else format_id,
-                    'height': height,
-                    'tbr': tbr,
-                })
-        m3u8_url = url_or_none(video.get('HLS'))
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                m3u8_id='hls', fatal=False))
-
-        view_count = str_to_int(video.get('viewed'))
-
-        thumbnails = []
-        for preference, t in enumerate(('', '2x'), start=0):
-            thumbnail_url = url_or_none(video.get('poster%s' % t))
-            if not thumbnail_url:
-                continue
-            thumbnails.append({
-                'url': thumbnail_url,
-                'preference': preference,
-            })
-
-        def extract_names(key):
-            entries_list = video.get(key)
-            if not isinstance(entries_list, list):
-                return
-            entries = []
-            for entry in entries_list:
-                name = str_or_none(entry.get('name'))
-                if name:
-                    entries.append(name)
-            return entries
-
-        categories = extract_names('categories')
-        tags = extract_names('tags')
-
-        uploader = None
-        info = {}
-
-        webpage = self._download_webpage(
-            'https://www.spankwire.com/_/video%s/' % video_id, video_id,
-            fatal=False)
-        if webpage:
-            info = self._search_json_ld(webpage, video_id, default={})
-            thumbnail_url = None
-            if 'thumbnail' in info:
-                thumbnail_url = url_or_none(info['thumbnail'])
-                del info['thumbnail']
-            if not thumbnail_url:
-                thumbnail_url = self._og_search_thumbnail(webpage)
-            if thumbnail_url:
-                thumbnails.append({
-                    'url': thumbnail_url,
-                    'preference': 10,
-                })
-            uploader = self._html_search_regex(
-                r'(?s)by\s*<a[^>]+\bclass=["\']uploaded__by[^>]*>(.+?)</a>',
-                webpage, 'uploader', fatal=False)
-            if not view_count:
-                view_count = str_to_int(self._search_regex(
-                    r'data-views=["\']([\d,.]+)', webpage, 'view count',
-                    fatal=False))
-
-        return merge_dicts({
-            'id': video_id,
-            'title': title,
-            'description': video.get('description'),
-            'duration': int_or_none(video.get('duration')),
-            'thumbnails': thumbnails,
-            'uploader': uploader,
-            'uploader_id': str_or_none(video.get('userId')),
-            'timestamp': int_or_none(video.get('time_approved_on')),
-            'average_rating': float_or_none(video.get('rating')),
-            'view_count': view_count,
-            'comment_count': int_or_none(video.get('comments')),
-            'age_limit': 18,
-            'categories': categories,
-            'tags': tags,
-            'formats': formats,
-        }, info)
diff --git a/yt_dlp/extractor/srmediathek.py b/yt_dlp/extractor/srmediathek.py
index 3cc39870f..f0b3b585f 100644
--- a/yt_dlp/extractor/srmediathek.py
+++ b/yt_dlp/extractor/srmediathek.py
@@ -6,6 +6,7 @@
 
 
 class SRMediathekIE(ARDMediathekBaseIE):
+    _WORKING = False
     IE_NAME = 'sr:mediathek'
     IE_DESC = 'Saarländischer Rundfunk'
     _VALID_URL = r'https?://sr-mediathek(?:\.sr-online)?\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
diff --git a/yt_dlp/extractor/streamcloud.py b/yt_dlp/extractor/streamcloud.py
deleted file mode 100644
index 728980921..000000000
--- a/yt_dlp/extractor/streamcloud.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    urlencode_postdata,
-)
-
-
-class StreamcloudIE(InfoExtractor):
-    IE_NAME = 'streamcloud.eu'
-    _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)(?:/(?P<fname>[^#?]*)\.html)?'
-
-    _TESTS = [{
-        'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
-        'md5': '6bea4c7fa5daaacc2a946b7146286686',
-        'info_dict': {
-            'id': 'skp9j99s4bpz',
-            'ext': 'mp4',
-            'title': 'youtube-dl test video  \'/\\ ä ↭',
-        },
-        'skip': 'Only available from the EU'
-    }, {
-        'url': 'http://streamcloud.eu/ua8cmfh1nbe6/NSHIP-148--KUC-NG--H264-.mp4.html',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url = 'http://streamcloud.eu/%s' % video_id
-
-        orig_webpage = self._download_webpage(url, video_id)
-
-        if '>File Not Found<' in orig_webpage:
-            raise ExtractorError(
-                'Video %s does not exist' % video_id, expected=True)
-
-        fields = re.findall(r'''(?x)<input\s+
-            type="(?:hidden|submit)"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', orig_webpage)
-
-        self._sleep(6, video_id)
-
-        webpage = self._download_webpage(
-            url, video_id, data=urlencode_postdata(fields), headers={
-                b'Content-Type': b'application/x-www-form-urlencoded',
-            })
-
-        try:
-            title = self._html_search_regex(
-                r'<h1[^>]*>([^<]+)<', webpage, 'title')
-            video_url = self._search_regex(
-                r'file:\s*"([^"]+)"', webpage, 'video URL')
-        except ExtractorError:
-            message = self._html_search_regex(
-                r'(?s)<div[^>]+class=(["\']).*?msgboxinfo.*?\1[^>]*>(?P<message>.+?)</div>',
-                webpage, 'message', default=None, group='message')
-            if message:
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
-            raise
-        thumbnail = self._search_regex(
-            r'image:\s*"([^"]+)"', webpage, 'thumbnail URL', fatal=False)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'url': video_url,
-            'thumbnail': thumbnail,
-            'http_headers': {
-                'Referer': url,
-            },
-        }
diff --git a/yt_dlp/extractor/swrmediathek.py b/yt_dlp/extractor/swrmediathek.py
deleted file mode 100644
index 38bdfced7..000000000
--- a/yt_dlp/extractor/swrmediathek.py
+++ /dev/null
@@ -1,111 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    parse_duration,
-    int_or_none,
-    determine_protocol,
-)
-
-
-class SWRMediathekIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?swrmediathek\.de/(?:content/)?player\.htm\?show=(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
-
-    _TESTS = [{
-        'url': 'http://swrmediathek.de/player.htm?show=849790d0-dab8-11e3-a953-0026b975f2e6',
-        'md5': '8c5f6f0172753368547ca8413a7768ac',
-        'info_dict': {
-            'id': '849790d0-dab8-11e3-a953-0026b975f2e6',
-            'ext': 'mp4',
-            'title': 'SWR odysso',
-            'description': 'md5:2012e31baad36162e97ce9eb3f157b8a',
-            'thumbnail': r're:^http:.*\.jpg$',
-            'duration': 2602,
-            'upload_date': '20140515',
-            'uploader': 'SWR Fernsehen',
-            'uploader_id': '990030',
-        },
-    }, {
-        'url': 'http://swrmediathek.de/player.htm?show=0e1a8510-ddf2-11e3-9be3-0026b975f2e6',
-        'md5': 'b10ab854f912eecc5a6b55cd6fc1f545',
-        'info_dict': {
-            'id': '0e1a8510-ddf2-11e3-9be3-0026b975f2e6',
-            'ext': 'mp4',
-            'title': 'Nachtcafé - Alltagsdroge Alkohol - zwischen Sektempfang und Komasaufen',
-            'description': 'md5:e0a3adc17e47db2c23aab9ebc36dbee2',
-            'thumbnail': r're:http://.*\.jpg',
-            'duration': 5305,
-            'upload_date': '20140516',
-            'uploader': 'SWR Fernsehen',
-            'uploader_id': '990030',
-        },
-        'skip': 'redirect to http://swrmediathek.de/index.htm?hinweis=swrlink',
-    }, {
-        'url': 'http://swrmediathek.de/player.htm?show=bba23e10-cb93-11e3-bf7f-0026b975f2e6',
-        'md5': '4382e4ef2c9d7ce6852535fa867a0dd3',
-        'info_dict': {
-            'id': 'bba23e10-cb93-11e3-bf7f-0026b975f2e6',
-            'ext': 'mp3',
-            'title': 'Saša Stanišic: Vor dem Fest',
-            'description': 'md5:5b792387dc3fbb171eb709060654e8c9',
-            'thumbnail': r're:http://.*\.jpg',
-            'duration': 3366,
-            'upload_date': '20140520',
-            'uploader': 'SWR 2',
-            'uploader_id': '284670',
-        },
-        'skip': 'redirect to http://swrmediathek.de/index.htm?hinweis=swrlink',
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        video = self._download_json(
-            'http://swrmediathek.de/AjaxEntry?ekey=%s' % video_id,
-            video_id, 'Downloading video JSON')
-
-        attr = video['attr']
-        title = attr['entry_title']
-        media_type = attr.get('entry_etype')
-
-        formats = []
-        for entry in video.get('sub', []):
-            if entry.get('name') != 'entry_media':
-                continue
-
-            entry_attr = entry.get('attr', {})
-            f_url = entry_attr.get('val2')
-            if not f_url:
-                continue
-            codec = entry_attr.get('val0')
-            if codec == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    f_url, video_id, 'mp4', 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-            elif codec == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    f_url + '?hdcore=3.7.0', video_id,
-                    f4m_id='hds', fatal=False))
-            else:
-                formats.append({
-                    'format_id': determine_protocol({'url': f_url}),
-                    'url': f_url,
-                    'quality': int_or_none(entry_attr.get('val1')),
-                    'vcodec': codec if media_type == 'Video' else 'none',
-                    'acodec': codec if media_type == 'Audio' else None,
-                })
-
-        upload_date = None
-        entry_pdatet = attr.get('entry_pdatet')
-        if entry_pdatet:
-            upload_date = entry_pdatet[:-4]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': attr.get('entry_descl'),
-            'thumbnail': attr.get('entry_image_16_9'),
-            'duration': parse_duration(attr.get('entry_durat')),
-            'upload_date': upload_date,
-            'uploader': attr.get('channel_title'),
-            'uploader_id': attr.get('channel_idkey'),
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/techtalks.py b/yt_dlp/extractor/techtalks.py
deleted file mode 100644
index d37de360b..000000000
--- a/yt_dlp/extractor/techtalks.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    get_element_by_attribute,
-    clean_html,
-)
-
-
-class TechTalksIE(InfoExtractor):
-    _VALID_URL = r'https?://techtalks\.tv/talks/(?:[^/]+/)?(?P<id>\d+)'
-
-    _TESTS = [{
-        'url': 'http://techtalks.tv/talks/learning-topic-models-going-beyond-svd/57758/',
-        'info_dict': {
-            'id': '57758',
-            'title': 'Learning Topic Models --- Going beyond SVD',
-        },
-        'playlist': [
-            {
-                'info_dict': {
-                    'id': '57758',
-                    'ext': 'flv',
-                    'title': 'Learning Topic Models --- Going beyond SVD',
-                },
-            },
-            {
-                'info_dict': {
-                    'id': '57758-slides',
-                    'ext': 'flv',
-                    'title': 'Learning Topic Models --- Going beyond SVD',
-                },
-            },
-        ],
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://techtalks.tv/talks/57758',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        talk_id = mobj.group('id')
-        webpage = self._download_webpage(url, talk_id)
-        rtmp_url = self._search_regex(
-            r'netConnectionUrl: \'(.*?)\'', webpage, 'rtmp url')
-        play_path = self._search_regex(
-            r'href=\'(.*?)\' [^>]*id="flowplayer_presenter"',
-            webpage, 'presenter play path')
-        title = clean_html(get_element_by_attribute('class', 'title', webpage))
-        video_info = {
-            'id': talk_id,
-            'title': title,
-            'url': rtmp_url,
-            'play_path': play_path,
-            'ext': 'flv',
-        }
-        m_slides = re.search(r'<a class="slides" href=\'(.*?)\'', webpage)
-        if m_slides is None:
-            return video_info
-        else:
-            return {
-                '_type': 'playlist',
-                'id': talk_id,
-                'title': title,
-                'entries': [
-                    video_info,
-                    # The slides video
-                    {
-                        'id': talk_id + '-slides',
-                        'title': title,
-                        'url': rtmp_url,
-                        'play_path': m_slides.group(1),
-                        'ext': 'flv',
-                    },
-                ],
-            }
diff --git a/yt_dlp/extractor/telecinco.py b/yt_dlp/extractor/telecinco.py
index 20bb82420..a3f0c7cda 100644
--- a/yt_dlp/extractor/telecinco.py
+++ b/yt_dlp/extractor/telecinco.py
@@ -77,7 +77,6 @@ class TelecincoIE(InfoExtractor):
         'url': 'http://www.telecinco.es/espanasinirmaslejos/Espana-gran-destino-turistico_2_1240605043.html',
         'only_matching': True,
     }, {
-        # ooyala video
         'url': 'http://www.cuatro.com/chesterinlove/a-carta/chester-chester_in_love-chester_edu_2_2331030022.html',
         'only_matching': True,
     }]
diff --git a/yt_dlp/extractor/tinypic.py b/yt_dlp/extractor/tinypic.py
deleted file mode 100644
index 216208cbd..000000000
--- a/yt_dlp/extractor/tinypic.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import ExtractorError
-
-
-class TinyPicIE(InfoExtractor):
-    IE_NAME = 'tinypic'
-    IE_DESC = 'tinypic.com videos'
-    _VALID_URL = r'https?://(?:.+?\.)?tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
-
-    _TESTS = [
-        {
-            'url': 'http://tinypic.com/player.php?v=6xw7tc%3E&s=5#.UtqZmbRFCM8',
-            'md5': '609b74432465364e72727ebc6203f044',
-            'info_dict': {
-                'id': '6xw7tc',
-                'ext': 'flv',
-                'title': 'shadow phenomenon weird',
-            },
-        },
-        {
-            'url': 'http://de.tinypic.com/player.php?v=dy90yh&s=8',
-            'only_matching': True,
-        }
-    ]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, video_id, 'Downloading page')
-
-        mobj = re.search(r'(?m)fo\.addVariable\("file",\s"(?P<fileid>[\da-z]+)"\);\n'
-                         r'\s+fo\.addVariable\("s",\s"(?P<serverid>\d+)"\);', webpage)
-        if mobj is None:
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
-        file_id = mobj.group('fileid')
-        server_id = mobj.group('serverid')
-
-        KEYWORDS_SUFFIX = ', Video, images, photos, videos, myspace, ebay, video hosting, photo hosting'
-        keywords = self._html_search_meta('keywords', webpage, 'title')
-        title = keywords[:-len(KEYWORDS_SUFFIX)] if keywords.endswith(KEYWORDS_SUFFIX) else ''
-
-        video_url = 'http://v%s.tinypic.com/%s.flv' % (server_id, file_id)
-        thumbnail = 'http://v%s.tinypic.com/%s_th.jpg' % (server_id, file_id)
-
-        return {
-            'id': file_id,
-            'url': video_url,
-            'thumbnail': thumbnail,
-            'title': title
-        }
diff --git a/yt_dlp/extractor/tokentube.py b/yt_dlp/extractor/tokentube.py
deleted file mode 100644
index d022e2753..000000000
--- a/yt_dlp/extractor/tokentube.py
+++ /dev/null
@@ -1,153 +0,0 @@
-import functools
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    get_element_by_class,
-    parse_count,
-    remove_end,
-    unified_strdate,
-    js_to_json,
-    OnDemandPagedList,
-)
-
-
-class TokentubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tokentube\.net/(?:view\?[vl]=|[vl]/)(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://tokentube.net/l/3236632011/Praise-A-Thon-Pastori-Chrisin-ja-Pastori-Bennyn-kanssa-27-8-2021',
-        'info_dict': {
-            'id': '3236632011',
-            'ext': 'mp4',
-            'title': 'Praise-A-Thon Pastori Chrisin ja Pastori Bennyn kanssa 27.8.2021',
-            'description': '',
-            'uploader': 'Pastori Chris - Rapsodia.fi',
-            'upload_date': '20210827',
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://tokentube.net/v/3950239124/Linux-Ubuntu-Studio-perus-k%C3%A4ytt%C3%B6',
-        'md5': '0e1f00421f501f5eada9890d38fcfb56',
-        'info_dict': {
-            'id': '3950239124',
-            'ext': 'mp4',
-            'title': 'Linux Ubuntu Studio perus käyttö',
-            'description': 'md5:46077d0daaba1974f2dc381257f9d64c',
-            'uploader': 'jyrilehtonen',
-            'upload_date': '20210825',
-        },
-    }, {
-        'url': 'https://tokentube.net/view?v=3582463289',
-        'info_dict': {
-            'id': '3582463289',
-            'ext': 'mp4',
-            'title': 'Police for Freedom - toiminta aloitetaan Suomessa ❤️??',
-            'description': 'md5:37ebf1cb44264e0bf23ed98b337ee63e',
-            'uploader': 'Voitontie',
-            'upload_date': '20210428',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(r'<h1\s*class=["\']title-text["\']>(.+?)</h1>', webpage, 'title')
-
-        data_json = self._html_search_regex(r'({["\']html5["\'].+?}}}+)', webpage, 'data json')
-        data_json = self._parse_json(js_to_json(data_json), video_id, fatal=False)
-
-        sources = data_json.get('sources') or self._parse_json(
-            self._html_search_regex(r'updateSrc\(([^\)]+)\)', webpage, 'sources'),
-            video_id, transform_source=js_to_json)
-
-        formats = [{
-            'url': format.get('src'),
-            'format_id': format.get('label'),
-            'height': format.get('res'),
-        } for format in sources]
-
-        view_count = parse_count(self._html_search_regex(
-            r'<p\s*class=["\']views_counter["\']>\s*([\d\.,]+)\s*<span>views?</span></p>',
-            webpage, 'view_count', fatal=False))
-
-        like_count = parse_count(self._html_search_regex(
-            r'<div\s*class="sh_button\s*likes_count">\s*(\d+)\s*</div>',
-            webpage, 'like count', fatal=False))
-
-        dislike_count = parse_count(self._html_search_regex(
-            r'<div\s*class="sh_button\s*dislikes_count">\s*(\d+)\s*</div>',
-            webpage, 'dislike count', fatal=False))
-
-        upload_date = unified_strdate(self._html_search_regex(
-            r'<span\s*class="p-date">Published\s*on\s+([^<]+)',
-            webpage, 'upload date', fatal=False))
-
-        uploader = self._html_search_regex(
-            r'<a\s*class="place-left"[^>]+>(.+?)</a>',
-            webpage, 'uploader', fatal=False)
-
-        description = (clean_html(get_element_by_class('p-d-txt', webpage))
-                       or self._html_search_meta(('og:description', 'description', 'twitter:description'), webpage))
-
-        description = remove_end(description, 'Category')
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': title,
-            'view_count': view_count,
-            'like_count': like_count,
-            'dislike_count': dislike_count,
-            'upload_date': upload_date,
-            'description': description,
-            'uploader': uploader,
-        }
-
-
-class TokentubeChannelIE(InfoExtractor):
-    _PAGE_SIZE = 20
-    IE_NAME = 'Tokentube:channel'
-    _VALID_URL = r'https?://(?:www\.)?tokentube\.net/channel/(?P<id>\d+)/[^/]+(?:/videos)?'
-    _TESTS = [{
-        'url': 'https://tokentube.net/channel/3697658904/TokenTube',
-        'info_dict': {
-            'id': '3697658904',
-        },
-        'playlist_mincount': 7,
-    }, {
-        'url': 'https://tokentube.net/channel/3353234420/Linux/videos',
-        'info_dict': {
-            'id': '3353234420',
-        },
-        'playlist_mincount': 20,
-    }, {
-        'url': 'https://tokentube.net/channel/3475834195/Voitontie',
-        'info_dict': {
-            'id': '3475834195',
-        },
-        'playlist_mincount': 150,
-    }]
-
-    def _fetch_page(self, channel_id, page):
-        page += 1
-        videos_info = self._download_webpage(
-            f'https://tokentube.net/videos?p=0&m=1&sort=recent&u={channel_id}&page={page}',
-            channel_id, headers={'X-Requested-With': 'XMLHttpRequest'},
-            note=f'Downloading page {page}', fatal=False)
-        if '</i> Sorry, no results were found.' not in videos_info:
-            for path, media_id in re.findall(
-                    r'<a[^>]+\bhref=["\']([^"\']+/[lv]/(\d+)/\S+)["\'][^>]+>',
-                    videos_info):
-                yield self.url_result(path, ie=TokentubeIE.ie_key(), video_id=media_id)
-
-    def _real_extract(self, url):
-        channel_id = self._match_id(url)
-
-        entries = OnDemandPagedList(functools.partial(
-            self._fetch_page, channel_id), self._PAGE_SIZE)
-
-        return self.playlist_result(entries, channel_id)
diff --git a/yt_dlp/extractor/toypics.py b/yt_dlp/extractor/toypics.py
index bc7336186..aa7ee6c48 100644
--- a/yt_dlp/extractor/toypics.py
+++ b/yt_dlp/extractor/toypics.py
@@ -3,6 +3,7 @@
 
 
 class ToypicsIE(InfoExtractor):
+    _WORKING = False
     IE_DESC = 'Toypics video'
     _VALID_URL = r'https?://videos\.toypics\.net/view/(?P<id>[0-9]+)'
     _TEST = {
@@ -43,6 +44,7 @@ def _real_extract(self, url):
 
 
 class ToypicsUserIE(InfoExtractor):
+    _WORKING = False
     IE_DESC = 'Toypics user profile'
     _VALID_URL = r'https?://videos\.toypics\.net/(?!view)(?P<id>[^/?#&]+)'
     _TEST = {
diff --git a/yt_dlp/extractor/trilulilu.py b/yt_dlp/extractor/trilulilu.py
deleted file mode 100644
index fb97be737..000000000
--- a/yt_dlp/extractor/trilulilu.py
+++ /dev/null
@@ -1,100 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    parse_iso8601,
-)
-
-
-class TriluliluIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?trilulilu\.ro/(?:[^/]+/)?(?P<id>[^/#\?]+)'
-    _TESTS = [{
-        'url': 'http://www.trilulilu.ro/big-buck-bunny-1',
-        'md5': '68da087b676a6196a413549212f60cc6',
-        'info_dict': {
-            'id': 'ae2899e124140b',
-            'ext': 'mp4',
-            'title': 'Big Buck Bunny',
-            'description': ':) pentru copilul din noi',
-            'uploader_id': 'chipy',
-            'upload_date': '20120304',
-            'timestamp': 1330830647,
-            'uploader': 'chipy',
-            'view_count': int,
-            'like_count': int,
-            'comment_count': int,
-        },
-    }, {
-        'url': 'http://www.trilulilu.ro/adena-ft-morreti-inocenta',
-        'md5': '929dfb8729dc71750463af88bbbbf4a4',
-        'info_dict': {
-            'id': 'f299710e3c91c5',
-            'ext': 'mp4',
-            'title': 'Adena ft. Morreti - Inocenta',
-            'description': 'pop music',
-            'uploader_id': 'VEVOmixt',
-            'upload_date': '20151204',
-            'uploader': 'VEVOmixt',
-            'timestamp': 1449187937,
-            'view_count': int,
-            'like_count': int,
-            'comment_count': int,
-        },
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        media_info = self._download_json('http://m.trilulilu.ro/%s?format=json' % display_id, display_id)
-
-        age_limit = 0
-        errors = media_info.get('errors', {})
-        if errors.get('friends'):
-            raise ExtractorError('This video is private.', expected=True)
-        elif errors.get('geoblock'):
-            raise ExtractorError('This video is not available in your country.', expected=True)
-        elif errors.get('xxx_unlogged'):
-            age_limit = 18
-
-        media_class = media_info.get('class')
-        if media_class not in ('video', 'audio'):
-            raise ExtractorError('not a video or an audio')
-
-        user = media_info.get('user', {})
-
-        thumbnail = media_info.get('cover_url')
-        if thumbnail:
-            thumbnail.format(width='1600', height='1200')
-
-        # TODO: get correct ext for audio files
-        stream_type = media_info.get('stream_type')
-        formats = [{
-            'url': media_info['href'],
-            'ext': stream_type,
-        }]
-        if media_info.get('is_hd'):
-            formats.append({
-                'format_id': 'hd',
-                'url': media_info['hrefhd'],
-                'ext': stream_type,
-            })
-        if media_class == 'audio':
-            formats[0]['vcodec'] = 'none'
-        else:
-            formats[0]['format_id'] = 'sd'
-
-        return {
-            'id': media_info['identifier'].split('|')[1],
-            'display_id': display_id,
-            'formats': formats,
-            'title': media_info['title'],
-            'description': media_info.get('description'),
-            'thumbnail': thumbnail,
-            'uploader_id': user.get('username'),
-            'uploader': user.get('fullname'),
-            'timestamp': parse_iso8601(media_info.get('published'), ' '),
-            'duration': int_or_none(media_info.get('duration')),
-            'view_count': int_or_none(media_info.get('count_views')),
-            'like_count': int_or_none(media_info.get('count_likes')),
-            'comment_count': int_or_none(media_info.get('count_comments')),
-            'age_limit': age_limit,
-        }
diff --git a/yt_dlp/extractor/tube8.py b/yt_dlp/extractor/tube8.py
index 77ed05ffd..5f15b4581 100644
--- a/yt_dlp/extractor/tube8.py
+++ b/yt_dlp/extractor/tube8.py
@@ -1,13 +1,20 @@
 import re
 
+from .common import InfoExtractor
+from ..aes import aes_decrypt_text
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
+    determine_ext,
+    format_field,
     int_or_none,
     str_to_int,
+    strip_or_none,
+    url_or_none,
 )
-from .keezmovies import KeezMoviesIE
 
 
-class Tube8IE(KeezMoviesIE):  # XXX: Do not subclass from concrete IE
+class Tube8IE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/(?P<id>\d+)'
     _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?tube8\.com/embed/(?:[^/]+/)+\d+)']
     _TESTS = [{
@@ -30,6 +37,90 @@ class Tube8IE(KeezMoviesIE):  # XXX: Do not subclass from concrete IE
         'only_matching': True,
     }]
 
+    def _extract_info(self, url, fatal=True):
+        mobj = self._match_valid_url(url)
+        video_id = mobj.group('id')
+        display_id = (mobj.group('display_id')
+                      if 'display_id' in mobj.groupdict()
+                      else None) or mobj.group('id')
+
+        webpage = self._download_webpage(
+            url, display_id, headers={'Cookie': 'age_verified=1'})
+
+        formats = []
+        format_urls = set()
+
+        title = None
+        thumbnail = None
+        duration = None
+        encrypted = False
+
+        def extract_format(format_url, height=None):
+            format_url = url_or_none(format_url)
+            if not format_url or not format_url.startswith(('http', '//')):
+                return
+            if format_url in format_urls:
+                return
+            format_urls.add(format_url)
+            tbr = int_or_none(self._search_regex(
+                r'[/_](\d+)[kK][/_]', format_url, 'tbr', default=None))
+            if not height:
+                height = int_or_none(self._search_regex(
+                    r'[/_](\d+)[pP][/_]', format_url, 'height', default=None))
+            if encrypted:
+                format_url = aes_decrypt_text(
+                    video_url, title, 32).decode('utf-8')
+            formats.append({
+                'url': format_url,
+                'format_id': format_field(height, None, '%dp'),
+                'height': height,
+                'tbr': tbr,
+            })
+
+        flashvars = self._parse_json(
+            self._search_regex(
+                r'flashvars\s*=\s*({.+?});', webpage,
+                'flashvars', default='{}'),
+            display_id, fatal=False)
+
+        if flashvars:
+            title = flashvars.get('video_title')
+            thumbnail = flashvars.get('image_url')
+            duration = int_or_none(flashvars.get('video_duration'))
+            encrypted = flashvars.get('encrypted') is True
+            for key, value in flashvars.items():
+                mobj = re.search(r'quality_(\d+)[pP]', key)
+                if mobj:
+                    extract_format(value, int(mobj.group(1)))
+            video_url = flashvars.get('video_url')
+            if video_url and determine_ext(video_url, None):
+                extract_format(video_url)
+
+        video_url = self._html_search_regex(
+            r'flashvars\.video_url\s*=\s*(["\'])(?P<url>http.+?)\1',
+            webpage, 'video url', default=None, group='url')
+        if video_url:
+            extract_format(compat_urllib_parse_unquote(video_url))
+
+        if not formats:
+            if 'title="This video is no longer available"' in webpage:
+                self.raise_no_formats(
+                    'Video %s is no longer available' % video_id, expected=True)
+
+        if not title:
+            title = self._html_search_regex(
+                r'<h1[^>]*>([^<]+)', webpage, 'title')
+
+        return webpage, {
+            'id': video_id,
+            'display_id': display_id,
+            'title': strip_or_none(title),
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'age_limit': 18,
+            'formats': formats,
+        }
+
     def _real_extract(self, url):
         webpage, info = self._extract_info(url)
 
diff --git a/yt_dlp/extractor/tunepk.py b/yt_dlp/extractor/tunepk.py
deleted file mode 100644
index e4e507b00..000000000
--- a/yt_dlp/extractor/tunepk.py
+++ /dev/null
@@ -1,87 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    int_or_none,
-    try_get,
-    unified_timestamp,
-)
-
-
-class TunePkIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:
-                            (?:www\.)?tune\.pk/(?:video/|player/embed_player.php?.*?\bvid=)|
-                            embed\.tune\.pk/play/
-                        )
-                        (?P<id>\d+)
-                    '''
-    _TESTS = [{
-        'url': 'https://tune.pk/video/6919541/maudie-2017-international-trailer-1-ft-ethan-hawke-sally-hawkins',
-        'md5': '0c537163b7f6f97da3c5dd1e3ef6dd55',
-        'info_dict': {
-            'id': '6919541',
-            'ext': 'mp4',
-            'title': 'Maudie (2017) | International Trailer # 1 ft Ethan Hawke, Sally Hawkins',
-            'description': 'md5:eb5a04114fafef5cec90799a93a2d09c',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1487327564,
-            'upload_date': '20170217',
-            'uploader': 'Movie Trailers',
-            'duration': 107,
-            'view_count': int,
-        }
-    }, {
-        'url': 'https://tune.pk/player/embed_player.php?vid=6919541&folder=2017/02/17/&width=600&height=350&autoplay=no',
-        'only_matching': True,
-    }, {
-        'url': 'https://embed.tune.pk/play/6919541?autoplay=no&ssl=yes&inline=true',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'https://tune.pk/video/%s' % video_id, video_id)
-
-        details = self._parse_json(
-            self._search_regex(
-                r'new\s+TunePlayer\(({.+?})\)\s*;\s*\n', webpage, 'tune player'),
-            video_id)['details']
-
-        video = details['video']
-        title = video.get('title') or self._og_search_title(
-            webpage, default=None) or self._html_search_meta(
-            'title', webpage, 'title', fatal=True)
-
-        formats = self._parse_jwplayer_formats(
-            details['player']['sources'], video_id)
-
-        description = self._og_search_description(
-            webpage, default=None) or self._html_search_meta(
-            'description', webpage, 'description')
-
-        thumbnail = video.get('thumb') or self._og_search_thumbnail(
-            webpage, default=None) or self._html_search_meta(
-            'thumbnail', webpage, 'thumbnail')
-
-        timestamp = unified_timestamp(video.get('date_added'))
-        uploader = try_get(
-            video, lambda x: x['uploader']['name'],
-            compat_str) or self._html_search_meta('author', webpage, 'author')
-
-        duration = int_or_none(video.get('duration'))
-        view_count = int_or_none(video.get('views'))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'uploader': uploader,
-            'duration': duration,
-            'view_count': view_count,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/tvnet.py b/yt_dlp/extractor/tvnet.py
deleted file mode 100644
index 77426f7e6..000000000
--- a/yt_dlp/extractor/tvnet.py
+++ /dev/null
@@ -1,138 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    unescapeHTML,
-    url_or_none,
-)
-
-
-class TVNetIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+)\.tvnet\.gov\.vn/[^/]+/(?:\d+/)?(?P<id>\d+)(?:/|$)'
-    _TESTS = [{
-        # video
-        'url': 'http://de.tvnet.gov.vn/video/109788/vtv1---bac-tuyet-tai-lao-cai-va-ha-giang/tin-nong-24h',
-        'md5': 'b4d7abe0252c9b47774760b7519c7558',
-        'info_dict': {
-            'id': '109788',
-            'ext': 'mp4',
-            'title': 'VTV1 - Bắc tuyết tại Lào Cai và Hà Giang',
-            'thumbnail': r're:(?i)https?://.*\.(?:jpg|png)',
-            'is_live': False,
-            'view_count': int,
-        },
-    }, {
-        # audio
-        'url': 'http://vn.tvnet.gov.vn/radio/27017/vov1---ban-tin-chieu-10062018/doi-song-va-xa-hoi',
-        'md5': 'b5875ce9b0a2eecde029216d0e6db2ae',
-        'info_dict': {
-            'id': '27017',
-            'ext': 'm4a',
-            'title': 'VOV1 - Bản tin chiều (10/06/2018)',
-            'thumbnail': r're:(?i)https?://.*\.(?:jpg|png)',
-            'is_live': False,
-        },
-    }, {
-        'url': 'http://us.tvnet.gov.vn/video/118023/129999/ngay-0705',
-        'info_dict': {
-            'id': '129999',
-            'ext': 'mp4',
-            'title': 'VTV1 - Quốc hội với cử tri (11/06/2018)',
-            'thumbnail': r're:(?i)https?://.*\.(?:jpg|png)',
-            'is_live': False,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        # live stream
-        'url': 'http://us.tvnet.gov.vn/kenh-truyen-hinh/1011/vtv1',
-        'info_dict': {
-            'id': '1011',
-            'ext': 'mp4',
-            'title': r're:^VTV1 \| LiveTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'thumbnail': r're:(?i)https?://.*\.(?:jpg|png)',
-            'is_live': True,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        # radio live stream
-        'url': 'http://vn.tvnet.gov.vn/kenh-truyen-hinh/1014',
-        'info_dict': {
-            'id': '1014',
-            'ext': 'm4a',
-            'title': r're:VOV1 \| LiveTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'thumbnail': r're:(?i)https?://.*\.(?:jpg|png)',
-            'is_live': True,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://us.tvnet.gov.vn/phim/6136/25510/vtv3---ca-mot-doi-an-oan-tap-1-50/phim-truyen-hinh',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._og_search_title(
-            webpage, default=None) or self._html_search_meta(
-            'title', webpage, default=None) or self._search_regex(
-            r'<title>([^<]+)<', webpage, 'title')
-        title = re.sub(r'\s*-\s*TV Net\s*$', '', title)
-
-        if '/video/' in url or '/radio/' in url:
-            is_live = False
-        elif '/kenh-truyen-hinh/' in url:
-            is_live = True
-        else:
-            is_live = None
-
-        data_file = unescapeHTML(self._search_regex(
-            r'data-file=(["\'])(?P<url>(?:https?:)?//.+?)\1', webpage,
-            'data file', group='url'))
-
-        stream_urls = set()
-        formats = []
-        for stream in self._download_json(data_file, video_id):
-            if not isinstance(stream, dict):
-                continue
-            stream_url = url_or_none(stream.get('url'))
-            if stream_url in stream_urls or not stream_url:
-                continue
-            stream_urls.add(stream_url)
-            formats.extend(self._extract_m3u8_formats(
-                stream_url, video_id, 'mp4', live=is_live, m3u8_id='hls', fatal=False))
-
-        # better support for radio streams
-        if title.startswith('VOV'):
-            for f in formats:
-                f.update({
-                    'ext': 'm4a',
-                    'vcodec': 'none',
-                })
-
-        thumbnail = self._og_search_thumbnail(
-            webpage, default=None) or unescapeHTML(
-            self._search_regex(
-                r'data-image=(["\'])(?P<url>(?:https?:)?//.+?)\1', webpage,
-                'thumbnail', default=None, group='url'))
-
-        view_count = int_or_none(self._search_regex(
-            r'(?s)<div[^>]+\bclass=["\'].*?view-count[^>]+>.*?(\d+).*?</div>',
-            webpage, 'view count', default=None))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'is_live': is_live,
-            'view_count': view_count,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/tvnow.py b/yt_dlp/extractor/tvnow.py
deleted file mode 100644
index 0acc306df..000000000
--- a/yt_dlp/extractor/tvnow.py
+++ /dev/null
@@ -1,639 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    get_element_by_id,
-    int_or_none,
-    parse_iso8601,
-    parse_duration,
-    str_or_none,
-    try_get,
-    update_url_query,
-    urljoin,
-)
-
-
-class TVNowBaseIE(InfoExtractor):
-    _VIDEO_FIELDS = (
-        'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
-        'broadcastStartDate', 'isDrm', 'duration', 'season', 'episode',
-        'manifest.dashclear', 'manifest.hlsclear', 'manifest.smoothclear',
-        'format.title', 'format.defaultImage169Format', 'format.defaultImage169Logo')
-
-    def _call_api(self, path, video_id, query):
-        return self._download_json(
-            'https://api.tvnow.de/v3/' + path, video_id, query=query)
-
-    def _extract_video(self, info, display_id):
-        video_id = compat_str(info['id'])
-        title = info['title']
-
-        paths = []
-        for manifest_url in (info.get('manifest') or {}).values():
-            if not manifest_url:
-                continue
-            manifest_url = update_url_query(manifest_url, {'filter': ''})
-            path = self._search_regex(r'https?://[^/]+/(.+?)\.ism/', manifest_url, 'path')
-            if path in paths:
-                continue
-            paths.append(path)
-
-            def url_repl(proto, suffix):
-                return re.sub(
-                    r'(?:hls|dash|hss)([.-])', proto + r'\1', re.sub(
-                        r'\.ism/(?:[^.]*\.(?:m3u8|mpd)|[Mm]anifest)',
-                        '.ism/' + suffix, manifest_url))
-
-            def make_urls(proto, suffix):
-                urls = [url_repl(proto, suffix)]
-                hd_url = urls[0].replace('/manifest/', '/ngvod/')
-                if hd_url != urls[0]:
-                    urls.append(hd_url)
-                return urls
-
-            for man_url in make_urls('dash', '.mpd'):
-                formats = self._extract_mpd_formats(
-                    man_url, video_id, mpd_id='dash', fatal=False)
-            for man_url in make_urls('hss', 'Manifest'):
-                formats.extend(self._extract_ism_formats(
-                    man_url, video_id, ism_id='mss', fatal=False))
-            for man_url in make_urls('hls', '.m3u8'):
-                formats.extend(self._extract_m3u8_formats(
-                    man_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
-                    fatal=False))
-            if formats:
-                break
-        else:
-            if not self.get_param('allow_unplayable_formats') and info.get('isDrm'):
-                raise ExtractorError(
-                    'Video %s is DRM protected' % video_id, expected=True)
-            if info.get('geoblocked'):
-                raise self.raise_geo_restricted()
-            if not info.get('free', True):
-                raise ExtractorError(
-                    'Video %s is not available for free' % video_id, expected=True)
-
-        description = info.get('articleLong') or info.get('articleShort')
-        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
-        duration = parse_duration(info.get('duration'))
-
-        f = info.get('format', {})
-
-        thumbnails = [{
-            'url': 'https://aistvnow-a.akamaihd.net/tvnow/movie/%s' % video_id,
-        }]
-        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
-        if thumbnail:
-            thumbnails.append({
-                'url': thumbnail,
-            })
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnails': thumbnails,
-            'timestamp': timestamp,
-            'duration': duration,
-            'series': f.get('title'),
-            'season_number': int_or_none(info.get('season')),
-            'episode_number': int_or_none(info.get('episode')),
-            'episode': title,
-            'formats': formats,
-        }
-
-
-class TVNowIE(TVNowBaseIE):
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?tvnow\.(?:de|at|ch)/(?P<station>[^/]+)/
-                        (?P<show_id>[^/]+)/
-                        (?!(?:list|jahr)(?:/|$))(?P<id>[^/?\#&]+)
-                    '''
-
-    @classmethod
-    def suitable(cls, url):
-        return (False if TVNowNewIE.suitable(url) or TVNowSeasonIE.suitable(url) or TVNowAnnualIE.suitable(url) or TVNowShowIE.suitable(url)
-                else super(TVNowIE, cls).suitable(url))
-
-    _TESTS = [{
-        'url': 'https://www.tvnow.de/rtl2/grip-das-motormagazin/der-neue-porsche-911-gt-3/player',
-        'info_dict': {
-            'id': '331082',
-            'display_id': 'grip-das-motormagazin/der-neue-porsche-911-gt-3',
-            'ext': 'mp4',
-            'title': 'Der neue Porsche 911 GT 3',
-            'description': 'md5:6143220c661f9b0aae73b245e5d898bb',
-            'timestamp': 1495994400,
-            'upload_date': '20170528',
-            'duration': 5283,
-            'series': 'GRIP - Das Motormagazin',
-            'season_number': 14,
-            'episode_number': 405,
-            'episode': 'Der neue Porsche 911 GT 3',
-        },
-    }, {
-        # rtl2
-        'url': 'https://www.tvnow.de/rtl2/armes-deutschland/episode-0008/player',
-        'only_matching': True,
-    }, {
-        # rtlnitro
-        'url': 'https://www.tvnow.de/nitro/alarm-fuer-cobra-11-die-autobahnpolizei/auf-eigene-faust-pilot/player',
-        'only_matching': True,
-    }, {
-        # superrtl
-        'url': 'https://www.tvnow.de/superrtl/die-lustigsten-schlamassel-der-welt/u-a-ketchup-effekt/player',
-        'only_matching': True,
-    }, {
-        # ntv
-        'url': 'https://www.tvnow.de/ntv/startup-news/goetter-in-weiss/player',
-        'only_matching': True,
-    }, {
-        # vox
-        'url': 'https://www.tvnow.de/vox/auto-mobil/neues-vom-automobilmarkt-2017-11-19-17-00-00/player',
-        'only_matching': True,
-    }, {
-        # rtlplus
-        'url': 'https://www.tvnow.de/rtlplus/op-ruft-dr-bruckner/die-vernaehte-frau/player',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.tvnow.de/rtl2/grip-das-motormagazin/der-neue-porsche-911-gt-3',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        display_id = '%s/%s' % mobj.group(2, 3)
-
-        info = self._call_api(
-            'movies/' + display_id, display_id, query={
-                'fields': ','.join(self._VIDEO_FIELDS),
-            })
-
-        return self._extract_video(info, display_id)
-
-
-class TVNowNewIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    (?P<base_url>https?://
-                        (?:www\.)?tvnow\.(?:de|at|ch)/
-                        (?:shows|serien))/
-                        (?P<show>[^/]+)-\d+/
-                        [^/]+/
-                        episode-\d+-(?P<episode>[^/?$&]+)-(?P<id>\d+)
-                    '''
-
-    _TESTS = [{
-        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05/episode-405-der-neue-porsche-911-gt-3-331082',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        base_url = re.sub(r'(?:shows|serien)', '_', mobj.group('base_url'))
-        show, episode = mobj.group('show', 'episode')
-        return self.url_result(
-            # Rewrite new URLs to the old format and use extraction via old API
-            # at api.tvnow.de as a loophole for bypassing premium content checks
-            '%s/%s/%s' % (base_url, show, episode),
-            ie=TVNowIE.ie_key(), video_id=mobj.group('id'))
-
-
-class TVNowFilmIE(TVNowBaseIE):
-    _VALID_URL = r'''(?x)
-                    (?P<base_url>https?://
-                        (?:www\.)?tvnow\.(?:de|at|ch)/
-                        (?:filme))/
-                        (?P<title>[^/?$&]+)-(?P<id>\d+)
-                    '''
-    _TESTS = [{
-        'url': 'https://www.tvnow.de/filme/lord-of-war-haendler-des-todes-7959',
-        'info_dict': {
-            'id': '1426690',
-            'display_id': 'lord-of-war-haendler-des-todes',
-            'ext': 'mp4',
-            'title': 'Lord of War',
-            'description': 'md5:5eda15c0d5b8cb70dac724c8a0ff89a9',
-            'timestamp': 1550010000,
-            'upload_date': '20190212',
-            'duration': 7016,
-        },
-    }, {
-        'url': 'https://www.tvnow.de/filme/the-machinist-12157',
-        'info_dict': {
-            'id': '328160',
-            'display_id': 'the-machinist',
-            'ext': 'mp4',
-            'title': 'The Machinist',
-            'description': 'md5:9a0e363fdd74b3a9e1cdd9e21d0ecc28',
-            'timestamp': 1496469720,
-            'upload_date': '20170603',
-            'duration': 5836,
-        },
-    }, {
-        'url': 'https://www.tvnow.de/filme/horst-schlaemmer-isch-kandidiere-17777',
-        'only_matching': True,  # DRM protected
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        display_id = mobj.group('title')
-
-        webpage = self._download_webpage(url, display_id, fatal=False)
-        if not webpage:
-            raise ExtractorError('Cannot download "%s"' % url, expected=True)
-
-        json_text = get_element_by_id('now-web-state', webpage)
-        if not json_text:
-            raise ExtractorError('Cannot read video data', expected=True)
-
-        json_data = self._parse_json(
-            json_text,
-            display_id,
-            transform_source=lambda x: x.replace('&q;', '"'),
-            fatal=False)
-        if not json_data:
-            raise ExtractorError('Cannot read video data', expected=True)
-
-        player_key = next(
-            (key for key in json_data.keys() if 'module/player' in key),
-            None)
-        page_key = next(
-            (key for key in json_data.keys() if 'page/filme' in key),
-            None)
-        movie_id = try_get(
-            json_data,
-            [
-                lambda x: x[player_key]['body']['id'],
-                lambda x: x[page_key]['body']['modules'][0]['id'],
-                lambda x: x[page_key]['body']['modules'][1]['id']],
-            int)
-        if not movie_id:
-            raise ExtractorError('Cannot extract movie ID', expected=True)
-
-        info = self._call_api(
-            'movies/%d' % movie_id,
-            display_id,
-            query={'fields': ','.join(self._VIDEO_FIELDS)})
-
-        return self._extract_video(info, display_id)
-
-
-class TVNowNewBaseIE(InfoExtractor):
-    def _call_api(self, path, video_id, query={}):
-        result = self._download_json(
-            'https://apigw.tvnow.de/module/' + path, video_id, query=query)
-        error = result.get('error')
-        if error:
-            raise ExtractorError(
-                '%s said: %s' % (self.IE_NAME, error), expected=True)
-        return result
-
-
-r"""
-TODO: new apigw.tvnow.de based version of TVNowIE. Replace old TVNowIE with it
-when api.tvnow.de is shut down. This version can't bypass premium checks though.
-class TVNowIE(TVNowNewBaseIE):
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?tvnow\.(?:de|at|ch)/
-                        (?:shows|serien)/[^/]+/
-                        (?:[^/]+/)+
-                        (?P<display_id>[^/?$&]+)-(?P<id>\d+)
-                    '''
-
-    _TESTS = [{
-        # episode with annual navigation
-        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05/episode-405-der-neue-porsche-911-gt-3-331082',
-        'info_dict': {
-            'id': '331082',
-            'display_id': 'grip-das-motormagazin/der-neue-porsche-911-gt-3',
-            'ext': 'mp4',
-            'title': 'Der neue Porsche 911 GT 3',
-            'description': 'md5:6143220c661f9b0aae73b245e5d898bb',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1495994400,
-            'upload_date': '20170528',
-            'duration': 5283,
-            'series': 'GRIP - Das Motormagazin',
-            'season_number': 14,
-            'episode_number': 405,
-            'episode': 'Der neue Porsche 911 GT 3',
-        },
-    }, {
-        # rtl2, episode with season navigation
-        'url': 'https://www.tvnow.de/shows/armes-deutschland-11471/staffel-3/episode-14-bernd-steht-seit-der-trennung-von-seiner-frau-allein-da-526124',
-        'only_matching': True,
-    }, {
-        # rtlnitro
-        'url': 'https://www.tvnow.de/serien/alarm-fuer-cobra-11-die-autobahnpolizei-1815/staffel-13/episode-5-auf-eigene-faust-pilot-366822',
-        'only_matching': True,
-    }, {
-        # superrtl
-        'url': 'https://www.tvnow.de/shows/die-lustigsten-schlamassel-der-welt-1221/staffel-2/episode-14-u-a-ketchup-effekt-364120',
-        'only_matching': True,
-    }, {
-        # ntv
-        'url': 'https://www.tvnow.de/shows/startup-news-10674/staffel-2/episode-39-goetter-in-weiss-387630',
-        'only_matching': True,
-    }, {
-        # vox
-        'url': 'https://www.tvnow.de/shows/auto-mobil-174/2017-11/episode-46-neues-vom-automobilmarkt-2017-11-19-17-00-00-380072',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05/episode-405-der-neue-porsche-911-gt-3-331082',
-        'only_matching': True,
-    }]
-
-    def _extract_video(self, info, url, display_id):
-        config = info['config']
-        source = config['source']
-
-        video_id = compat_str(info.get('id') or source['videoId'])
-        title = source['title'].strip()
-
-        paths = []
-        for manifest_url in (info.get('manifest') or {}).values():
-            if not manifest_url:
-                continue
-            manifest_url = update_url_query(manifest_url, {'filter': ''})
-            path = self._search_regex(r'https?://[^/]+/(.+?)\.ism/', manifest_url, 'path')
-            if path in paths:
-                continue
-            paths.append(path)
-
-            def url_repl(proto, suffix):
-                return re.sub(
-                    r'(?:hls|dash|hss)([.-])', proto + r'\1', re.sub(
-                        r'\.ism/(?:[^.]*\.(?:m3u8|mpd)|[Mm]anifest)',
-                        '.ism/' + suffix, manifest_url))
-
-            formats = self._extract_mpd_formats(
-                url_repl('dash', '.mpd'), video_id,
-                mpd_id='dash', fatal=False)
-            formats.extend(self._extract_ism_formats(
-                url_repl('hss', 'Manifest'),
-                video_id, ism_id='mss', fatal=False))
-            formats.extend(self._extract_m3u8_formats(
-                url_repl('hls', '.m3u8'), video_id, 'mp4',
-                'm3u8_native', m3u8_id='hls', fatal=False))
-            if formats:
-                break
-        else:
-            if try_get(info, lambda x: x['rights']['isDrm']):
-                raise ExtractorError(
-                    'Video %s is DRM protected' % video_id, expected=True)
-            if try_get(config, lambda x: x['boards']['geoBlocking']['block']):
-                raise self.raise_geo_restricted()
-            if not info.get('free', True):
-                raise ExtractorError(
-                    'Video %s is not available for free' % video_id, expected=True)
-
-        description = source.get('description')
-        thumbnail = url_or_none(source.get('poster'))
-        timestamp = unified_timestamp(source.get('previewStart'))
-        duration = parse_duration(source.get('length'))
-
-        series = source.get('format')
-        season_number = int_or_none(self._search_regex(
-            r'staffel-(\d+)', url, 'season number', default=None))
-        episode_number = int_or_none(self._search_regex(
-            r'episode-(\d+)', url, 'episode number', default=None))
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'series': series,
-            'season_number': season_number,
-            'episode_number': episode_number,
-            'episode': title,
-            'formats': formats,
-        }
-
-    def _real_extract(self, url):
-        display_id, video_id = self._match_valid_url(url).groups()
-        info = self._call_api('player/' + video_id, video_id)
-        return self._extract_video(info, video_id, display_id)
-
-
-class TVNowFilmIE(TVNowIE):  # XXX: Do not subclass from concrete IE
-    _VALID_URL = r'''(?x)
-                    (?P<base_url>https?://
-                        (?:www\.)?tvnow\.(?:de|at|ch)/
-                        (?:filme))/
-                        (?P<title>[^/?$&]+)-(?P<id>\d+)
-                    '''
-    _TESTS = [{
-        'url': 'https://www.tvnow.de/filme/lord-of-war-haendler-des-todes-7959',
-        'info_dict': {
-            'id': '1426690',
-            'display_id': 'lord-of-war-haendler-des-todes',
-            'ext': 'mp4',
-            'title': 'Lord of War',
-            'description': 'md5:5eda15c0d5b8cb70dac724c8a0ff89a9',
-            'timestamp': 1550010000,
-            'upload_date': '20190212',
-            'duration': 7016,
-        },
-    }, {
-        'url': 'https://www.tvnow.de/filme/the-machinist-12157',
-        'info_dict': {
-            'id': '328160',
-            'display_id': 'the-machinist',
-            'ext': 'mp4',
-            'title': 'The Machinist',
-            'description': 'md5:9a0e363fdd74b3a9e1cdd9e21d0ecc28',
-            'timestamp': 1496469720,
-            'upload_date': '20170603',
-            'duration': 5836,
-        },
-    }, {
-        'url': 'https://www.tvnow.de/filme/horst-schlaemmer-isch-kandidiere-17777',
-        'only_matching': True,  # DRM protected
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        display_id = mobj.group('title')
-
-        webpage = self._download_webpage(url, display_id, fatal=False)
-        if not webpage:
-            raise ExtractorError('Cannot download "%s"' % url, expected=True)
-
-        json_text = get_element_by_id('now-web-state', webpage)
-        if not json_text:
-            raise ExtractorError('Cannot read video data', expected=True)
-
-        json_data = self._parse_json(
-            json_text,
-            display_id,
-            transform_source=lambda x: x.replace('&q;', '"'),
-            fatal=False)
-        if not json_data:
-            raise ExtractorError('Cannot read video data', expected=True)
-
-        player_key = next(
-            (key for key in json_data.keys() if 'module/player' in key),
-            None)
-        page_key = next(
-            (key for key in json_data.keys() if 'page/filme' in key),
-            None)
-        movie_id = try_get(
-            json_data,
-            [
-                lambda x: x[player_key]['body']['id'],
-                lambda x: x[page_key]['body']['modules'][0]['id'],
-                lambda x: x[page_key]['body']['modules'][1]['id']],
-            int)
-        if not movie_id:
-            raise ExtractorError('Cannot extract movie ID', expected=True)
-
-        info = self._call_api('player/%d' % movie_id, display_id)
-        return self._extract_video(info, url, display_id)
-"""
-
-
-class TVNowListBaseIE(TVNowNewBaseIE):
-    _SHOW_VALID_URL = r'''(?x)
-                    (?P<base_url>
-                        https?://
-                            (?:www\.)?tvnow\.(?:de|at|ch)/(?:shows|serien)/
-                            [^/?#&]+-(?P<show_id>\d+)
-                    )
-                    '''
-
-    @classmethod
-    def suitable(cls, url):
-        return (False if TVNowNewIE.suitable(url)
-                else super(TVNowListBaseIE, cls).suitable(url))
-
-    def _extract_items(self, url, show_id, list_id, query):
-        items = self._call_api(
-            'teaserrow/format/episode/' + show_id, list_id,
-            query=query)['items']
-
-        entries = []
-        for item in items:
-            if not isinstance(item, dict):
-                continue
-            item_url = urljoin(url, item.get('url'))
-            if not item_url:
-                continue
-            video_id = str_or_none(item.get('id') or item.get('videoId'))
-            item_title = item.get('subheadline') or item.get('text')
-            entries.append(self.url_result(
-                item_url, ie=TVNowNewIE.ie_key(), video_id=video_id,
-                video_title=item_title))
-
-        return self.playlist_result(entries, '%s/%s' % (show_id, list_id))
-
-
-class TVNowSeasonIE(TVNowListBaseIE):
-    _VALID_URL = r'%s/staffel-(?P<id>\d+)' % TVNowListBaseIE._SHOW_VALID_URL
-    _TESTS = [{
-        'url': 'https://www.tvnow.de/serien/alarm-fuer-cobra-11-die-autobahnpolizei-1815/staffel-13',
-        'info_dict': {
-            'id': '1815/13',
-        },
-        'playlist_mincount': 22,
-    }]
-
-    def _real_extract(self, url):
-        _, show_id, season_id = self._match_valid_url(url).groups()
-        return self._extract_items(
-            url, show_id, season_id, {'season': season_id})
-
-
-class TVNowAnnualIE(TVNowListBaseIE):
-    _VALID_URL = r'%s/(?P<year>\d{4})-(?P<month>\d{2})' % TVNowListBaseIE._SHOW_VALID_URL
-    _TESTS = [{
-        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05',
-        'info_dict': {
-            'id': '1669/2017-05',
-        },
-        'playlist_mincount': 2,
-    }]
-
-    def _real_extract(self, url):
-        _, show_id, year, month = self._match_valid_url(url).groups()
-        return self._extract_items(
-            url, show_id, '%s-%s' % (year, month), {
-                'year': int(year),
-                'month': int(month),
-            })
-
-
-class TVNowShowIE(TVNowListBaseIE):
-    _VALID_URL = TVNowListBaseIE._SHOW_VALID_URL
-    _TESTS = [{
-        # annual navigationType
-        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669',
-        'info_dict': {
-            'id': '1669',
-        },
-        'playlist_mincount': 73,
-    }, {
-        # season navigationType
-        'url': 'https://www.tvnow.de/shows/armes-deutschland-11471',
-        'info_dict': {
-            'id': '11471',
-        },
-        'playlist_mincount': 3,
-    }]
-
-    @classmethod
-    def suitable(cls, url):
-        return (False if TVNowNewIE.suitable(url) or TVNowSeasonIE.suitable(url) or TVNowAnnualIE.suitable(url)
-                else super(TVNowShowIE, cls).suitable(url))
-
-    def _real_extract(self, url):
-        base_url, show_id = self._match_valid_url(url).groups()
-
-        result = self._call_api(
-            'teaserrow/format/navigation/' + show_id, show_id)
-
-        items = result['items']
-
-        entries = []
-        navigation = result.get('navigationType')
-        if navigation == 'annual':
-            for item in items:
-                if not isinstance(item, dict):
-                    continue
-                year = int_or_none(item.get('year'))
-                if year is None:
-                    continue
-                months = item.get('months')
-                if not isinstance(months, list):
-                    continue
-                for month_dict in months:
-                    if not isinstance(month_dict, dict) or not month_dict:
-                        continue
-                    month_number = int_or_none(list(month_dict.keys())[0])
-                    if month_number is None:
-                        continue
-                    entries.append(self.url_result(
-                        '%s/%04d-%02d' % (base_url, year, month_number),
-                        ie=TVNowAnnualIE.ie_key()))
-        elif navigation == 'season':
-            for item in items:
-                if not isinstance(item, dict):
-                    continue
-                season_number = int_or_none(item.get('season'))
-                if season_number is None:
-                    continue
-                entries.append(self.url_result(
-                    '%s/staffel-%d' % (base_url, season_number),
-                    ie=TVNowSeasonIE.ie_key()))
-        else:
-            raise ExtractorError('Unknown navigationType')
-
-        return self.playlist_result(entries, show_id)
diff --git a/yt_dlp/extractor/twentyfourvideo.py b/yt_dlp/extractor/twentyfourvideo.py
deleted file mode 100644
index baeb85d47..000000000
--- a/yt_dlp/extractor/twentyfourvideo.py
+++ /dev/null
@@ -1,128 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    parse_iso8601,
-    int_or_none,
-    xpath_attr,
-    xpath_element,
-)
-
-
-class TwentyFourVideoIE(InfoExtractor):
-    IE_NAME = '24video'
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?P<host>
-                            (?:(?:www|porno?)\.)?24video\.
-                            (?:net|me|xxx|sexy?|tube|adult|site|vip)
-                        )/
-                        (?:
-                            video/(?:(?:view|xml)/)?|
-                            player/new24_play\.swf\?id=
-                        )
-                        (?P<id>\d+)
-                    '''
-
-    _TESTS = [{
-        'url': 'http://www.24video.net/video/view/1044982',
-        'md5': 'e09fc0901d9eaeedac872f154931deeb',
-        'info_dict': {
-            'id': '1044982',
-            'ext': 'mp4',
-            'title': 'Эротика каменного века',
-            'description': 'Как смотрели порно в каменном веке.',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'SUPERTELO',
-            'duration': 31,
-            'timestamp': 1275937857,
-            'upload_date': '20100607',
-            'age_limit': 18,
-            'like_count': int,
-            'dislike_count': int,
-        },
-    }, {
-        'url': 'http://www.24video.net/player/new24_play.swf?id=1044982',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.24video.me/video/view/1044982',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.24video.tube/video/view/2363750',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.24video.site/video/view/2640421',
-        'only_matching': True,
-    }, {
-        'url': 'https://porno.24video.net/video/2640421-vsya-takaya-gibkaya-i-v-masle',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.24video.vip/video/view/1044982',
-        'only_matching': True,
-    }, {
-        'url': 'https://porn.24video.net/video/2640421-vsya-takay',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-        host = mobj.group('host')
-
-        webpage = self._download_webpage(
-            'http://%s/video/view/%s' % (host, video_id), video_id)
-
-        title = self._og_search_title(webpage)
-        description = self._html_search_regex(
-            r'<(p|span)[^>]+itemprop="description"[^>]*>(?P<description>[^<]+)</\1>',
-            webpage, 'description', fatal=False, group='description')
-        thumbnail = self._og_search_thumbnail(webpage)
-        duration = int_or_none(self._og_search_property(
-            'duration', webpage, 'duration', fatal=False))
-        timestamp = parse_iso8601(self._search_regex(
-            r'<time[^>]+\bdatetime="([^"]+)"[^>]+itemprop="uploadDate"',
-            webpage, 'upload date', fatal=False))
-
-        uploader = self._html_search_regex(
-            r'class="video-uploaded"[^>]*>\s*<a href="/jsecUser/movies/[^"]+"[^>]*>([^<]+)</a>',
-            webpage, 'uploader', fatal=False)
-
-        view_count = int_or_none(self._html_search_regex(
-            r'<span class="video-views">(\d+) просмотр',
-            webpage, 'view count', fatal=False))
-        comment_count = int_or_none(self._html_search_regex(
-            r'<a[^>]+href="#tab-comments"[^>]*>(\d+) комментари',
-            webpage, 'comment count', default=None))
-
-        # Sets some cookies
-        self._download_xml(
-            r'http://%s/video/xml/%s?mode=init' % (host, video_id),
-            video_id, 'Downloading init XML')
-
-        video_xml = self._download_xml(
-            'http://%s/video/xml/%s?mode=play' % (host, video_id),
-            video_id, 'Downloading video XML')
-
-        video = xpath_element(video_xml, './/video', 'video', fatal=True)
-
-        formats = [{
-            'url': xpath_attr(video, '', 'url', 'video URL', fatal=True),
-        }]
-
-        like_count = int_or_none(video.get('ratingPlus'))
-        dislike_count = int_or_none(video.get('ratingMinus'))
-        age_limit = 18 if video.get('adult') == 'true' else 0
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'duration': duration,
-            'timestamp': timestamp,
-            'view_count': view_count,
-            'comment_count': comment_count,
-            'like_count': like_count,
-            'dislike_count': dislike_count,
-            'age_limit': age_limit,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/unscripted.py b/yt_dlp/extractor/unscripted.py
deleted file mode 100644
index 6643a71b1..000000000
--- a/yt_dlp/extractor/unscripted.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from .common import InfoExtractor
-from ..utils import parse_duration, traverse_obj
-
-
-class UnscriptedNewsVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.unscripted\.news/videos/(?P<id>[\w-]+)'
-    _TESTS = [{
-        'url': 'https://www.unscripted.news/videos/a-day-at-the-farmers-protest',
-        'info_dict': {
-            'id': '60c0a55cd1e99b1079918a57',
-            'display_id': 'a-day-at-the-farmers-protest',
-            'ext': 'mp4',
-            'title': 'A Day at the Farmers\' Protest',
-            'description': 'md5:4b3df22747a03e8f14f746dd72190384',
-            'thumbnail': 'https://s3.unscripted.news/anj2/60c0a55cd1e99b1079918a57/5f199a65-c803-4a5c-8fce-2077359c3b72.jpg',
-            'duration': 2251.0,
-            'series': 'Ground Reports',
-        }
-    }, {
-        'url': 'https://www.unscripted.news/videos/you-get-the-politicians-you-deserve-ft-shashi-tharoor',
-        'info_dict': {
-            'id': '5fb3afbf18ac817d341a74d8',
-            'display_id': 'you-get-the-politicians-you-deserve-ft-shashi-tharoor',
-            'ext': 'mp4',
-            'cast': ['Avalok Langer', 'Ashwin Mehta'],
-            'thumbnail': 'https://s3.unscripted.news/anj2/5fb3afbf18ac817d341a74d8/82bd7942-4f20-4cd8-98ae-83f9e814f998.jpg',
-            'description': 'md5:1e91b069238a705ca3a40f87e6f1182c',
-            'duration': 1046.0,
-            'series': 'Dumb Questions Only',
-            'title': 'You Get The Politicians You Deserve! ft. Shashi Tharoor',
-        }
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        nextjs_data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['dataLocal']
-
-        # TODO: get subtitle from srt key
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(nextjs_data['alt_content'], display_id)
-
-        return {
-            'id': nextjs_data['_id'],
-            'display_id': display_id,
-            'title': nextjs_data.get('title') or self._og_search_title(webpage),
-            'description': nextjs_data.get('sh_heading') or self._og_search_description(webpage),
-            'formats': formats,
-            'subtitles': subtitles,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'duration': parse_duration(nextjs_data.get('duration')),
-            'series': traverse_obj(nextjs_data, ('show', 'topic')),
-            'cast': traverse_obj(nextjs_data, ('cast_crew', ..., 'displayname')),
-        }
diff --git a/yt_dlp/extractor/veehd.py b/yt_dlp/extractor/veehd.py
deleted file mode 100644
index 5ecd88726..000000000
--- a/yt_dlp/extractor/veehd.py
+++ /dev/null
@@ -1,116 +0,0 @@
-import re
-import json
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urlparse,
-)
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    get_element_by_id,
-)
-
-
-class VeeHDIE(InfoExtractor):
-    _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'
-
-    # Seems VeeHD videos have multiple copies on several servers, all of
-    # whom have different MD5 checksums, so omit md5 field in all tests
-    _TESTS = [{
-        'url': 'http://veehd.com/video/4639434_Solar-Sinter',
-        'info_dict': {
-            'id': '4639434',
-            'ext': 'mp4',
-            'title': 'Solar Sinter',
-            'uploader_id': 'VideoEyes',
-            'description': 'md5:46a840e8692ddbaffb5f81d9885cb457',
-        },
-        'skip': 'Video deleted',
-    }, {
-        'url': 'http://veehd.com/video/4905758_Elysian-Fields-Channeling',
-        'info_dict': {
-            'id': '4905758',
-            'ext': 'mp4',
-            'title': 'Elysian Fields - Channeling',
-            'description': 'md5:360e4e95fdab58aefbea0f2a19e5604b',
-            'uploader_id': 'spotted',
-        }
-    }, {
-        'url': 'http://veehd.com/video/2046729_2012-2009-DivX-Trailer',
-        'info_dict': {
-            'id': '2046729',
-            'ext': 'avi',
-            'title': '2012 (2009) DivX Trailer',
-            'description': 'md5:75435ee95255e6a9838ac6f6f3a2396b',
-            'uploader_id': 'Movie_Trailers',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        # VeeHD seems to send garbage on the first request.
-        # See https://github.com/ytdl-org/youtube-dl/issues/2102
-        self._download_webpage(url, video_id, 'Requesting webpage')
-        webpage = self._download_webpage(url, video_id)
-
-        if 'This video has been removed<' in webpage:
-            raise ExtractorError('Video %s has been removed' % video_id, expected=True)
-
-        player_path = self._search_regex(
-            r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
-            webpage, 'player path')
-        player_url = compat_urlparse.urljoin(url, player_path)
-
-        self._download_webpage(player_url, video_id, 'Requesting player page')
-        player_page = self._download_webpage(
-            player_url, video_id, 'Downloading player page')
-
-        video_url = None
-
-        config_json = self._search_regex(
-            r'value=\'config=({.+?})\'', player_page, 'config json', default=None)
-
-        if config_json:
-            config = json.loads(config_json)
-            video_url = compat_urllib_parse_unquote(config['clip']['url'])
-
-        if not video_url:
-            video_url = self._html_search_regex(
-                r'<embed[^>]+type="video/divx"[^>]+src="([^"]+)"',
-                player_page, 'video url', default=None)
-
-        if not video_url:
-            iframe_src = self._search_regex(
-                r'<iframe[^>]+src="/?([^"]+)"', player_page, 'iframe url')
-            iframe_url = 'http://veehd.com/%s' % iframe_src
-
-            self._download_webpage(iframe_url, video_id, 'Requesting iframe page')
-            iframe_page = self._download_webpage(
-                iframe_url, video_id, 'Downloading iframe page')
-
-            video_url = self._search_regex(
-                r"file\s*:\s*'([^']+)'", iframe_page, 'video url')
-
-        title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
-        uploader_id = self._html_search_regex(
-            r'<a href="/profile/\d+">(.+?)</a>',
-            webpage, 'uploader')
-        thumbnail = self._search_regex(
-            r'<img id="veehdpreview" src="(.+?)"',
-            webpage, 'thumbnail')
-        description = self._html_search_regex(
-            r'<td class="infodropdown".*?<div>(.*?)<ul',
-            webpage, 'description', flags=re.DOTALL)
-
-        return {
-            '_type': 'video',
-            'id': video_id,
-            'title': title,
-            'url': video_url,
-            'uploader_id': uploader_id,
-            'thumbnail': thumbnail,
-            'description': description,
-        }
diff --git a/yt_dlp/extractor/vice.py b/yt_dlp/extractor/vice.py
index 8a7126853..1a2d667e7 100644
--- a/yt_dlp/extractor/vice.py
+++ b/yt_dlp/extractor/vice.py
@@ -302,12 +302,6 @@ def _url_res(video_url, ie_key):
         if vice_url:
             return _url_res(vice_url, ViceIE.ie_key())
 
-        embed_code = self._search_regex(
-            r'embedCode=([^&\'"]+)', body,
-            'ooyala embed code', default=None)
-        if embed_code:
-            return _url_res('ooyala:%s' % embed_code, 'Ooyala')
-
         youtube_url = YoutubeIE._extract_url(body)
         if youtube_url:
             return _url_res(youtube_url, YoutubeIE.ie_key())
diff --git a/yt_dlp/extractor/vidbit.py b/yt_dlp/extractor/vidbit.py
deleted file mode 100644
index 2813032db..000000000
--- a/yt_dlp/extractor/vidbit.py
+++ /dev/null
@@ -1,82 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    int_or_none,
-    js_to_json,
-    remove_end,
-    unified_strdate,
-)
-
-
-class VidbitIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vidbit\.co/(?:watch|embed)\?.*?\bv=(?P<id>[\da-zA-Z]+)'
-    _TESTS = [{
-        'url': 'http://www.vidbit.co/watch?v=jkL2yDOEq2',
-        'md5': '1a34b7f14defe3b8fafca9796892924d',
-        'info_dict': {
-            'id': 'jkL2yDOEq2',
-            'ext': 'mp4',
-            'title': 'Intro to VidBit',
-            'description': 'md5:5e0d6142eec00b766cbf114bfd3d16b7',
-            'thumbnail': r're:https?://.*\.jpg$',
-            'upload_date': '20160618',
-            'view_count': int,
-            'comment_count': int,
-        }
-    }, {
-        'url': 'http://www.vidbit.co/embed?v=jkL2yDOEq2&auto=0&water=0',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            compat_urlparse.urljoin(url, '/watch?v=%s' % video_id), video_id)
-
-        video_url, title = [None] * 2
-
-        config = self._parse_json(self._search_regex(
-            r'(?s)\.setup\(({.+?})\);', webpage, 'setup', default='{}'),
-            video_id, transform_source=js_to_json)
-        if config:
-            if config.get('file'):
-                video_url = compat_urlparse.urljoin(url, config['file'])
-            title = config.get('title')
-
-        if not video_url:
-            video_url = compat_urlparse.urljoin(url, self._search_regex(
-                r'file\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1',
-                webpage, 'video URL', group='url'))
-
-        if not title:
-            title = remove_end(
-                self._html_search_regex(
-                    (r'<h1>(.+?)</h1>', r'<title>(.+?)</title>'),
-                    webpage, 'title', default=None) or self._og_search_title(webpage),
-                ' - VidBit')
-
-        description = self._html_search_meta(
-            ('description', 'og:description', 'twitter:description'),
-            webpage, 'description')
-
-        upload_date = unified_strdate(self._html_search_meta(
-            'datePublished', webpage, 'upload date'))
-
-        view_count = int_or_none(self._search_regex(
-            r'<strong>(\d+)</strong> views',
-            webpage, 'view count', fatal=False))
-        comment_count = int_or_none(self._search_regex(
-            r'id=["\']cmt_num["\'][^>]*>\((\d+)\)',
-            webpage, 'comment count', fatal=False))
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'description': description,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'upload_date': upload_date,
-            'view_count': view_count,
-            'comment_count': comment_count,
-        }
diff --git a/yt_dlp/extractor/vimple.py b/yt_dlp/extractor/vimple.py
deleted file mode 100644
index fdccf465e..000000000
--- a/yt_dlp/extractor/vimple.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class SprutoBaseIE(InfoExtractor):
-    def _extract_spruto(self, spruto, video_id):
-        playlist = spruto['playlist'][0]
-        title = playlist['title']
-        video_id = playlist.get('videoId') or video_id
-        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
-        duration = int_or_none(playlist.get('duration'))
-
-        formats = [{
-            'url': f['url'],
-        } for f in playlist['video']]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
-
-
-class VimpleIE(SprutoBaseIE):
-    IE_DESC = 'Vimple - one-click video hosting'
-    _VALID_URL = r'https?://(?:player\.vimple\.(?:ru|co)/iframe|vimple\.(?:ru|co))/(?P<id>[\da-f-]{32,36})'
-    _TESTS = [{
-        'url': 'http://vimple.ru/c0f6b1687dcd4000a97ebe70068039cf',
-        'md5': '2e750a330ed211d3fd41821c6ad9a279',
-        'info_dict': {
-            'id': 'c0f6b168-7dcd-4000-a97e-be70068039cf',
-            'ext': 'mp4',
-            'title': 'Sunset',
-            'duration': 20,
-            'thumbnail': r're:https?://.*?\.jpg',
-        },
-    }, {
-        'url': 'http://player.vimple.ru/iframe/52e1beec-1314-4a83-aeac-c61562eadbf9',
-        'only_matching': True,
-    }, {
-        'url': 'http://vimple.co/04506a053f124483b8fb05ed73899f19',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'http://player.vimple.ru/iframe/%s' % video_id, video_id)
-
-        spruto = self._parse_json(
-            self._search_regex(
-                r'sprutoData\s*:\s*({.+?}),\r\n', webpage, 'spruto data'),
-            video_id)
-
-        return self._extract_spruto(spruto, video_id)
diff --git a/yt_dlp/extractor/vodlocker.py b/yt_dlp/extractor/vodlocker.py
deleted file mode 100644
index b215d6c9d..000000000
--- a/yt_dlp/extractor/vodlocker.py
+++ /dev/null
@@ -1,73 +0,0 @@
-from .common import InfoExtractor
-from ..networking import Request
-from ..utils import NO_DEFAULT, ExtractorError, urlencode_postdata
-
-
-class VodlockerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vodlocker\.(?:com|city)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
-
-    _TESTS = [{
-        'url': 'http://vodlocker.com/e8wvyzz4sl42',
-        'md5': 'ce0c2d18fa0735f1bd91b69b0e54aacf',
-        'info_dict': {
-            'id': 'e8wvyzz4sl42',
-            'ext': 'mp4',
-            'title': 'Germany vs Brazil',
-            'thumbnail': r're:http://.*\.jpg',
-        },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        if any(p in webpage for p in (
-                '>THIS FILE WAS DELETED<',
-                '>File Not Found<',
-                'The file you were looking for could not be found, sorry for any inconvenience.<',
-                '>The file was removed')):
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
-        fields = self._hidden_inputs(webpage)
-
-        if fields['op'] == 'download1':
-            self._sleep(3, video_id)  # they do detect when requests happen too fast!
-            post = urlencode_postdata(fields)
-            req = Request(url, post)
-            req.headers['Content-type'] = 'application/x-www-form-urlencoded'
-            webpage = self._download_webpage(
-                req, video_id, 'Downloading video page')
-
-        def extract_file_url(html, default=NO_DEFAULT):
-            return self._search_regex(
-                r'file:\s*"(http[^\"]+)",', html, 'file url', default=default)
-
-        video_url = extract_file_url(webpage, default=None)
-
-        if not video_url:
-            embed_url = self._search_regex(
-                r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?vodlocker\.(?:com|city)/embed-.+?)\1',
-                webpage, 'embed url', group='url')
-            embed_webpage = self._download_webpage(
-                embed_url, video_id, 'Downloading embed webpage')
-            video_url = extract_file_url(embed_webpage)
-            thumbnail_webpage = embed_webpage
-        else:
-            thumbnail_webpage = webpage
-
-        title = self._search_regex(
-            r'id="file_title".*?>\s*(.*?)\s*<(?:br|span)', webpage, 'title')
-        thumbnail = self._search_regex(
-            r'image:\s*"(http[^\"]+)",', thumbnail_webpage, 'thumbnail', fatal=False)
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/voicerepublic.py b/yt_dlp/extractor/voicerepublic.py
deleted file mode 100644
index 47502afb4..000000000
--- a/yt_dlp/extractor/voicerepublic.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    determine_ext,
-    int_or_none,
-    urljoin,
-)
-
-
-class VoiceRepublicIE(InfoExtractor):
-    _VALID_URL = r'https?://voicerepublic\.com/(?:talks|embed)/(?P<id>[0-9a-z-]+)'
-    _TESTS = [{
-        'url': 'http://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
-        'md5': 'b9174d651323f17783000876347116e3',
-        'info_dict': {
-            'id': '2296',
-            'display_id': 'watching-the-watchers-building-a-sousveillance-state',
-            'ext': 'm4a',
-            'title': 'Watching the Watchers: Building a Sousveillance State',
-            'description': 'Secret surveillance programs have metadata too. The people and companies that operate secret surveillance programs can be surveilled.',
-            'duration': 1556,
-            'view_count': int,
-        }
-    }, {
-        'url': 'http://voicerepublic.com/embed/watching-the-watchers-building-a-sousveillance-state',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        if '>Queued for processing, please stand by...<' in webpage:
-            raise ExtractorError(
-                'Audio is still queued for processing', expected=True)
-
-        talk = self._parse_json(self._search_regex(
-            r'initialSnapshot\s*=\s*({.+?});',
-            webpage, 'talk'), display_id)['talk']
-        title = talk['title']
-        formats = [{
-            'url': urljoin(url, talk_url),
-            'format_id': format_id,
-            'ext': determine_ext(talk_url) or format_id,
-            'vcodec': 'none',
-        } for format_id, talk_url in talk['media_links'].items()]
-
-        return {
-            'id': compat_str(talk.get('id') or display_id),
-            'display_id': display_id,
-            'title': title,
-            'description': talk.get('teaser'),
-            'thumbnail': talk.get('image_url'),
-            'duration': int_or_none(talk.get('archived_duration')),
-            'view_count': int_or_none(talk.get('play_count')),
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/voot.py b/yt_dlp/extractor/voot.py
index b19a27934..ef77bedd2 100644
--- a/yt_dlp/extractor/voot.py
+++ b/yt_dlp/extractor/voot.py
@@ -81,6 +81,7 @@ def _real_initialize(self):
 
 
 class VootIE(VootBaseIE):
+    _WORKING = False
     _VALID_URL = r'''(?x)
                     (?:
                         voot:|
@@ -169,6 +170,7 @@ def _real_extract(self, url):
 
 
 class VootSeriesIE(VootBaseIE):
+    _WORKING = False
     _VALID_URL = r'https?://(?:www\.)?voot\.com/shows/[^/]+/(?P<id>\d{3,})'
     _TESTS = [{
         'url': 'https://www.voot.com/shows/chakravartin-ashoka-samrat/100002',
diff --git a/yt_dlp/extractor/voxmedia.py b/yt_dlp/extractor/voxmedia.py
index f9362002f..f36908754 100644
--- a/yt_dlp/extractor/voxmedia.py
+++ b/yt_dlp/extractor/voxmedia.py
@@ -51,7 +51,7 @@ def _real_extract(self, url):
             info['duration'] = int_or_none(asset.get('duration'))
             return info
 
-        for provider_video_type in ('ooyala', 'youtube', 'brightcove'):
+        for provider_video_type in ('youtube', 'brightcove'):
             provider_video_id = video_data.get('%s_id' % provider_video_type)
             if not provider_video_id:
                 continue
@@ -177,7 +177,6 @@ def _real_extract(self, url):
         def create_entry(provider_video_id, provider_video_type, title=None, description=None):
             video_url = {
                 'youtube': '%s',
-                'ooyala': 'ooyala:%s',
                 'volume': 'http://volume.vox-cdn.com/embed/%s',
             }[provider_video_type] % provider_video_id
             return {
@@ -205,11 +204,6 @@ def create_entry(provider_video_id, provider_video_type, title=None, description
                         provider_video_id, provider_video_type,
                         video_data.get('title'), video_data.get('description')))
 
-        provider_video_id = self._search_regex(
-            r'data-ooyala-id="([^"]+)"', webpage, 'ooyala id', default=None)
-        if provider_video_id:
-            entries.append(create_entry(provider_video_id, 'ooyala'))
-
         volume_uuid = self._search_regex(
             r'data-volume-uuid="([^"]+)"', webpage, 'volume uuid', default=None)
         if volume_uuid:
diff --git a/yt_dlp/extractor/vrak.py b/yt_dlp/extractor/vrak.py
deleted file mode 100644
index 198c0a294..000000000
--- a/yt_dlp/extractor/vrak.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from .brightcove import BrightcoveNewIE
-from ..utils import (
-    int_or_none,
-    parse_age_limit,
-    smuggle_url,
-    unescapeHTML,
-)
-
-
-class VrakIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vrak\.tv/videos\?.*?\btarget=(?P<id>[\d.]+)'
-    _TEST = {
-        'url': 'http://www.vrak.tv/videos?target=1.2306782&filtre=emission&id=1.1806721',
-        'info_dict': {
-            'id': '5345661243001',
-            'ext': 'mp4',
-            'title': 'Obésité, film de hockey et Roseline Filion',
-            'timestamp': 1488492126,
-            'upload_date': '20170302',
-            'uploader_id': '2890187628001',
-            'creator': 'VRAK.TV',
-            'age_limit': 8,
-            'series': 'ALT (Actualité Légèrement Tordue)',
-            'episode': 'Obésité, film de hockey et Roseline Filion',
-            'tags': list,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/2890187628001/default_default/index.html?videoId=%s'
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(
-            r'<h\d\b[^>]+\bclass=["\']videoTitle["\'][^>]*>([^<]+)',
-            webpage, 'title', default=None) or self._og_search_title(webpage)
-
-        content = self._parse_json(
-            self._search_regex(
-                r'data-player-options-content=(["\'])(?P<content>{.+?})\1',
-                webpage, 'content', default='{}', group='content'),
-            video_id, transform_source=unescapeHTML)
-
-        ref_id = content.get('refId') or self._search_regex(
-            r'refId&quot;:&quot;([^&]+)&quot;', webpage, 'ref id')
-
-        brightcove_id = self._search_regex(
-            r'''(?x)
-                java\.lang\.String\s+value\s*=\s*["']brightcove\.article\.\d+\.%s
-                [^>]*
-                java\.lang\.String\s+value\s*=\s*["'](\d+)
-            ''' % re.escape(ref_id), webpage, 'brightcove id')
-
-        return {
-            '_type': 'url_transparent',
-            'ie_key': BrightcoveNewIE.ie_key(),
-            'url': smuggle_url(
-                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
-                {'geo_countries': ['CA']}),
-            'id': brightcove_id,
-            'description': content.get('description'),
-            'creator': content.get('brand'),
-            'age_limit': parse_age_limit(content.get('rating')),
-            'series': content.get('showName') or content.get(
-                'episodeName'),  # this is intentional
-            'season_number': int_or_none(content.get('seasonNumber')),
-            'episode': title,
-            'episode_number': int_or_none(content.get('episodeNumber')),
-            'tags': content.get('tags', []),
-        }
diff --git a/yt_dlp/extractor/vrv.py b/yt_dlp/extractor/vrv.py
deleted file mode 100644
index 523c442e6..000000000
--- a/yt_dlp/extractor/vrv.py
+++ /dev/null
@@ -1,269 +0,0 @@
-import base64
-import hashlib
-import hmac
-import json
-import random
-import string
-import time
-import urllib.parse
-
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
-from ..networking.exceptions import HTTPError
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    int_or_none,
-    join_nonempty,
-    traverse_obj,
-)
-
-
-class VRVBaseIE(InfoExtractor):
-    _API_DOMAIN = None
-    _API_PARAMS = {}
-    _CMS_SIGNING = {}
-    _TOKEN = None
-    _TOKEN_SECRET = ''
-
-    def _call_api(self, path, video_id, note, data=None):
-        # https://tools.ietf.org/html/rfc5849#section-3
-        base_url = self._API_DOMAIN + '/core/' + path
-        query = [
-            ('oauth_consumer_key', self._API_PARAMS['oAuthKey']),
-            ('oauth_nonce', ''.join(random.choices(string.ascii_letters, k=32))),
-            ('oauth_signature_method', 'HMAC-SHA1'),
-            ('oauth_timestamp', int(time.time())),
-        ]
-        if self._TOKEN:
-            query.append(('oauth_token', self._TOKEN))
-        encoded_query = compat_urllib_parse_urlencode(query)
-        headers = self.geo_verification_headers()
-        if data:
-            data = json.dumps(data).encode()
-            headers['Content-Type'] = 'application/json'
-        base_string = '&'.join([
-            'POST' if data else 'GET',
-            urllib.parse.quote(base_url, ''),
-            urllib.parse.quote(encoded_query, '')])
-        oauth_signature = base64.b64encode(hmac.new(
-            (self._API_PARAMS['oAuthSecret'] + '&' + self._TOKEN_SECRET).encode('ascii'),
-            base_string.encode(), hashlib.sha1).digest()).decode()
-        encoded_query += '&oauth_signature=' + urllib.parse.quote(oauth_signature, '')
-        try:
-            return self._download_json(
-                '?'.join([base_url, encoded_query]), video_id,
-                note='Downloading %s JSON metadata' % note, headers=headers, data=data)
-        except ExtractorError as e:
-            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
-                raise ExtractorError(json.loads(e.cause.response.read().decode())['message'], expected=True)
-            raise
-
-    def _call_cms(self, path, video_id, note):
-        if not self._CMS_SIGNING:
-            index = self._call_api('index', video_id, 'CMS Signing')
-            self._CMS_SIGNING = index.get('cms_signing') or {}
-            if not self._CMS_SIGNING:
-                for signing_policy in index.get('signing_policies', []):
-                    signing_path = signing_policy.get('path')
-                    if signing_path and signing_path.startswith('/cms/'):
-                        name, value = signing_policy.get('name'), signing_policy.get('value')
-                        if name and value:
-                            self._CMS_SIGNING[name] = value
-        return self._download_json(
-            self._API_DOMAIN + path, video_id, query=self._CMS_SIGNING,
-            note='Downloading %s JSON metadata' % note, headers=self.geo_verification_headers())
-
-    def _get_cms_resource(self, resource_key, video_id):
-        return self._call_api(
-            'cms_resource', video_id, 'resource path', data={
-                'resource_key': resource_key,
-            })['__links__']['cms_resource']['href']
-
-    def _extract_vrv_formats(self, url, video_id, stream_format, audio_lang, hardsub_lang):
-        if not url or stream_format not in ('hls', 'dash', 'adaptive_hls'):
-            return []
-        format_id = join_nonempty(
-            stream_format,
-            audio_lang and 'audio-%s' % audio_lang,
-            hardsub_lang and 'hardsub-%s' % hardsub_lang)
-        if 'hls' in stream_format:
-            adaptive_formats = self._extract_m3u8_formats(
-                url, video_id, 'mp4', m3u8_id=format_id,
-                note='Downloading %s information' % format_id,
-                fatal=False)
-        elif stream_format == 'dash':
-            adaptive_formats = self._extract_mpd_formats(
-                url, video_id, mpd_id=format_id,
-                note='Downloading %s information' % format_id,
-                fatal=False)
-        if audio_lang:
-            for f in adaptive_formats:
-                if f.get('acodec') != 'none':
-                    f['language'] = audio_lang
-        return adaptive_formats
-
-    def _set_api_params(self):
-        webpage = self._download_webpage(
-            'https://vrv.co/', None, headers=self.geo_verification_headers())
-        self._API_PARAMS = self._parse_json(self._search_regex(
-            [
-                r'window\.__APP_CONFIG__\s*=\s*({.+?})(?:</script>|;)',
-                r'window\.__APP_CONFIG__\s*=\s*({.+})'
-            ], webpage, 'app config'), None)['cxApiParams']
-        self._API_DOMAIN = self._API_PARAMS.get('apiDomain', 'https://api.vrv.co')
-
-
-class VRVIE(VRVBaseIE):
-    IE_NAME = 'vrv'
-    _VALID_URL = r'https?://(?:www\.)?vrv\.co/watch/(?P<id>[A-Z0-9]+)'
-    _TESTS = [{
-        'url': 'https://vrv.co/watch/GR9PNZ396/Hidden-America-with-Jonah-Ray:BOSTON-WHERE-THE-PAST-IS-THE-PRESENT',
-        'info_dict': {
-            'id': 'GR9PNZ396',
-            'ext': 'mp4',
-            'title': 'BOSTON: WHERE THE PAST IS THE PRESENT',
-            'description': 'md5:4ec8844ac262ca2df9e67c0983c6b83f',
-            'uploader_id': 'seeso',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        # movie listing
-        'url': 'https://vrv.co/watch/G6NQXZ1J6/Lily-CAT',
-        'info_dict': {
-            'id': 'G6NQXZ1J6',
-            'title': 'Lily C.A.T',
-            'description': 'md5:988b031e7809a6aeb60968be4af7db07',
-        },
-        'playlist_count': 2,
-    }]
-    _NETRC_MACHINE = 'vrv'
-
-    def _perform_login(self, username, password):
-        token_credentials = self._call_api(
-            'authenticate/by:credentials', None, 'Token Credentials', data={
-                'email': username,
-                'password': password,
-            })
-        self._TOKEN = token_credentials['oauth_token']
-        self._TOKEN_SECRET = token_credentials['oauth_token_secret']
-
-    def _initialize_pre_login(self):
-        return self._set_api_params()
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        object_data = self._call_cms(self._get_cms_resource(
-            'cms:/objects/' + video_id, video_id), video_id, 'object')['items'][0]
-        resource_path = object_data['__links__']['resource']['href']
-        video_data = self._call_cms(resource_path, video_id, 'video')
-        title = video_data['title']
-        description = video_data.get('description')
-
-        if video_data.get('__class__') == 'movie_listing':
-            items = self._call_cms(
-                video_data['__links__']['movie_listing/movies']['href'],
-                video_id, 'movie listing').get('items') or []
-            if len(items) != 1:
-                entries = []
-                for item in items:
-                    item_id = item.get('id')
-                    if not item_id:
-                        continue
-                    entries.append(self.url_result(
-                        'https://vrv.co/watch/' + item_id,
-                        self.ie_key(), item_id, item.get('title')))
-                return self.playlist_result(entries, video_id, title, description)
-            video_data = items[0]
-
-        streams_path = video_data['__links__'].get('streams', {}).get('href')
-        if not streams_path:
-            self.raise_login_required()
-        streams_json = self._call_cms(streams_path, video_id, 'streams')
-
-        audio_locale = streams_json.get('audio_locale')
-        formats = []
-        for stream_type, streams in streams_json.get('streams', {}).items():
-            if stream_type in ('adaptive_hls', 'adaptive_dash'):
-                for stream in streams.values():
-                    formats.extend(self._extract_vrv_formats(
-                        stream.get('url'), video_id, stream_type.split('_')[1],
-                        audio_locale, stream.get('hardsub_locale')))
-
-        subtitles = {}
-        for k in ('captions', 'subtitles'):
-            for subtitle in streams_json.get(k, {}).values():
-                subtitle_url = subtitle.get('url')
-                if not subtitle_url:
-                    continue
-                subtitles.setdefault(subtitle.get('locale', 'en-US'), []).append({
-                    'url': subtitle_url,
-                    'ext': subtitle.get('format', 'ass'),
-                })
-
-        thumbnails = []
-        for thumbnail in traverse_obj(video_data, ('images', 'thumbnail', ..., ...)) or []:
-            thumbnail_url = thumbnail.get('source')
-            if not thumbnail_url:
-                continue
-            thumbnails.append({
-                'url': thumbnail_url,
-                'width': int_or_none(thumbnail.get('width')),
-                'height': int_or_none(thumbnail.get('height')),
-            })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'subtitles': subtitles,
-            'thumbnails': thumbnails,
-            'description': description,
-            'duration': float_or_none(video_data.get('duration_ms'), 1000),
-            'uploader_id': video_data.get('channel_id'),
-            'series': video_data.get('series_title'),
-            'season': video_data.get('season_title'),
-            'season_number': int_or_none(video_data.get('season_number')),
-            'season_id': video_data.get('season_id'),
-            'episode': title,
-            'episode_number': int_or_none(video_data.get('episode_number')),
-            'episode_id': video_data.get('production_episode_id'),
-        }
-
-
-class VRVSeriesIE(VRVBaseIE):
-    IE_NAME = 'vrv:series'
-    _VALID_URL = r'https?://(?:www\.)?vrv\.co/series/(?P<id>[A-Z0-9]+)'
-    _TEST = {
-        'url': 'https://vrv.co/series/G68VXG3G6/The-Perfect-Insider',
-        'info_dict': {
-            'id': 'G68VXG3G6',
-        },
-        'playlist_mincount': 11,
-    }
-
-    def _initialize_pre_login(self):
-        return self._set_api_params()
-
-    def _real_extract(self, url):
-        series_id = self._match_id(url)
-
-        seasons_path = self._get_cms_resource(
-            'cms:/seasons?series_id=' + series_id, series_id)
-        seasons_data = self._call_cms(seasons_path, series_id, 'seasons')
-
-        entries = []
-        for season in seasons_data.get('items', []):
-            episodes_path = season['__links__']['season/episodes']['href']
-            episodes = self._call_cms(episodes_path, series_id, 'episodes')
-            for episode in episodes.get('items', []):
-                episode_id = episode['id']
-                entries.append(self.url_result(
-                    'https://vrv.co/watch/' + episode_id,
-                    'VRV', episode_id, episode.get('title')))
-
-        return self.playlist_result(entries, series_id)
diff --git a/yt_dlp/extractor/vshare.py b/yt_dlp/extractor/vshare.py
deleted file mode 100644
index 443ed43cc..000000000
--- a/yt_dlp/extractor/vshare.py
+++ /dev/null
@@ -1,57 +0,0 @@
-from .common import InfoExtractor
-from ..utils import ExtractorError, decode_packed_codes
-
-
-class VShareIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vshare\.io/[dv]/(?P<id>[^/?#&]+)'
-    _EMBED_REGEX = [r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?vshare\.io/v/[^/?#&]+)']
-    _TESTS = [{
-        'url': 'https://vshare.io/d/0f64ce6',
-        'md5': '17b39f55b5497ae8b59f5fbce8e35886',
-        'info_dict': {
-            'id': '0f64ce6',
-            'title': 'vl14062007715967',
-            'ext': 'mp4',
-        }
-    }, {
-        'url': 'https://vshare.io/v/0f64ce6/width-650/height-430/1',
-        'only_matching': True,
-    }]
-
-    def _extract_packed(self, webpage):
-        packed = self._search_regex(
-            r'(eval\(function.+)', webpage, 'packed code')
-        unpacked = decode_packed_codes(packed)
-        digits = self._search_regex(r'\[([\d,]+)\]', unpacked, 'digits')
-        digits = [int(digit) for digit in digits.split(',')]
-        key_digit = self._search_regex(
-            r'fromCharCode\(.+?(\d+)\)}', unpacked, 'key digit')
-        chars = [chr(d - int(key_digit)) for d in digits]
-        return ''.join(chars)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'https://vshare.io/v/%s/width-650/height-430/1' % video_id,
-            video_id, headers={'Referer': url})
-
-        title = self._html_extract_title(webpage)
-        title = title.split(' - ')[0]
-
-        error = self._html_search_regex(
-            r'(?s)<div[^>]+\bclass=["\']xxx-error[^>]+>(.+?)</div', webpage,
-            'error', default=None)
-        if error:
-            raise ExtractorError(error, expected=True)
-
-        info = self._parse_html5_media_entries(
-            url, '<video>%s</video>' % self._extract_packed(webpage),
-            video_id)[0]
-
-        info.update({
-            'id': video_id,
-            'title': title,
-        })
-
-        return info
diff --git a/yt_dlp/extractor/vupload.py b/yt_dlp/extractor/vupload.py
deleted file mode 100644
index 23ea70c77..000000000
--- a/yt_dlp/extractor/vupload.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    parse_duration,
-    parse_filesize,
-    extract_attributes,
-    int_or_none,
-    js_to_json
-)
-
-
-class VuploadIE(InfoExtractor):
-    _VALID_URL = r'https://vupload\.com/v/(?P<id>[a-z0-9]+)'
-    _TESTS = [{
-        'url': 'https://vupload.com/v/u28d0pl2tphy',
-        'md5': '9b42a4a193cca64d80248e58527d83c8',
-        'info_dict': {
-            'id': 'u28d0pl2tphy',
-            'ext': 'mp4',
-            'description': 'md5:e9e6c0045c78cbf0d5bb19a55ce199fb',
-            'title': 'md5:e9e6c0045c78cbf0d5bb19a55ce199fb',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_extract_title(webpage)
-        video_json = self._parse_json(self._html_search_regex(r'sources:\s*(.+?]),', webpage, 'video'), video_id, transform_source=js_to_json)
-        formats = []
-        for source in video_json:
-            if source['src'].endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(source['src'], video_id, m3u8_id='hls'))
-        duration = parse_duration(self._html_search_regex(
-            r'<i\s*class=["\']fad\s*fa-clock["\']></i>\s*([\d:]+)\s*</div>', webpage, 'duration', fatal=False))
-        filesize_approx = parse_filesize(self._html_search_regex(
-            r'<i\s*class=["\']fad\s*fa-save["\']></i>\s*([^<]+)\s*</div>', webpage, 'filesize', fatal=False))
-        extra_video_info = extract_attributes(self._html_search_regex(
-            r'(<video[^>]+>)', webpage, 'video_info', fatal=False))
-        description = self._html_search_meta('description', webpage)
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'duration': duration,
-            'filesize_approx': filesize_approx,
-            'width': int_or_none(extra_video_info.get('width')),
-            'height': int_or_none(extra_video_info.get('height')),
-            'format_id': extra_video_info.get('height', '') + 'p',
-            'title': title,
-            'description': description,
-        }
diff --git a/yt_dlp/extractor/vyborymos.py b/yt_dlp/extractor/vyborymos.py
deleted file mode 100644
index 386518795..000000000
--- a/yt_dlp/extractor/vyborymos.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_str
-
-
-class VyboryMosIE(InfoExtractor):
-    _VALID_URL = r'https?://vybory\.mos\.ru/(?:#precinct/|account/channels\?.*?\bstation_id=)(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://vybory.mos.ru/#precinct/13636',
-        'info_dict': {
-            'id': '13636',
-            'ext': 'mp4',
-            'title': 're:^Участковая избирательная комиссия №2231 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 'Россия, Москва, улица Введенского, 32А',
-            'is_live': True,
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://vybory.mos.ru/account/channels?station_id=13636',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        station_id = self._match_id(url)
-
-        channels = self._download_json(
-            'http://vybory.mos.ru/account/channels?station_id=%s' % station_id,
-            station_id, 'Downloading channels JSON')
-
-        formats = []
-        for cam_num, (sid, hosts, name, _) in enumerate(channels, 1):
-            for num, host in enumerate(hosts, 1):
-                formats.append({
-                    'url': 'http://%s/master.m3u8?sid=%s' % (host, sid),
-                    'ext': 'mp4',
-                    'format_id': 'camera%d-host%d' % (cam_num, num),
-                    'format_note': '%s, %s' % (name, host),
-                })
-
-        info = self._download_json(
-            'http://vybory.mos.ru/json/voting_stations/%s/%s.json'
-            % (compat_str(station_id)[:3], station_id),
-            station_id, 'Downloading station JSON', fatal=False) or {}
-
-        return {
-            'id': station_id,
-            'title': info.get('name') or station_id,
-            'description': info.get('address'),
-            'is_live': True,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/vzaar.py b/yt_dlp/extractor/vzaar.py
deleted file mode 100644
index 19908a929..000000000
--- a/yt_dlp/extractor/vzaar.py
+++ /dev/null
@@ -1,100 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    int_or_none,
-    float_or_none,
-    unified_timestamp,
-    url_or_none,
-)
-
-
-class VzaarIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|view)\.)?vzaar\.com/(?:videos/)?(?P<id>\d+)'
-    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?:)?//(?:view\.vzaar\.com)/[0-9]+)']
-    _TESTS = [{
-        # HTTP and HLS
-        'url': 'https://vzaar.com/videos/1152805',
-        'md5': 'bde5ddfeb104a6c56a93a06b04901dbf',
-        'info_dict': {
-            'id': '1152805',
-            'ext': 'mp4',
-            'title': 'sample video (public)',
-        },
-    }, {
-        'url': 'https://view.vzaar.com/27272/player',
-        'md5': '3b50012ac9bbce7f445550d54e0508f2',
-        'info_dict': {
-            'id': '27272',
-            'ext': 'mp3',
-            'title': 'MP3',
-        },
-    }, {
-        # hlsAes = true
-        'url': 'https://view.vzaar.com/11379930/player',
-        'info_dict': {
-            'id': '11379930',
-            'ext': 'mp4',
-            'title': 'Videoaula',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        # with null videoTitle
-        'url': 'https://view.vzaar.com/20313539/download',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        video_data = self._download_json(
-            'http://view.vzaar.com/v2/%s/video' % video_id, video_id)
-
-        title = video_data.get('videoTitle') or video_id
-
-        formats = []
-
-        source_url = url_or_none(video_data.get('sourceUrl'))
-        if source_url:
-            f = {
-                'url': source_url,
-                'format_id': 'http',
-                'quality': 1,
-            }
-            if 'audio' in source_url:
-                f.update({
-                    'vcodec': 'none',
-                    'ext': 'mp3',
-                })
-            else:
-                f.update({
-                    'width': int_or_none(video_data.get('width')),
-                    'height': int_or_none(video_data.get('height')),
-                    'ext': 'mp4',
-                    'fps': float_or_none(video_data.get('fps')),
-                })
-            formats.append(f)
-
-        video_guid = video_data.get('guid')
-        usp = video_data.get('usp')
-        if video_data.get('uspEnabled') and isinstance(video_guid, compat_str) and isinstance(usp, dict):
-            hls_aes = video_data.get('hlsAes')
-            qs = '&'.join('%s=%s' % (k, v) for k, v in usp.items())
-            url_templ = 'http://%%s.vzaar.com/v5/usp%s/%s/%s.ism%%s?' % ('aes' if hls_aes else '', video_guid, video_id)
-            m3u8_formats = self._extract_m3u8_formats(
-                url_templ % ('fable', '/.m3u8') + qs, video_id, 'mp4', 'm3u8_native',
-                m3u8_id='hls', fatal=False)
-            if hls_aes:
-                for f in m3u8_formats:
-                    f['hls_aes'] = {'uri': url_templ % ('goose', '') + qs}
-            formats.extend(m3u8_formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': self._proto_relative_url(video_data.get('poster')),
-            'duration': float_or_none(video_data.get('videoDuration')),
-            'timestamp': unified_timestamp(video_data.get('ts')),
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/wakanim.py b/yt_dlp/extractor/wakanim.py
deleted file mode 100644
index 155008f8c..000000000
--- a/yt_dlp/extractor/wakanim.py
+++ /dev/null
@@ -1,75 +0,0 @@
-from urllib.parse import unquote
-
-from .common import InfoExtractor
-from ..utils import (
-    merge_dicts,
-    urljoin,
-)
-
-
-class WakanimIE(InfoExtractor):
-    _VALID_URL = r'https://(?:www\.)?wakanim\.tv/[^/]+/v2/catalogue/episode/(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://www.wakanim.tv/de/v2/catalogue/episode/2997/the-asterisk-war-omu-staffel-1-episode-02-omu',
-        'info_dict': {
-            'id': '2997',
-            'ext': 'mp4',
-            'title': 'Episode 02',
-            'description': 'md5:2927701ea2f7e901de8bfa8d39b2852d',
-            'series': 'The Asterisk War  (OmU.)',
-            'season_number': 1,
-            'episode': 'Episode 02',
-            'episode_number': 2,
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        # DRM Protected
-        'url': 'https://www.wakanim.tv/de/v2/catalogue/episode/7843/sword-art-online-alicization-omu-arc-2-folge-15-omu',
-        'only_matching': True,
-    }]
-    _GEO_BYPASS = False
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        if 'Geoblocking' in webpage:
-            if '/de/' in url:
-                self.raise_geo_restricted(countries=['DE', 'AT', 'CH'])
-            else:
-                self.raise_geo_restricted(countries=['RU'])
-
-        manifest_url = urljoin(url, self._search_regex(
-            r'file\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage, 'manifest url',
-            group='url'))
-        if not self.get_param('allow_unplayable_formats'):
-            # https://docs.microsoft.com/en-us/azure/media-services/previous/media-services-content-protection-overview#streaming-urls
-            encryption = self._search_regex(
-                r'encryption%3D(c(?:enc|bc(?:s-aapl)?))',
-                manifest_url, 'encryption', default=None)
-            if encryption in ('cenc', 'cbcs-aapl'):
-                self.report_drm(video_id)
-
-        if 'format=mpd-time-cmaf' in unquote(manifest_url):
-            formats = self._extract_mpd_formats(
-                manifest_url, video_id, mpd_id='dash')
-        else:
-            formats = self._extract_m3u8_formats(
-                manifest_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                m3u8_id='hls')
-
-        info = self._search_json_ld(webpage, video_id, default={})
-
-        title = self._search_regex(
-            (r'<h1[^>]+\bclass=["\']episode_h1[^>]+\btitle=(["\'])(?P<title>(?:(?!\1).)+)\1',
-             r'<span[^>]+\bclass=["\']episode_title["\'][^>]*>(?P<title>[^<]+)'),
-            webpage, 'title', default=None, group='title')
-
-        return merge_dicts(info, {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-        })
diff --git a/yt_dlp/extractor/watchbox.py b/yt_dlp/extractor/watchbox.py
deleted file mode 100644
index c973ca998..000000000
--- a/yt_dlp/extractor/watchbox.py
+++ /dev/null
@@ -1,153 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    int_or_none,
-    js_to_json,
-    strip_or_none,
-    try_get,
-    unescapeHTML,
-    unified_timestamp,
-)
-
-
-class WatchBoxIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?watchbox\.de/(?P<kind>serien|filme)/(?:[^/]+/)*[^/]+-(?P<id>\d+)'
-    _TESTS = [{
-        # film
-        'url': 'https://www.watchbox.de/filme/free-jimmy-12325.html',
-        'info_dict': {
-            'id': '341368',
-            'ext': 'mp4',
-            'title': 'Free Jimmy',
-            'description': 'md5:bcd8bafbbf9dc0ef98063d344d7cc5f6',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 4890,
-            'age_limit': 16,
-            'release_year': 2009,
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'expected_warnings': ['Failed to download m3u8 information'],
-    }, {
-        # episode
-        'url': 'https://www.watchbox.de/serien/ugly-americans-12231/staffel-1/date-in-der-hoelle-328286.html',
-        'info_dict': {
-            'id': '328286',
-            'ext': 'mp4',
-            'title': 'S01 E01 - Date in der Hölle',
-            'description': 'md5:2f31c74a8186899f33cb5114491dae2b',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 1291,
-            'age_limit': 12,
-            'release_year': 2010,
-            'series': 'Ugly Americans',
-            'season_number': 1,
-            'episode': 'Date in der Hölle',
-            'episode_number': 1,
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'expected_warnings': ['Failed to download m3u8 information'],
-    }, {
-        'url': 'https://www.watchbox.de/serien/ugly-americans-12231/staffel-2/der-ring-des-powers-328270',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        kind, video_id = mobj.group('kind', 'id')
-
-        webpage = self._download_webpage(url, video_id)
-
-        player_config = self._parse_json(
-            self._search_regex(
-                r'data-player-conf=(["\'])(?P<data>{.+?})\1', webpage,
-                'player config', default='{}', group='data'),
-            video_id, transform_source=unescapeHTML, fatal=False)
-
-        if not player_config:
-            player_config = self._parse_json(
-                self._search_regex(
-                    r'playerConf\s*=\s*({.+?})\s*;', webpage, 'player config',
-                    default='{}'),
-                video_id, transform_source=js_to_json, fatal=False) or {}
-
-        source = player_config.get('source') or {}
-
-        video_id = compat_str(source.get('videoId') or video_id)
-
-        devapi = self._download_json(
-            'http://api.watchbox.de/devapi/id/%s' % video_id, video_id, query={
-                'format': 'json',
-                'apikey': 'hbbtv',
-            }, fatal=False)
-
-        item = try_get(devapi, lambda x: x['items'][0], dict) or {}
-
-        title = item.get('title') or try_get(
-            item, lambda x: x['movie']['headline_movie'],
-            compat_str) or source['title']
-
-        formats = []
-        hls_url = item.get('media_videourl_hls') or source.get('hls')
-        if hls_url:
-            formats.extend(self._extract_m3u8_formats(
-                hls_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                m3u8_id='hls', fatal=False))
-        dash_url = item.get('media_videourl_wv') or source.get('dash')
-        if dash_url:
-            formats.extend(self._extract_mpd_formats(
-                dash_url, video_id, mpd_id='dash', fatal=False))
-        mp4_url = item.get('media_videourl')
-        if mp4_url:
-            formats.append({
-                'url': mp4_url,
-                'format_id': 'mp4',
-                'width': int_or_none(item.get('width')),
-                'height': int_or_none(item.get('height')),
-                'tbr': int_or_none(item.get('bitrate')),
-            })
-
-        description = strip_or_none(item.get('descr'))
-        thumbnail = item.get('media_content_thumbnail_large') or source.get('poster') or item.get('media_thumbnail')
-        duration = int_or_none(item.get('media_length') or source.get('length'))
-        timestamp = unified_timestamp(item.get('pubDate'))
-        view_count = int_or_none(item.get('media_views'))
-        age_limit = int_or_none(try_get(item, lambda x: x['movie']['fsk']))
-        release_year = int_or_none(try_get(item, lambda x: x['movie']['rel_year']))
-
-        info = {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
-            'view_count': view_count,
-            'age_limit': age_limit,
-            'release_year': release_year,
-            'formats': formats,
-        }
-
-        if kind.lower() == 'serien':
-            series = try_get(
-                item, lambda x: x['special']['title'],
-                compat_str) or source.get('format')
-            season_number = int_or_none(self._search_regex(
-                r'^S(\d{1,2})\s*E\d{1,2}', title, 'season number',
-                default=None) or self._search_regex(
-                    r'/staffel-(\d+)/', url, 'season number', default=None))
-            episode = source.get('title')
-            episode_number = int_or_none(self._search_regex(
-                r'^S\d{1,2}\s*E(\d{1,2})', title, 'episode number',
-                default=None))
-            info.update({
-                'series': series,
-                'season_number': season_number,
-                'episode': episode,
-                'episode_number': episode_number,
-            })
-
-        return info
diff --git a/yt_dlp/extractor/watchindianporn.py b/yt_dlp/extractor/watchindianporn.py
deleted file mode 100644
index 3ded2d1d4..000000000
--- a/yt_dlp/extractor/watchindianporn.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..utils import parse_duration
-
-
-class WatchIndianPornIE(InfoExtractor):
-    IE_DESC = 'Watch Indian Porn'
-    _VALID_URL = r'https?://(?:www\.)?watchindianporn\.net/(?:[^/]+/)*video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
-    _TEST = {
-        'url': 'http://www.watchindianporn.net/video/hot-milf-from-kerala-shows-off-her-gorgeous-large-breasts-on-camera-RZa2avywNPa.html',
-        'md5': '249589a164dde236ec65832bfce17440',
-        'info_dict': {
-            'id': 'RZa2avywNPa',
-            'display_id': 'hot-milf-from-kerala-shows-off-her-gorgeous-large-breasts-on-camera',
-            'ext': 'mp4',
-            'title': 'Hot milf from kerala shows off her gorgeous large breasts on camera',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 226,
-            'view_count': int,
-            'categories': list,
-            'age_limit': 18,
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
-
-        title = self._html_search_regex((
-            r'<title>(.+?)\s*-\s*Indian\s+Porn</title>',
-            r'<h4>(.+?)</h4>'
-        ), webpage, 'title')
-
-        duration = parse_duration(self._search_regex(
-            r'Time:\s*<strong>\s*(.+?)\s*</strong>',
-            webpage, 'duration', fatal=False))
-
-        view_count = int(self._search_regex(
-            r'(?s)Time:\s*<strong>.*?</strong>.*?<strong>\s*(\d+)\s*</strong>',
-            webpage, 'view count', fatal=False))
-
-        categories = re.findall(
-            r'<a[^>]+class=[\'"]categories[\'"][^>]*>\s*([^<]+)\s*</a>',
-            webpage)
-
-        info_dict.update({
-            'id': video_id,
-            'display_id': display_id,
-            'http_headers': {
-                'Referer': url,
-            },
-            'title': title,
-            'duration': duration,
-            'view_count': view_count,
-            'categories': categories,
-            'age_limit': 18,
-        })
-
-        return info_dict
diff --git a/yt_dlp/extractor/willow.py b/yt_dlp/extractor/willow.py
deleted file mode 100644
index 0ec9c9d6e..000000000
--- a/yt_dlp/extractor/willow.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from ..utils import ExtractorError
-from .common import InfoExtractor
-
-
-class WillowIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?willow\.tv/videos/(?P<id>[0-9a-z-_]+)'
-    _GEO_COUNTRIES = ['US']
-
-    _TESTS = [{
-        'url': 'http://willow.tv/videos/d5winning-moment-eng-vs-ind-streaming-online-4th-test-india-tour-of-england-2021',
-        'info_dict': {
-            'id': '169662',
-            'display_id': 'd5winning-moment-eng-vs-ind-streaming-online-4th-test-india-tour-of-england-2021',
-            'ext': 'mp4',
-            'title': 'Winning Moment: 4th Test, England vs India',
-            'thumbnail': 'https://aimages.willow.tv/ytThumbnails/6748_D5winning_moment.jpg',
-            'duration': 233,
-            'timestamp': 1630947954,
-            'upload_date': '20210906',
-            'location': 'Kennington Oval, London',
-            'series': 'India tour of England 2021',
-        },
-        'params': {
-            'skip_download': True,  # AES-encrypted m3u8
-        },
-    }, {
-        'url': 'http://willow.tv/videos/highlights-short-ind-vs-nz-streaming-online-2nd-t20i-new-zealand-tour-of-india-2021',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_data = self._parse_json(self._html_search_regex(
-            r'var\s+data_js\s*=\s*JSON\.parse\(\'(.+)\'\)', webpage,
-            'data_js'), video_id)
-
-        video = next((v for v in video_data.get('trending_videos') or []
-                      if v.get('secureurl')), None)
-        if not video:
-            raise ExtractorError('No videos found')
-
-        formats = self._extract_m3u8_formats(video['secureurl'], video_id, 'mp4')
-
-        return {
-            'id': str(video.get('content_id')),
-            'display_id': video.get('video_slug'),
-            'title': video.get('video_name') or self._html_search_meta('twitter:title', webpage),
-            'formats': formats,
-            'thumbnail': video.get('yt_thumb_url') or self._html_search_meta(
-                'twitter:image', webpage, default=None),
-            'duration': video.get('duration_seconds'),
-            'timestamp': video.get('created_date'),
-            'location': video.get('venue'),
-            'series': video.get('series_name'),
-        }
diff --git a/yt_dlp/extractor/xbef.py b/yt_dlp/extractor/xbef.py
deleted file mode 100644
index ac69528a3..000000000
--- a/yt_dlp/extractor/xbef.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
-
-
-class XBefIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?xbef\.com/video/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://xbef.com/video/5119-glamourous-lesbians-smoking-drinking-and-fucking',
-        'md5': 'a478b565baff61634a98f5e5338be995',
-        'info_dict': {
-            'id': '5119',
-            'ext': 'mp4',
-            'title': 'md5:7358a9faef8b7b57acda7c04816f170e',
-            'age_limit': 18,
-            'thumbnail': r're:^http://.*\.jpg',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(
-            r'<h1[^>]*>(.*?)</h1>', webpage, 'title')
-
-        config_url_enc = self._download_webpage(
-            'http://xbef.com/Main/GetVideoURLEncoded/%s' % video_id, video_id,
-            note='Retrieving config URL')
-        config_url = compat_urllib_parse_unquote(config_url_enc)
-        config = self._download_xml(
-            config_url, video_id, note='Retrieving config')
-
-        video_url = config.find('./file').text
-        thumbnail = config.find('./image').text
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-            'age_limit': 18,
-        }
diff --git a/yt_dlp/extractor/xtube.py b/yt_dlp/extractor/xtube.py
deleted file mode 100644
index db8292589..000000000
--- a/yt_dlp/extractor/xtube.py
+++ /dev/null
@@ -1,214 +0,0 @@
-import itertools
-import re
-
-from .common import InfoExtractor
-from ..networking import Request
-from ..utils import (
-    int_or_none,
-    js_to_json,
-    orderedSet,
-    parse_duration,
-    str_to_int,
-    url_or_none,
-)
-
-
-class XTubeIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                        (?:
-                            xtube:|
-                            https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?:embedded/)?(?P<display_id>[^/]+)-)
-                        )
-                        (?P<id>[^/?&#]+)
-                    '''
-
-    _TESTS = [{
-        # old URL schema
-        'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
-        'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
-        'info_dict': {
-            'id': 'kVTUy_G222_',
-            'ext': 'mp4',
-            'title': 'strange erotica',
-            'description': 'contains:an ET kind of thing',
-            'uploader': 'greenshowers',
-            'duration': 450,
-            'view_count': int,
-            'comment_count': int,
-            'age_limit': 18,
-        }
-    }, {
-        # new URL schema
-        'url': 'http://www.xtube.com/video-watch/strange-erotica-625837',
-        'only_matching': True,
-    }, {
-        'url': 'xtube:625837',
-        'only_matching': True,
-    }, {
-        'url': 'xtube:kVTUy_G222_',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.xtube.com/video-watch/embedded/milf-tara-and-teen-shared-and-cum-covered-extreme-bukkake-32203482?embedsize=big',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-
-        if not display_id:
-            display_id = video_id
-
-        if video_id.isdigit() and len(video_id) < 11:
-            url_pattern = 'http://www.xtube.com/video-watch/-%s'
-        else:
-            url_pattern = 'http://www.xtube.com/watch.php?v=%s'
-
-        webpage = self._download_webpage(
-            url_pattern % video_id, display_id, headers={
-                'Cookie': 'age_verified=1; cookiesAccepted=1',
-            })
-
-        title, thumbnail, duration, sources, media_definition = [None] * 5
-
-        config = self._parse_json(self._search_regex(
-            r'playerConf\s*=\s*({.+?})\s*,\s*(?:\n|loaderConf|playerWrapper)', webpage, 'config',
-            default='{}'), video_id, transform_source=js_to_json, fatal=False)
-        if config:
-            config = config.get('mainRoll')
-            if isinstance(config, dict):
-                title = config.get('title')
-                thumbnail = config.get('poster')
-                duration = int_or_none(config.get('duration'))
-                sources = config.get('sources') or config.get('format')
-                media_definition = config.get('mediaDefinition')
-
-        if not isinstance(sources, dict) and not media_definition:
-            sources = self._parse_json(self._search_regex(
-                r'(["\'])?sources\1?\s*:\s*(?P<sources>{.+?}),',
-                webpage, 'sources', group='sources'), video_id,
-                transform_source=js_to_json)
-
-        formats = []
-        format_urls = set()
-
-        if isinstance(sources, dict):
-            for format_id, format_url in sources.items():
-                format_url = url_or_none(format_url)
-                if not format_url:
-                    continue
-                if format_url in format_urls:
-                    continue
-                format_urls.add(format_url)
-                formats.append({
-                    'url': format_url,
-                    'format_id': format_id,
-                    'height': int_or_none(format_id),
-                })
-
-        if isinstance(media_definition, list):
-            for media in media_definition:
-                video_url = url_or_none(media.get('videoUrl'))
-                if not video_url:
-                    continue
-                if video_url in format_urls:
-                    continue
-                format_urls.add(video_url)
-                format_id = media.get('format')
-                if format_id == 'hls':
-                    formats.extend(self._extract_m3u8_formats(
-                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                        m3u8_id='hls', fatal=False))
-                elif format_id == 'mp4':
-                    height = int_or_none(media.get('quality'))
-                    formats.append({
-                        'url': video_url,
-                        'format_id': '%s-%d' % (format_id, height) if height else format_id,
-                        'height': height,
-                    })
-
-        self._remove_duplicate_formats(formats)
-
-        if not title:
-            title = self._search_regex(
-                (r'<h1>\s*(?P<title>[^<]+?)\s*</h1>', r'videoTitle\s*:\s*(["\'])(?P<title>.+?)\1'),
-                webpage, 'title', group='title')
-        description = self._og_search_description(
-            webpage, default=None) or self._html_search_meta(
-            'twitter:description', webpage, default=None) or self._search_regex(
-            r'</h1>\s*<p>([^<]+)', webpage, 'description', fatal=False)
-        uploader = self._search_regex(
-            (r'<input[^>]+name="contentOwnerId"[^>]+value="([^"]+)"',
-             r'<span[^>]+class="nickname"[^>]*>([^<]+)'),
-            webpage, 'uploader', fatal=False)
-        if not duration:
-            duration = parse_duration(self._search_regex(
-                r'<dt>Runtime:?</dt>\s*<dd>([^<]+)</dd>',
-                webpage, 'duration', fatal=False))
-        view_count = str_to_int(self._search_regex(
-            (r'["\']viewsCount["\'][^>]*>(\d+)\s+views',
-             r'<dt>Views:?</dt>\s*<dd>([\d,\.]+)</dd>'),
-            webpage, 'view count', fatal=False))
-        comment_count = str_to_int(self._html_search_regex(
-            r'>Comments? \(([\d,\.]+)\)<',
-            webpage, 'comment count', fatal=False))
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'duration': duration,
-            'view_count': view_count,
-            'comment_count': comment_count,
-            'age_limit': 18,
-            'formats': formats,
-        }
-
-
-class XTubeUserIE(InfoExtractor):
-    IE_DESC = 'XTube user profile'
-    _VALID_URL = r'https?://(?:www\.)?xtube\.com/profile/(?P<id>[^/]+-\d+)'
-    _TEST = {
-        'url': 'http://www.xtube.com/profile/greenshowers-4056496',
-        'info_dict': {
-            'id': 'greenshowers-4056496',
-            'age_limit': 18,
-        },
-        'playlist_mincount': 154,
-    }
-
-    def _real_extract(self, url):
-        user_id = self._match_id(url)
-
-        entries = []
-        for pagenum in itertools.count(1):
-            request = Request(
-                'http://www.xtube.com/profile/%s/videos/%d' % (user_id, pagenum),
-                headers={
-                    'Cookie': 'popunder=4',
-                    'X-Requested-With': 'XMLHttpRequest',
-                    'Referer': url,
-                })
-
-            page = self._download_json(
-                request, user_id, 'Downloading videos JSON page %d' % pagenum)
-
-            html = page.get('html')
-            if not html:
-                break
-
-            for video_id in orderedSet([video_id for _, video_id in re.findall(
-                    r'data-plid=(["\'])(.+?)\1', html)]):
-                entries.append(self.url_result('xtube:%s' % video_id, XTubeIE.ie_key()))
-
-            page_count = int_or_none(page.get('pageCount'))
-            if not page_count or pagenum == page_count:
-                break
-
-        playlist = self.playlist_result(entries, user_id)
-        playlist['age_limit'] = 18
-        return playlist
diff --git a/yt_dlp/extractor/xuite.py b/yt_dlp/extractor/xuite.py
deleted file mode 100644
index 71ddadd42..000000000
--- a/yt_dlp/extractor/xuite.py
+++ /dev/null
@@ -1,149 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    get_element_by_attribute,
-    parse_iso8601,
-    remove_end,
-)
-
-
-class XuiteIE(InfoExtractor):
-    IE_DESC = '隨意窩Xuite影音'
-    _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
-    _VALID_URL = r'https?://vlog\.xuite\.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
-    _TESTS = [{
-        # Audio
-        'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
-        'md5': 'e79284c87b371424885448d11f6398c8',
-        'info_dict': {
-            'id': '3860914',
-            'ext': 'mp3',
-            'title': '孤單南半球-歐德陽',
-            'description': '孤單南半球-歐德陽',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 247.246,
-            'timestamp': 1314932940,
-            'upload_date': '20110902',
-            'uploader': '阿能',
-            'uploader_id': '15973816',
-            'categories': ['個人短片'],
-        },
-    }, {
-        # Video with only one format
-        'url': 'http://vlog.xuite.net/play/WUxxR2xCLTI1OTI1MDk5LmZsdg==',
-        'md5': '21f7b39c009b5a4615b4463df6eb7a46',
-        'info_dict': {
-            'id': '25925099',
-            'ext': 'mp4',
-            'title': 'BigBuckBunny_320x180',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 596.458,
-            'timestamp': 1454242500,
-            'upload_date': '20160131',
-            'uploader': '屁姥',
-            'uploader_id': '12158353',
-            'categories': ['個人短片'],
-            'description': 'http://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
-        },
-    }, {
-        # Video with two formats
-        'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==',
-        'md5': '1166e0f461efe55b62e26a2d2a68e6de',
-        'info_dict': {
-            'id': '21301170',
-            'ext': 'mp4',
-            'title': '暗殺教室 02',
-            'description': '字幕:【極影字幕社】',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 1384.907,
-            'timestamp': 1421481240,
-            'upload_date': '20150117',
-            'uploader': '我只是想認真點',
-            'uploader_id': '242127761',
-            'categories': ['電玩動漫'],
-        },
-        'skip': 'Video removed',
-    }, {
-        # Video with encoded media id
-        # from http://forgetfulbc.blogspot.com/2016/06/date.html
-        'url': 'http://vlog.xuite.net/embed/cE1xbENoLTI3NDQ3MzM2LmZsdg==?ar=0&as=0',
-        'info_dict': {
-            'id': '27447336',
-            'ext': 'mp4',
-            'title': '男女平權只是口號？專家解釋約會時男生是否該幫女生付錢 (中字)',
-            'description': 'md5:1223810fa123b179083a3aed53574706',
-            'timestamp': 1466160960,
-            'upload_date': '20160617',
-            'uploader': 'B.C. & Lowy',
-            'uploader_id': '232279340',
-        },
-    }, {
-        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        # /play/ URLs provide embedded video URL and more metadata
-        url = url.replace('/embed/', '/play/')
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        error_msg = self._search_regex(
-            r'<div id="error-message-content">([^<]+)',
-            webpage, 'error message', default=None)
-        if error_msg:
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error_msg),
-                expected=True)
-
-        media_info = self._parse_json(self._search_regex(
-            r'var\s+mediaInfo\s*=\s*({.*});', webpage, 'media info'), video_id)
-
-        video_id = media_info['MEDIA_ID']
-
-        formats = []
-        for key in ('html5Url', 'html5HQUrl'):
-            video_url = media_info.get(key)
-            if not video_url:
-                continue
-            format_id = self._search_regex(
-                r'\bq=(.+?)\b', video_url, 'format id', default=None)
-            formats.append({
-                'url': video_url,
-                'ext': 'mp4' if format_id.isnumeric() else format_id,
-                'format_id': format_id,
-                'height': int(format_id) if format_id.isnumeric() else None,
-            })
-
-        timestamp = media_info.get('PUBLISH_DATETIME')
-        if timestamp:
-            timestamp = parse_iso8601(timestamp + ' +0800', ' ')
-
-        category = media_info.get('catName')
-        categories = [category] if category else []
-
-        uploader = media_info.get('NICKNAME')
-        uploader_url = None
-
-        author_div = get_element_by_attribute('itemprop', 'author', webpage)
-        if author_div:
-            uploader = uploader or self._html_search_meta('name', author_div)
-            uploader_url = self._html_search_regex(
-                r'<link[^>]+itemprop="url"[^>]+href="([^"]+)"', author_div,
-                'uploader URL', fatal=False)
-
-        return {
-            'id': video_id,
-            'title': media_info['TITLE'],
-            'description': remove_end(media_info.get('metaDesc'), ' (Xuite 影音)'),
-            'thumbnail': media_info.get('ogImageUrl'),
-            'timestamp': timestamp,
-            'uploader': uploader,
-            'uploader_id': media_info.get('MEMBER_ID'),
-            'uploader_url': uploader_url,
-            'duration': float_or_none(media_info.get('MEDIA_DURATION'), 1000000),
-            'categories': categories,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/yesjapan.py b/yt_dlp/extractor/yesjapan.py
deleted file mode 100644
index 94e41660d..000000000
--- a/yt_dlp/extractor/yesjapan.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from .common import InfoExtractor
-from ..networking import HEADRequest
-from ..utils import get_element_by_attribute, parse_iso8601
-
-
-class YesJapanIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?yesjapan\.com/video/(?P<slug>[A-Za-z0-9\-]*)_(?P<id>[A-Za-z0-9]+)\.html'
-    _TEST = {
-        'url': 'http://www.yesjapan.com/video/japanese-in-5-20-wa-and-ga-particle-usages_726497834.html',
-        'md5': 'f0be416314e5be21a12b499b330c21cf',
-        'info_dict': {
-            'id': '726497834',
-            'title': 'Japanese in 5! #20 - WA And GA Particle Usages',
-            'description': 'This should clear up some issues most students of Japanese encounter with WA and GA....',
-            'ext': 'mp4',
-            'timestamp': 1416391590,
-            'upload_date': '20141119',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        title = self._og_search_title(webpage)
-        video_url = self._og_search_video_url(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-
-        timestamp = None
-        submit_info = get_element_by_attribute('class', 'pm-submit-data', webpage)
-        if submit_info:
-            timestamp = parse_iso8601(self._search_regex(
-                r'datetime="([^"]+)"', submit_info, 'upload date', fatal=False, default=None))
-
-        # attempt to resolve the final URL in order to get a proper extension
-        redirect_req = HEADRequest(video_url)
-        req = self._request_webpage(
-            redirect_req, video_id, note='Resolving final URL', errnote='Could not resolve final URL', fatal=False)
-        if req:
-            video_url = req.url
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'description': description,
-            'timestamp': timestamp,
-            'thumbnail': thumbnail,
-        }
diff --git a/yt_dlp/extractor/yinyuetai.py b/yt_dlp/extractor/yinyuetai.py
deleted file mode 100644
index b2e3172f9..000000000
--- a/yt_dlp/extractor/yinyuetai.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from .common import InfoExtractor
-from ..utils import ExtractorError
-
-
-class YinYueTaiIE(InfoExtractor):
-    IE_NAME = 'yinyuetai:video'
-    IE_DESC = '音悦Tai'
-    _VALID_URL = r'https?://v\.yinyuetai\.com/video(?:/h5)?/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://v.yinyuetai.com/video/2322376',
-        'md5': '6e3abe28d38e3a54b591f9f040595ce0',
-        'info_dict': {
-            'id': '2322376',
-            'ext': 'mp4',
-            'title': '少女时代_PARTY_Music Video Teaser',
-            'creator': '少女时代',
-            'duration': 25,
-            'thumbnail': r're:^https?://.*\.jpg$',
-        },
-    }, {
-        'url': 'http://v.yinyuetai.com/video/h5/2322376',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        info = self._download_json(
-            'http://ext.yinyuetai.com/main/get-h-mv-info?json=true&videoId=%s' % video_id, video_id,
-            'Downloading mv info')['videoInfo']['coreVideoInfo']
-
-        if info['error']:
-            raise ExtractorError(info['errorMsg'], expected=True)
-
-        formats = [{
-            'url': format_info['videoUrl'],
-            'format_id': format_info['qualityLevel'],
-            'format': format_info.get('qualityLevelName'),
-            'filesize': format_info.get('fileSize'),
-            # though URLs ends with .flv, the downloaded files are in fact mp4
-            'ext': 'mp4',
-            'tbr': format_info.get('bitrate'),
-        } for format_info in info['videoUrlModels']]
-
-        return {
-            'id': video_id,
-            'title': info['videoName'],
-            'thumbnail': info.get('bigHeadImage'),
-            'creator': info.get('artistNames'),
-            'duration': info.get('duration'),
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/ynet.py b/yt_dlp/extractor/ynet.py
deleted file mode 100644
index a7d7371f3..000000000
--- a/yt_dlp/extractor/ynet.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import json
-import re
-import urllib.parse
-
-from .common import InfoExtractor
-
-
-class YnetIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?ynet\.co\.il/(?:.+?/)?0,7340,(?P<id>L(?:-[0-9]+)+),00\.html'
-    _TESTS = [
-        {
-            'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',
-            'info_dict': {
-                'id': 'L-11659-99244',
-                'ext': 'flv',
-                'title': 'איש לא יודע מאיפה באנו',
-                'thumbnail': r're:^https?://.*\.jpg',
-            }
-        }, {
-            'url': 'http://hot.ynet.co.il/home/0,7340,L-8859-84418,00.html',
-            'info_dict': {
-                'id': 'L-8859-84418',
-                'ext': 'flv',
-                'title': "צפו: הנשיקה הלוהטת של תורגי' ויוליה פלוטקין",
-                'thumbnail': r're:^https?://.*\.jpg',
-            }
-        }
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        content = urllib.parse.unquote_plus(self._og_search_video_url(webpage))
-        config = json.loads(self._search_regex(r'config=({.+?})$', content, 'video config'))
-        f4m_url = config['clip']['url']
-        title = self._og_search_title(webpage)
-        m = re.search(r'ynet - HOT -- (["\']+)(?P<title>.+?)\1', title)
-        if m:
-            title = m.group('title')
-        formats = self._extract_f4m_formats(f4m_url, video_id)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': self._og_search_thumbnail(webpage),
-        }

From 15f22b4880b6b3f71f350c64d70976ae65b9f1ca Mon Sep 17 00:00:00 2001
From: TSRBerry <20988865+TSRBerry@users.noreply.github.com>
Date: Wed, 29 Nov 2023 00:20:06 +0100
Subject: [PATCH 720/871] [webvtt] Allow spaces before newlines for CueBlock
 (#7681)

Closes #7453

Ref: https://www.w3.org/TR/webvtt1/#webvtt-cue-block
---
 yt_dlp/webvtt.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/yt_dlp/webvtt.py b/yt_dlp/webvtt.py
index dd7298277..596e9bf73 100644
--- a/yt_dlp/webvtt.py
+++ b/yt_dlp/webvtt.py
@@ -95,6 +95,7 @@ def __init__(self, parser):
 _REGEX_EOF = re.compile(r'\Z')
 _REGEX_NL = re.compile(r'(?:\r\n|[\r\n]|$)')
 _REGEX_BLANK = re.compile(r'(?:\r\n|[\r\n])+')
+_REGEX_OPTIONAL_WHITESPACE = re.compile(r'[ \t]*')
 
 
 def _parse_ts(ts):
@@ -285,6 +286,7 @@ def parse(cls, parser):
         m1 = parser.consume(_REGEX_TS)
         if not m1:
             return None
+        parser.consume(_REGEX_OPTIONAL_WHITESPACE)
         m2 = parser.consume(cls._REGEX_SETTINGS)
         if not parser.consume(_REGEX_NL):
             return None

From a174c453ee1e853c584ceadeac17eef2bd433dc5 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 29 Nov 2023 03:18:17 +0530
Subject: [PATCH 721/871] Let `read_stdin` obey `--quiet`

Closes #8668
---
 yt_dlp/__init__.py     | 10 ++++++----
 yt_dlp/utils/_utils.py |  5 +++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 446f0c47b..57a487157 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -73,14 +73,16 @@ def _exit(status=0, *args):
 
 
 def get_urls(urls, batchfile, verbose):
-    # Batch file verification
+    """
+    @param verbose      -1: quiet, 0: normal, 1: verbose
+    """
     batch_urls = []
     if batchfile is not None:
         try:
             batch_urls = read_batch_urls(
-                read_stdin('URLs') if batchfile == '-'
+                read_stdin(None if verbose == -1 else 'URLs') if batchfile == '-'
                 else open(expand_path(batchfile), encoding='utf-8', errors='ignore'))
-            if verbose:
+            if verbose == 1:
                 write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
         except OSError:
             _exit(f'ERROR: batch file {batchfile} could not be read')
@@ -721,7 +723,7 @@ def get_postprocessors(opts):
 def parse_options(argv=None):
     """@returns ParsedOptions(parser, opts, urls, ydl_opts)"""
     parser, opts, urls = parseOpts(argv)
-    urls = get_urls(urls, opts.batchfile, opts.verbose)
+    urls = get_urls(urls, opts.batchfile, -1 if opts.quiet and not opts.verbose else opts.verbose)
 
     set_compat_opts(opts)
     try:
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index b0164a895..f1d7cead6 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -4789,8 +4789,9 @@ def parse_http_range(range):
 
 
 def read_stdin(what):
-    eof = 'Ctrl+Z' if compat_os_name == 'nt' else 'Ctrl+D'
-    write_string(f'Reading {what} from STDIN - EOF ({eof}) to end:\n')
+    if what:
+        eof = 'Ctrl+Z' if compat_os_name == 'nt' else 'Ctrl+D'
+        write_string(f'Reading {what} from STDIN - EOF ({eof}) to end:\n')
     return sys.stdin
 
 

From 6a9c7a2b52655bacfa7ab2da24fd0d14a6fff495 Mon Sep 17 00:00:00 2001
From: OIRNOIR <70721372+OIRNOIR@users.noreply.github.com>
Date: Wed, 29 Nov 2023 04:48:58 -0800
Subject: [PATCH 722/871] [ie/youtube] Support cf.piped.video (#8514)

Authored by: OIRNOIR
Closes #8457
---
 yt_dlp/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 449d9d1a5..b6212646e 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -428,7 +428,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         r'(?:www\.)?piped\.adminforge\.de',
         r'(?:www\.)?watch\.whatevertinfoil\.de',
         r'(?:www\.)?piped\.qdi\.fi',
-        r'(?:www\.)?piped\.video',
+        r'(?:(?:www|cf)\.)?piped\.video',
         r'(?:www\.)?piped\.aeong\.one',
         r'(?:www\.)?piped\.moomoo\.me',
         r'(?:www\.)?piped\.chauvet\.pro',

From 993edd3f6e17e966c763bc86dc34125445cec6b6 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 6 Dec 2023 03:44:11 +0530
Subject: [PATCH 723/871] [outtmpl] Support multiplication

Related: #8683
---
 README.md              | 2 +-
 test/test_YoutubeDL.py | 1 +
 yt_dlp/YoutubeDL.py    | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f67cab572..78d4799a1 100644
--- a/README.md
+++ b/README.md
@@ -1268,7 +1268,7 @@ # OUTPUT TEMPLATE
 
 1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a dot `.` separator; e.g. `%(tags.0)s`, `%(subtitles.en.-1.ext)s`. You can do Python slicing with colon `:`; E.g. `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. Curly braces `{}` can be used to build dictionaries with only specific keys; e.g. `%(formats.:.{format_id,height})#j`. An empty field name `%()s` refers to the entire infodict; e.g. `%(.{id,title})s`. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
 
-1. **Addition**: Addition and subtraction of numeric fields can be done using `+` and `-` respectively. E.g. `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
+1. **Arithmetic**: Simple arithmetic can be done on numeric fields using `+`, `-` and `*`. E.g. `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
 
 1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. E.g. `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s`
 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 0cf130db0..48c710e00 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -797,6 +797,7 @@ def expect_same_infodict(out):
         test('%(title|%)s %(title|%%)s', '% %%')
         test('%(id+1-height+3)05d', '00158')
         test('%(width+100)05d', 'NA')
+        test('%(filesize*8)d', '8192')
         test('%(formats.0) 15s', ('% 15s' % FORMATS[0], None))
         test('%(formats.0)r', (repr(FORMATS[0]), None))
         test('%(height.0)03d', '001')
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index e65bef862..29dd76186 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1179,6 +1179,7 @@ def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False):
         MATH_FUNCTIONS = {
             '+': float.__add__,
             '-': float.__sub__,
+            '*': float.__mul__,
         }
         # Field is of the form key1.key2...
         # where keys (except first) can be string, int, slice or "{field, ...}"

From 044886c220620a7679109e92352890e18b6079e3 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 6 Dec 2023 03:31:45 +0530
Subject: [PATCH 724/871] [ie/youtube] Return empty playlist when channel/tab
 has no videos

Closes #8634
---
 yt_dlp/extractor/youtube.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index b6212646e..86818a9dc 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -6469,6 +6469,9 @@ def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
     def _has_tab(self, tabs, tab_id):
         return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
 
+    def _empty_playlist(self, item_id, data):
+        return self.playlist_result([], item_id, **self._extract_metadata_from_tabs(item_id, data))
+
     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
     def _real_extract(self, url, smuggled_data):
         item_id = self._match_id(url)
@@ -6534,6 +6537,10 @@ def _real_extract(self, url, smuggled_data):
             selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url)  # NB: Name may be translated
             self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
 
+            # /about is no longer a tab
+            if original_tab_id == 'about':
+                return self._empty_playlist(item_id, data)
+
             if not original_tab_id and selected_tab_name:
                 self.to_screen('Downloading all uploads of the channel. '
                                'To download only the videos in a specific tab, pass the tab\'s URL')
@@ -6546,7 +6553,7 @@ def _real_extract(self, url, smuggled_data):
                 if not extra_tabs and selected_tab_id != 'videos':
                     # Channel does not have streams, shorts or videos tabs
                     if item_id[:2] != 'UC':
-                        raise ExtractorError('This channel has no uploads', expected=True)
+                        return self._empty_playlist(item_id, data)
 
                     # Topic channels don't have /videos. Use the equivalent playlist instead
                     pl_id = f'UU{item_id[2:]}'
@@ -6554,7 +6561,7 @@ def _real_extract(self, url, smuggled_data):
                     try:
                         data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
                     except ExtractorError:
-                        raise ExtractorError('This channel has no uploads', expected=True)
+                        return self._empty_playlist(item_id, data)
                     else:
                         item_id, url = pl_id, pl_url
                         self.to_screen(

From 71f28097fec1c9e029f74b68a4eadc8915399840 Mon Sep 17 00:00:00 2001
From: Pierrick Guillaume <34305318+Fymyte@users.noreply.github.com>
Date: Wed, 6 Dec 2023 16:10:11 +0100
Subject: [PATCH 725/871] [ie/francetv] Improve metadata extraction (#8409)

Authored by: Fymyte
---
 yt_dlp/extractor/francetv.py | 41 ++++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/yt_dlp/extractor/francetv.py b/yt_dlp/extractor/francetv.py
index 052317204..0ceecde74 100644
--- a/yt_dlp/extractor/francetv.py
+++ b/yt_dlp/extractor/francetv.py
@@ -1,12 +1,14 @@
 from .common import InfoExtractor
+from .dailymotion import DailymotionIE
 from ..utils import (
-    determine_ext,
     ExtractorError,
+    determine_ext,
     format_field,
+    int_or_none,
+    join_nonempty,
     parse_iso8601,
     parse_qs,
 )
-from .dailymotion import DailymotionIE
 
 
 class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -82,6 +84,8 @@ def _extract_video(self, video_id, catalogue=None):
         videos = []
         title = None
         subtitle = None
+        episode_number = None
+        season_number = None
         image = None
         duration = None
         timestamp = None
@@ -112,7 +116,9 @@ def _extract_video(self, video_id, catalogue=None):
             if meta:
                 if title is None:
                     title = meta.get('title')
-                # XXX: what is meta['pre_title']?
+                # meta['pre_title'] contains season and episode number for series in format "S<ID> E<ID>"
+                season_number, episode_number = self._search_regex(
+                    r'S(\d+)\s*E(\d+)', meta.get('pre_title'), 'episode info', group=(1, 2), default=(None, None))
                 if subtitle is None:
                     subtitle = meta.get('additional_title')
                 if image is None:
@@ -191,19 +197,19 @@ def _extract_video(self, video_id, catalogue=None):
                 } for sheet in spritesheets]
             })
 
-        if subtitle:
-            title += ' - %s' % subtitle
-        title = title.strip()
-
         return {
             'id': video_id,
-            'title': title,
+            'title': join_nonempty(title, subtitle, delim=' - ').strip(),
             'thumbnail': image,
             'duration': duration,
             'timestamp': timestamp,
             'is_live': is_live,
             'formats': formats,
             'subtitles': subtitles,
+            'episode': subtitle if episode_number else None,
+            'series': title if episode_number else None,
+            'episode_number': int_or_none(episode_number),
+            'season_number': int_or_none(season_number),
         }
 
     def _real_extract(self, url):
@@ -230,14 +236,31 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
             'id': 'ec217ecc-0733-48cf-ac06-af1347b849d1',
             'ext': 'mp4',
             'title': '13h15, le dimanche... - Les mystères de Jésus',
-            'description': 'md5:75efe8d4c0a8205e5904498ffe1e1a42',
             'timestamp': 1502623500,
+            'duration': 2580,
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20170813',
         },
         'params': {
             'skip_download': True,
         },
         'add_ie': [FranceTVIE.ie_key()],
+    }, {
+        'url': 'https://www.france.tv/enfants/six-huit-ans/foot2rue/saison-1/3066387-duel-au-vieux-port.html',
+        'info_dict': {
+            'id': 'a9050959-eedd-4b4a-9b0d-de6eeaa73e44',
+            'ext': 'mp4',
+            'title': 'Foot2Rue - Duel au vieux port',
+            'episode': 'Duel au vieux port',
+            'series': 'Foot2Rue',
+            'episode_number': 1,
+            'season_number': 1,
+            'timestamp': 1642761360,
+            'upload_date': '20220121',
+            'season': 'Season 1',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1441,
+        },
     }, {
         # france3
         'url': 'https://www.france.tv/france-3/des-chiffres-et-des-lettres/139063-emission-du-mardi-9-mai-2017.html',

From b03c89309eb141be1a1eceeeb7475dd3b7529ad9 Mon Sep 17 00:00:00 2001
From: Nicolas Cisco <ncis20@gmail.com>
Date: Wed, 6 Dec 2023 14:55:38 -0300
Subject: [PATCH 726/871] [ie/mediastream] Fix authenticated format extraction
 (#8657)

Authored by: NickCis
---
 yt_dlp/extractor/mediastream.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/mediastream.py b/yt_dlp/extractor/mediastream.py
index b8cb5a691..ae0fb2aed 100644
--- a/yt_dlp/extractor/mediastream.py
+++ b/yt_dlp/extractor/mediastream.py
@@ -3,8 +3,11 @@
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    filter_dict,
+    parse_qs,
     remove_end,
     traverse_obj,
+    update_url_query,
     urljoin,
 )
 
@@ -108,7 +111,9 @@ def _real_extract(self, url):
 
         for message in [
             'Debido a tu ubicación no puedes ver el contenido',
-            'You are not allowed to watch this video: Geo Fencing Restriction'
+            'You are not allowed to watch this video: Geo Fencing Restriction',
+            'Este contenido no está disponible en tu zona geográfica.',
+            'El contenido sólo está disponible dentro de',
         ]:
             if message in webpage:
                 self.raise_geo_restricted()
@@ -118,7 +123,16 @@ def _real_extract(self, url):
         formats, subtitles = [], {}
         for video_format in player_config['src']:
             if video_format == 'hls':
-                fmts, subs = self._extract_m3u8_formats_and_subtitles(player_config['src'][video_format], video_id)
+                params = {
+                    'at': 'web-app',
+                    'access_token': traverse_obj(parse_qs(url), ('access_token', 0)),
+                }
+                for name, key in (('MDSTRMUID', 'uid'), ('MDSTRMSID', 'sid'), ('MDSTRMPID', 'pid'), ('VERSION', 'av')):
+                    params[key] = self._search_regex(
+                        rf'window\.{name}\s*=\s*["\']([^"\']+)["\'];', webpage, key, default=None)
+
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    update_url_query(player_config['src'][video_format], filter_dict(params)), video_id)
                 formats.extend(fmts)
                 self._merge_subtitles(subs, target=subtitles)
             elif video_format == 'mpd':

From 04a5e06350e3ef7c03f94f2f3f90dd96c6411152 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Wed, 6 Dec 2023 18:58:00 +0100
Subject: [PATCH 727/871] [ie/ondemandkorea] Fix upgraded format extraction
 (#8677)

Closes #8675
Authored by: seproDev
---
 yt_dlp/extractor/ondemandkorea.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/ondemandkorea.py b/yt_dlp/extractor/ondemandkorea.py
index 81ce99fd9..94fcac720 100644
--- a/yt_dlp/extractor/ondemandkorea.py
+++ b/yt_dlp/extractor/ondemandkorea.py
@@ -3,7 +3,6 @@
 import uuid
 
 from .common import InfoExtractor
-from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
@@ -84,15 +83,17 @@ def _real_extract(self, url):
         def try_geo_bypass(url):
             return traverse_obj(url, ({parse_qs}, 'stream_url', 0, {url_or_none})) or url
 
-        def try_upgrade_quality(url):
-            mod_url = re.sub(r'_720(p?)\.m3u8', r'_1080\1.m3u8', url)
-            return mod_url if mod_url != url and self._request_webpage(
-                HEADRequest(mod_url), video_id, note='Checking for higher quality format',
-                errnote='No higher quality format found', fatal=False) else url
-
         formats = []
         for m3u8_url in traverse_obj(data, (('sources', 'manifest'), ..., 'url', {url_or_none}, {try_geo_bypass})):
-            formats.extend(self._extract_m3u8_formats(try_upgrade_quality(m3u8_url), video_id, fatal=False))
+            mod_url = re.sub(r'_720(p?)\.m3u8', r'_1080\1.m3u8', m3u8_url)
+            if mod_url != m3u8_url:
+                mod_format = self._extract_m3u8_formats(
+                    mod_url, video_id, note='Checking for higher quality format',
+                    errnote='No higher quality format found', fatal=False)
+                if mod_format:
+                    formats.extend(mod_format)
+                    continue
+            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, fatal=False))
 
         subtitles = {}
         for track in traverse_obj(data, ('text_tracks', lambda _, v: url_or_none(v['url']))):

From f98a3305eb124a0c375d03209d5c5a64fe1766c8 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 6 Dec 2023 21:44:54 +0100
Subject: [PATCH 728/871] [ie/pr0gramm] Support variant formats and subtitles
 (#8674)

Authored by: Grub4K
---
 yt_dlp/extractor/pr0gramm.py | 56 +++++++++++++++++++++++++++++-------
 1 file changed, 45 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/extractor/pr0gramm.py b/yt_dlp/extractor/pr0gramm.py
index c8e0bb493..2a6794208 100644
--- a/yt_dlp/extractor/pr0gramm.py
+++ b/yt_dlp/extractor/pr0gramm.py
@@ -4,7 +4,14 @@
 
 from .common import InfoExtractor
 from ..compat import functools
-from ..utils import ExtractorError, make_archive_id, urljoin
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    make_archive_id,
+    mimetype2ext,
+    urljoin,
+)
 from ..utils.traversal import traverse_obj
 
 
@@ -26,6 +33,7 @@ class Pr0grammIE(InfoExtractor):
             'dislike_count': int,
             'age_limit': 0,
             'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
+            '_old_archive_ids': ['pr0grammstatic 5466437'],
         },
     }, {
         # Tags require account
@@ -43,6 +51,7 @@ class Pr0grammIE(InfoExtractor):
             'dislike_count': int,
             'age_limit': 0,
             'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
+            '_old_archive_ids': ['pr0grammstatic 3052805'],
         },
     }, {
         # Requires verified account
@@ -60,6 +69,7 @@ class Pr0grammIE(InfoExtractor):
             'dislike_count': int,
             'age_limit': 18,
             'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
+            '_old_archive_ids': ['pr0grammstatic 5848332'],
         },
     }, {
         'url': 'https://pr0gramm.com/static/5466437',
@@ -110,37 +120,61 @@ def _call_api(self, endpoint, video_id, query={}, note='Downloading API json'):
 
         return data
 
+    @staticmethod
+    def _create_source_url(path):
+        return urljoin('https://img.pr0gramm.com', path)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_info = traverse_obj(
             self._call_api('get', video_id, {'id': video_id, 'flags': self._maximum_flags}),
             ('items', 0, {dict}))
 
-        source = urljoin('https://img.pr0gramm.com', video_info.get('image'))
+        source = video_info.get('image')
         if not source or not source.endswith('mp4'):
             self.raise_no_formats('Could not extract a video', expected=bool(source), video_id=video_id)
 
         tags = None
         if self._is_logged_in:
-            metadata = self._call_api('info', video_id, {'itemId': video_id})
+            metadata = self._call_api('info', video_id, {'itemId': video_id}, note='Downloading tags')
             tags = traverse_obj(metadata, ('tags', ..., 'tag', {str}))
             # Sorted by "confidence", higher confidence = earlier in list
             confidences = traverse_obj(metadata, ('tags', ..., 'confidence', ({int}, {float})))
             if confidences:
                 tags = [tag for _, tag in sorted(zip(confidences, tags), reverse=True)]
 
+        formats = traverse_obj(video_info, ('variants', ..., {
+            'format_id': ('name', {str}),
+            'url': ('path', {self._create_source_url}),
+            'ext': ('mimeType', {mimetype2ext}),
+            'vcodec': ('codec', {str}),
+            'width': ('width', {int_or_none}),
+            'height': ('height', {int_or_none}),
+            'bitrate': ('bitRate', {float_or_none}),
+            'filesize': ('fileSize', {int_or_none}),
+        })) if video_info.get('variants') else [{
+            'ext': 'mp4',
+            'format_id': 'source',
+            **traverse_obj(video_info, {
+                'url': ('image', {self._create_source_url}),
+                'width': ('width', {int_or_none}),
+                'height': ('height', {int_or_none}),
+            }),
+        }]
+
+        subtitles = {}
+        for subtitle in traverse_obj(video_info, ('subtitles', lambda _, v: v['language'])):
+            subtitles.setdefault(subtitle['language'], []).append(traverse_obj(subtitle, {
+                'url': ('path', {self._create_source_url}),
+                'note': ('label', {str}),
+            }))
+
         return {
             'id': video_id,
             'title': f'pr0gramm-{video_id} by {video_info.get("user")}',
-            'formats': [{
-                'url': source,
-                'ext': 'mp4',
-                **traverse_obj(video_info, {
-                    'width': ('width', {int}),
-                    'height': ('height', {int}),
-                }),
-            }],
             'tags': tags,
+            'formats': formats,
+            'subtitles': subtitles,
             'age_limit': 18 if traverse_obj(video_info, ('flags', {0b110.__and__})) else 0,
             '_old_archive_ids': [make_archive_id('Pr0grammStatic', video_id)],
             **traverse_obj(video_info, {

From 0b6f829b1dfda15d3c1d7d1fbe4ea6102c26dd24 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 6 Dec 2023 21:46:45 +0100
Subject: [PATCH 729/871] [utils] `traverse_obj`: Move `is_user_input` into
 output template (#8673)

Authored by: Grub4K
---
 test/test_utils.py        | 17 -----------------
 yt_dlp/YoutubeDL.py       | 14 ++++++++++++--
 yt_dlp/utils/traversal.py | 19 ++++++-------------
 3 files changed, 18 insertions(+), 32 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 77040f29c..100f11788 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2317,23 +2317,6 @@ def test_traverse_obj(self):
         self.assertEqual(traverse_obj({}, (0, slice(1)), traverse_string=True), [],
                          msg='branching should result in list if `traverse_string`')
 
-        # Test is_user_input behavior
-        _IS_USER_INPUT_DATA = {'range8': list(range(8))}
-        self.assertEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3'),
-                                      is_user_input=True), 3,
-                         msg='allow for string indexing if `is_user_input`')
-        self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3:'),
-                                           is_user_input=True), tuple(range(8))[3:],
-                              msg='allow for string slice if `is_user_input`')
-        self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':4:2'),
-                                           is_user_input=True), tuple(range(8))[:4:2],
-                              msg='allow step in string slice if `is_user_input`')
-        self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':'),
-                                           is_user_input=True), range(8),
-                              msg='`:` should be treated as `...` if `is_user_input`')
-        with self.assertRaises(TypeError, msg='too many params should result in error'):
-            traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':::'), is_user_input=True)
-
         # Test re.Match as input obj
         mobj = re.fullmatch(r'0(12)(?P<group>3)(4)?', '0123')
         self.assertEqual(traverse_obj(mobj, ...), [x for x in mobj.groups() if x is not None],
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 29dd76186..0c07866e4 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1201,6 +1201,15 @@ def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False):
                 (?:\|(?P<default>.*?))?
             )$''')
 
+        def _from_user_input(field):
+            if field == ':':
+                return ...
+            elif ':' in field:
+                return slice(*map(int_or_none, field.split(':')))
+            elif int_or_none(field) is not None:
+                return int(field)
+            return field
+
         def _traverse_infodict(fields):
             fields = [f for x in re.split(r'\.({.+?})\.?', fields)
                       for f in ([x] if x.startswith('{') else x.split('.'))]
@@ -1210,11 +1219,12 @@ def _traverse_infodict(fields):
 
             for i, f in enumerate(fields):
                 if not f.startswith('{'):
+                    fields[i] = _from_user_input(f)
                     continue
                 assert f.endswith('}'), f'No closing brace for {f} in {fields}'
-                fields[i] = {k: k.split('.') for k in f[1:-1].split(',')}
+                fields[i] = {k: list(map(_from_user_input, k.split('.'))) for k in f[1:-1].split(',')}
 
-            return traverse_obj(info_dict, fields, is_user_input=True, traverse_string=True)
+            return traverse_obj(info_dict, fields, traverse_string=True)
 
         def get_value(mdict):
             # Object traversal
diff --git a/yt_dlp/utils/traversal.py b/yt_dlp/utils/traversal.py
index 462c3ba5d..ff5703198 100644
--- a/yt_dlp/utils/traversal.py
+++ b/yt_dlp/utils/traversal.py
@@ -8,7 +8,7 @@
     IDENTITY,
     NO_DEFAULT,
     LazyList,
-    int_or_none,
+    deprecation_warning,
     is_iterable_like,
     try_call,
     variadic,
@@ -17,7 +17,7 @@
 
 def traverse_obj(
         obj, *paths, default=NO_DEFAULT, expected_type=None, get_all=True,
-        casesense=True, is_user_input=False, traverse_string=False):
+        casesense=True, is_user_input=NO_DEFAULT, traverse_string=False):
     """
     Safely traverse nested `dict`s and `Iterable`s
 
@@ -63,10 +63,8 @@ def traverse_obj(
     @param get_all          If `False`, return the first matching result, otherwise all matching ones.
     @param casesense        If `False`, consider string dictionary keys as case insensitive.
 
-    The following are only meant to be used by YoutubeDL.prepare_outtmpl and are not part of the API
+    `traverse_string` is only meant to be used by YoutubeDL.prepare_outtmpl and is not part of the API
 
-    @param is_user_input    Whether the keys are generated from user input.
-                            If `True` strings get converted to `int`/`slice` if needed.
     @param traverse_string  Whether to traverse into objects as strings.
                             If `True`, any non-compatible object will first be
                             converted into a string and then traversed into.
@@ -80,6 +78,9 @@ def traverse_obj(
                             If no `default` is given and the last path branches, a `list` of results
                             is always returned. If a path ends on a `dict` that result will always be a `dict`.
     """
+    if is_user_input is not NO_DEFAULT:
+        deprecation_warning('The is_user_input parameter is deprecated and no longer works')
+
     casefold = lambda k: k.casefold() if isinstance(k, str) else k
 
     if isinstance(expected_type, type):
@@ -195,14 +196,6 @@ def apply_path(start_obj, path, test_type):
 
         key = None
         for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
-            if is_user_input and isinstance(key, str):
-                if key == ':':
-                    key = ...
-                elif ':' in key:
-                    key = slice(*map(int_or_none, key.split(':')))
-                elif int_or_none(key) is not None:
-                    key = int(key)
-
             if not casesense and isinstance(key, str):
                 key = key.casefold()
 

From b1a1ec1540605d2ea7abdb63336ffb1c56bf6316 Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Mon, 11 Dec 2023 23:56:01 +0000
Subject: [PATCH 730/871] [ie/bitchute] Fix and improve metadata extraction
 (#8507)

Closes #8492
Authored by: SirElderling
---
 yt_dlp/extractor/bitchute.py | 44 ++++++++++++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/bitchute.py b/yt_dlp/extractor/bitchute.py
index 0805b8b46..41367c5b9 100644
--- a/yt_dlp/extractor/bitchute.py
+++ b/yt_dlp/extractor/bitchute.py
@@ -7,8 +7,10 @@
     ExtractorError,
     OnDemandPagedList,
     clean_html,
+    extract_attributes,
     get_element_by_class,
     get_element_by_id,
+    get_element_html_by_class,
     get_elements_html_by_class,
     int_or_none,
     orderedSet,
@@ -17,6 +19,7 @@
     traverse_obj,
     unified_strdate,
     urlencode_postdata,
+    urljoin,
 )
 
 
@@ -34,6 +37,25 @@ class BitChuteIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'BitChute',
             'upload_date': '20170103',
+            'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
+            'channel': 'BitChute',
+            'channel_url': 'https://www.bitchute.com/channel/bitchute/'
+        },
+    }, {
+        # test case: video with different channel and uploader
+        'url': 'https://www.bitchute.com/video/Yti_j9A-UZ4/',
+        'md5': 'f10e6a8e787766235946d0868703f1d0',
+        'info_dict': {
+            'id': 'Yti_j9A-UZ4',
+            'ext': 'mp4',
+            'title': 'Israel at War | Full Measure',
+            'description': 'md5:38cf7bc6f42da1a877835539111c69ef',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'sharylattkisson',
+            'upload_date': '20231106',
+            'uploader_url': 'https://www.bitchute.com/profile/9K0kUWA9zmd9/',
+            'channel': 'Full Measure with Sharyl Attkisson',
+            'channel_url': 'https://www.bitchute.com/channel/sharylattkisson/'
         },
     }, {
         # video not downloadable in browser, but we can recover it
@@ -48,6 +70,9 @@ class BitChuteIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'BitChute',
             'upload_date': '20181113',
+            'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
+            'channel': 'BitChute',
+            'channel_url': 'https://www.bitchute.com/channel/bitchute/'
         },
         'params': {'check_formats': None},
     }, {
@@ -99,6 +124,11 @@ def _raise_if_restricted(self, webpage):
             reason = clean_html(get_element_by_id('page-detail', webpage)) or page_title
             self.raise_geo_restricted(reason)
 
+    @staticmethod
+    def _make_url(html):
+        path = extract_attributes(get_element_html_by_class('spa', html) or '').get('href')
+        return urljoin('https://www.bitchute.com', path)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
@@ -121,12 +151,19 @@ def _real_extract(self, url):
                 'Video is unavailable. Please make sure this video is playable in the browser '
                 'before reporting this issue.', expected=True, video_id=video_id)
 
+        details = get_element_by_class('details', webpage) or ''
+        uploader_html = get_element_html_by_class('creator', details) or ''
+        channel_html = get_element_html_by_class('name', details) or ''
+
         return {
             'id': video_id,
             'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader': clean_html(get_element_by_class('owner', webpage)),
+            'uploader': clean_html(uploader_html),
+            'uploader_url': self._make_url(uploader_html),
+            'channel': clean_html(channel_html),
+            'channel_url': self._make_url(channel_html),
             'upload_date': unified_strdate(self._search_regex(
                 r'at \d+:\d+ UTC on (.+?)\.', publish_date, 'upload date', fatal=False)),
             'formats': formats,
@@ -154,6 +191,9 @@ class BitChuteChannelIE(InfoExtractor):
                     'thumbnail': r're:^https?://.*\.jpg$',
                     'uploader': 'BitChute',
                     'upload_date': '20170103',
+                    'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
+                    'channel': 'BitChute',
+                    'channel_url': 'https://www.bitchute.com/channel/bitchute/',
                     'duration': 16,
                     'view_count': int,
                 },
@@ -169,7 +209,7 @@ class BitChuteChannelIE(InfoExtractor):
         'info_dict': {
             'id': 'wV9Imujxasw9',
             'title': 'Bruce MacDonald and "The Light of Darkness"',
-            'description': 'md5:04913227d2714af1d36d804aa2ab6b1e',
+            'description': 'md5:747724ef404eebdfc04277714f81863e',
         }
     }]
 

From e370f9ec36972d06100a3db893b397bfc1b07b4d Mon Sep 17 00:00:00 2001
From: Benjamin Krausse <benjamin@ambcr.net>
Date: Mon, 11 Dec 2023 17:52:59 -0600
Subject: [PATCH 731/871] [ie] Add `media_type` field

Authored by: trainman261
---
 README.md                  | 1 +
 yt_dlp/extractor/common.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 78d4799a1..06aceec02 100644
--- a/README.md
+++ b/README.md
@@ -1333,6 +1333,7 @@ # OUTPUT TEMPLATE
  - `was_live` (boolean): Whether this video was originally a live stream
  - `playable_in_embed` (string): Whether this video is allowed to play in embedded players on other sites
  - `availability` (string): Whether the video is "private", "premium_only", "subscriber_only", "needs_auth", "unlisted" or "public"
+ - `media_type` (string): The type of media as classified by the site, e.g. "episode", "clip", "trailer"
  - `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL
  - `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL
  - `extractor` (string): Name of the extractor
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index b179f4038..af534775f 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -382,6 +382,7 @@ class InfoExtractor:
                     'private', 'premium_only', 'subscriber_only', 'needs_auth',
                     'unlisted' or 'public'. Use 'InfoExtractor._availability'
                     to set it
+    media_type:     The type of media as classified by the site, e.g. "episode", "clip", "trailer"
     _old_archive_ids: A list of old archive ids needed for backward compatibility
     _format_sort_fields: A list of fields to use for sorting formats
     __post_extractor: A function to be called just before the metadata is

From 7e09c147fdccb44806bbf601573adc4b77210a89 Mon Sep 17 00:00:00 2001
From: trainman261 <trainman261@users.noreply.github.com>
Date: Tue, 12 Dec 2023 01:00:35 +0100
Subject: [PATCH 732/871] [ie/theplatform] Extract more metadata (#8635)

Authored by: trainman261
---
 yt_dlp/extractor/aenetworks.py      | 23 ++++++++++++++++--
 yt_dlp/extractor/cbc.py             | 37 ++++++++++++++++++++++-------
 yt_dlp/extractor/cwtv.py            |  4 ++++
 yt_dlp/extractor/mediaset.py        |  2 ++
 yt_dlp/extractor/nbc.py             |  4 ++++
 yt_dlp/extractor/scrippsnetworks.py |  2 ++
 yt_dlp/extractor/theplatform.py     | 12 ++++++++++
 7 files changed, 73 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/extractor/aenetworks.py b/yt_dlp/extractor/aenetworks.py
index cc26653c1..63a0532ef 100644
--- a/yt_dlp/extractor/aenetworks.py
+++ b/yt_dlp/extractor/aenetworks.py
@@ -121,11 +121,21 @@ class AENetworksIE(AENetworksBaseIE):
         'info_dict': {
             'id': '22253814',
             'ext': 'mp4',
-            'title': 'Winter is Coming',
-            'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
+            'title': 'Winter Is Coming',
+            'description': 'md5:a40e370925074260b1c8a633c632c63a',
             'timestamp': 1338306241,
             'upload_date': '20120529',
             'uploader': 'AENE-NEW',
+            'duration': 2592.0,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'chapters': 'count:5',
+            'tags': 'count:14',
+            'categories': ['Mountain Men'],
+            'episode_number': 1,
+            'episode': 'Episode 1',
+            'season': 'Season 1',
+            'season_number': 1,
+            'series': 'Mountain Men',
         },
         'params': {
             # m3u8 download
@@ -143,6 +153,15 @@ class AENetworksIE(AENetworksBaseIE):
             'timestamp': 1452634428,
             'upload_date': '20160112',
             'uploader': 'AENE-NEW',
+            'duration': 1277.695,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'chapters': 'count:4',
+            'tags': 'count:23',
+            'episode': 'Episode 1',
+            'episode_number': 1,
+            'season': 'Season 9',
+            'season_number': 9,
+            'series': 'Duck Dynasty',
         },
         'params': {
             # m3u8 download
diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index 29f0e307d..b5beb1ec8 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -180,6 +180,13 @@ class CBCPlayerIE(InfoExtractor):
             'thumbnail': 'http://thumbnails.cbc.ca/maven_legacy/thumbnails/sonali-karnick-220.jpg',
             'chapters': [],
             'duration': 494.811,
+            'categories': ['AudioMobile/All in a Weekend Montreal'],
+            'tags': 'count:8',
+            'location': 'Quebec',
+            'series': 'All in a Weekend Montreal',
+            'season': 'Season 2015',
+            'season_number': 2015,
+            'media_type': 'Excerpt',
         },
     }, {
         'url': 'http://www.cbc.ca/player/play/2164402062',
@@ -195,25 +202,37 @@ class CBCPlayerIE(InfoExtractor):
             'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/277/67/cancer_852x480_2164412612.jpg',
             'chapters': [],
             'duration': 186.867,
+            'series': 'CBC News: Windsor at 6:00',
+            'categories': ['News/Canada/Windsor'],
+            'location': 'Windsor',
+            'tags': ['cancer'],
+            'creator': 'Allison Johnson',
+            'media_type': 'Excerpt',
         },
     }, {
         # Has subtitles
         # These broadcasts expire after ~1 month, can find new test URL here:
         # https://www.cbc.ca/player/news/TV%20Shows/The%20National/Latest%20Broadcast
-        'url': 'http://www.cbc.ca/player/play/2249992771553',
-        'md5': '2f2fb675dd4f0f8a5bb7588d1b13bacd',
+        'url': 'http://www.cbc.ca/player/play/2284799043667',
+        'md5': '9b49f0839e88b6ec0b01d840cf3d42b5',
         'info_dict': {
-            'id': '2249992771553',
+            'id': '2284799043667',
             'ext': 'mp4',
-            'title': 'The National | Women’s soccer pay, Florida seawater, Swift quake',
-            'description': 'md5:adba28011a56cfa47a080ff198dad27a',
-            'timestamp': 1690596000,
-            'duration': 2716.333,
+            'title': 'The National | Hockey coach charged, Green grants, Safer drugs',
+            'description': 'md5:84ef46321c94bcf7d0159bb565d26bfa',
+            'timestamp': 1700272800,
+            'duration': 2718.833,
             'subtitles': {'eng': [{'ext': 'vtt', 'protocol': 'm3u8_native'}]},
-            'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/481/326/thumbnail.jpeg',
+            'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/907/171/thumbnail.jpeg',
             'uploader': 'CBCC-NEW',
             'chapters': 'count:5',
-            'upload_date': '20230729',
+            'upload_date': '20231118',
+            'categories': 'count:4',
+            'series': 'The National - Full Show',
+            'tags': 'count:1',
+            'creator': 'News',
+            'location': 'Canada',
+            'media_type': 'Full Program',
         },
     }]
 
diff --git a/yt_dlp/extractor/cwtv.py b/yt_dlp/extractor/cwtv.py
index 9b83264ee..69d50daf6 100644
--- a/yt_dlp/extractor/cwtv.py
+++ b/yt_dlp/extractor/cwtv.py
@@ -46,6 +46,10 @@ class CWTVIE(InfoExtractor):
             'timestamp': 1444107300,
             'age_limit': 14,
             'uploader': 'CWTV',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'chapters': 'count:4',
+            'episode': 'Episode 20',
+            'season': 'Season 11',
         },
         'params': {
             # m3u8 download
diff --git a/yt_dlp/extractor/mediaset.py b/yt_dlp/extractor/mediaset.py
index 2d6204298..e04a1ce90 100644
--- a/yt_dlp/extractor/mediaset.py
+++ b/yt_dlp/extractor/mediaset.py
@@ -73,6 +73,7 @@ class MediasetIE(ThePlatformBaseIE):
             'season_number': 5,
             'episode_number': 5,
             'chapters': [{'start_time': 0.0, 'end_time': 3409.08}, {'start_time': 3409.08, 'end_time': 6565.008}],
+            'categories': ['Informazione'],
         },
     }, {
         # DRM
@@ -149,6 +150,7 @@ class MediasetIE(ThePlatformBaseIE):
             'season_number': 12,
             'episode': 'Episode 8',
             'episode_number': 8,
+            'categories': ['Intrattenimento'],
         },
         'params': {
             'skip_download': True,
diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index 2d3aa26ec..267fa8353 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -53,6 +53,8 @@ class NBCIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'chapters': 'count:1',
                 'tags': 'count:4',
                 'thumbnail': r're:https?://.+\.jpg',
+                'categories': ['Series/The Tonight Show Starring Jimmy Fallon'],
+                'media_type': 'Full Episode',
             },
             'params': {
                 'skip_download': 'm3u8',
@@ -131,6 +133,8 @@ class NBCIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
                 'tags': 'count:10',
                 'age_limit': 0,
                 'thumbnail': r're:https?://.+\.jpg',
+                'categories': ['Series/Quantum Leap 2022'],
+                'media_type': 'Highlight',
             },
             'params': {
                 'skip_download': 'm3u8',
diff --git a/yt_dlp/extractor/scrippsnetworks.py b/yt_dlp/extractor/scrippsnetworks.py
index 7f0bc9645..3912f7786 100644
--- a/yt_dlp/extractor/scrippsnetworks.py
+++ b/yt_dlp/extractor/scrippsnetworks.py
@@ -114,6 +114,8 @@ class ScrippsNetworksIE(InfoExtractor):
             'timestamp': 1475678834,
             'upload_date': '20161005',
             'uploader': 'SCNI-SCND',
+            'tags': 'count:10',
+            'creator': 'Cooking Channel',
             'duration': 29.995,
             'chapters': [{'start_time': 0.0, 'end_time': 29.995, 'title': '<Untitled Chapter 1>'}],
             'thumbnail': 'https://images.dds.discovery.com/up/tp/Scripps_-_Food_Category_Prod/122/987/0260338_630x355.jpg',
diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py
index 433ce8427..9160f5ec6 100644
--- a/yt_dlp/extractor/theplatform.py
+++ b/yt_dlp/extractor/theplatform.py
@@ -104,6 +104,10 @@ def _add_chapter(start_time, end_time):
                 _add_chapter(chapter.get('startTime'), chapter.get('endTime'))
             _add_chapter(tp_chapters[-1].get('startTime'), tp_chapters[-1].get('endTime') or duration)
 
+        def extract_site_specific_field(field):
+            # A number of sites have custom-prefixed keys, e.g. 'cbc$seasonNumber'
+            return traverse_obj(info, lambda k, v: v and k.endswith(f'${field}'), get_all=False)
+
         return {
             'title': info['title'],
             'subtitles': subtitles,
@@ -113,6 +117,14 @@ def _add_chapter(start_time, end_time):
             'timestamp': int_or_none(info.get('pubDate'), 1000) or None,
             'uploader': info.get('billingCode'),
             'chapters': chapters,
+            'creator': traverse_obj(info, ('author', {str})) or None,
+            'categories': traverse_obj(info, (
+                'categories', lambda _, v: v.get('label') in ('category', None), 'name', {str})) or None,
+            'tags': traverse_obj(info, ('keywords', {lambda x: re.split(r'[;,]\s?', x) if x else None})),
+            'location': extract_site_specific_field('region'),
+            'series': extract_site_specific_field('show'),
+            'season_number': int_or_none(extract_site_specific_field('seasonNumber')),
+            'media_type': extract_site_specific_field('programmingType') or extract_site_specific_field('type'),
         }
 
     def _extract_theplatform_metadata(self, path, video_id):

From d5d1517e7d838500800d193ac3234b06e89654cd Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Tue, 12 Dec 2023 08:29:36 +0800
Subject: [PATCH 733/871] [ie/eplus] Add login support and DRM detection
 (#8661)

Authored by: pzhlkj6612
---
 yt_dlp/extractor/eplus.py | 98 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 93 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/eplus.py b/yt_dlp/extractor/eplus.py
index 3ebdcf5fb..6383691a1 100644
--- a/yt_dlp/extractor/eplus.py
+++ b/yt_dlp/extractor/eplus.py
@@ -1,15 +1,20 @@
+import json
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     try_call,
     unified_timestamp,
+    urlencode_postdata,
 )
 
 
 class EplusIbIE(InfoExtractor):
-    IE_NAME = 'eplus:inbound'
-    IE_DESC = 'e+ (イープラス) overseas'
-    _VALID_URL = r'https?://live\.eplus\.jp/ex/player\?ib=(?P<id>(?:\w|%2B|%2F){86}%3D%3D)'
+    _NETRC_MACHINE = 'eplus'
+    IE_NAME = 'eplus'
+    IE_DESC = 'e+ (イープラス)'
+    _VALID_URL = [r'https?://live\.eplus\.jp/ex/player\?ib=(?P<id>(?:\w|%2B|%2F){86}%3D%3D)',
+                  r'https?://live\.eplus\.jp/(?P<id>sample|\d+)']
     _TESTS = [{
         'url': 'https://live.eplus.jp/ex/player?ib=YEFxb3Vyc2Dombnjg7blkrLlrablnJLjgrnjgq%2Fjg7zjg6vjgqLjgqTjg4njg6vlkIzlpb3kvJpgTGllbGxhIQ%3D%3D',
         'info_dict': {
@@ -29,14 +34,97 @@ class EplusIbIE(InfoExtractor):
             'No video formats found!',
             'Requested format is not available',
         ],
+    }, {
+        'url': 'https://live.eplus.jp/sample',
+        'info_dict': {
+            'id': 'stream1ng20210719-test-005',
+            'title': 'Online streaming test for DRM',
+            'live_status': 'was_live',
+            'release_date': '20210719',
+            'release_timestamp': 1626703200,
+            'description': None,
+        },
+        'params': {
+            'skip_download': True,
+            'ignore_no_formats_error': True,
+        },
+        'expected_warnings': [
+            'Could not find the playlist URL. This event may not be accessible',
+            'No video formats found!',
+            'Requested format is not available',
+            'This video is DRM protected',
+        ],
+    }, {
+        'url': 'https://live.eplus.jp/2053935',
+        'info_dict': {
+            'id': '331320-0001-001',
+            'title': '丘みどり2020配信LIVE Vol.2 ～秋麗～ 【Streaming+(配信チケット)】',
+            'live_status': 'was_live',
+            'release_date': '20200920',
+            'release_timestamp': 1600596000,
+        },
+        'params': {
+            'skip_download': True,
+            'ignore_no_formats_error': True,
+        },
+        'expected_warnings': [
+            'Could not find the playlist URL. This event may not be accessible',
+            'No video formats found!',
+            'Requested format is not available',
+        ],
     }]
 
+    _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0'
+
+    def _login(self, username, password, urlh):
+        if not self._get_cookies('https://live.eplus.jp/').get('ci_session'):
+            raise ExtractorError('Unable to get ci_session cookie')
+
+        cltft_token = urlh.headers.get('X-CLTFT-Token')
+        if not cltft_token:
+            raise ExtractorError('Unable to get X-CLTFT-Token')
+        self._set_cookie('live.eplus.jp', 'X-CLTFT-Token', cltft_token)
+
+        login_json = self._download_json(
+            'https://live.eplus.jp/member/api/v1/FTAuth/idpw', None,
+            note='Sending pre-login info', errnote='Unable to send pre-login info', headers={
+                'Content-Type': 'application/json; charset=UTF-8',
+                'Referer': urlh.url,
+                'X-Cltft-Token': cltft_token,
+                'Accept': '*/*',
+            }, data=json.dumps({
+                'loginId': username,
+                'loginPassword': password,
+            }).encode())
+        if not login_json.get('isSuccess'):
+            raise ExtractorError('Login failed: Invalid id or password', expected=True)
+
+        self._request_webpage(
+            urlh.url, None, note='Logging in', errnote='Unable to log in',
+            data=urlencode_postdata({
+                'loginId': username,
+                'loginPassword': password,
+                'Token.Default': cltft_token,
+                'op': 'nextPage',
+            }), headers={'Referer': urlh.url})
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage, urlh = self._download_webpage_handle(
+            url, video_id, headers={'User-Agent': self._USER_AGENT})
+        if urlh.url.startswith('https://live.eplus.jp/member/auth'):
+            username, password = self._get_login_info()
+            if not username:
+                self.raise_login_required()
+            self._login(username, password, urlh)
+            webpage = self._download_webpage(
+                url, video_id, headers={'User-Agent': self._USER_AGENT})
 
         data_json = self._search_json(r'<script>\s*var app\s*=', webpage, 'data json', video_id)
 
+        if data_json.get('drm_mode') == 'ON':
+            self.report_drm(video_id)
+
         delivery_status = data_json.get('delivery_status')
         archive_mode = data_json.get('archive_mode')
         release_timestamp = try_call(lambda: unified_timestamp(data_json['event_datetime']) - 32400)
@@ -64,7 +152,7 @@ def _real_extract(self, url):
         formats = []
 
         m3u8_playlist_urls = self._search_json(
-            r'var listChannels\s*=', webpage, 'hls URLs', video_id, contains_pattern=r'\[.+\]', default=[])
+            r'var\s+listChannels\s*=', webpage, 'hls URLs', video_id, contains_pattern=r'\[.+\]', default=[])
         if not m3u8_playlist_urls:
             if live_status == 'is_upcoming':
                 self.raise_no_formats(

From 298230e550886b746c266724dd701d842ca2696e Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 13 Dec 2023 05:10:35 +0530
Subject: [PATCH 734/871] [webvtt] Fix 15f22b4880b6b3f71f350c64d70976ae65b9f1ca

---
 yt_dlp/webvtt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/webvtt.py b/yt_dlp/webvtt.py
index 596e9bf73..c80c58631 100644
--- a/yt_dlp/webvtt.py
+++ b/yt_dlp/webvtt.py
@@ -286,8 +286,8 @@ def parse(cls, parser):
         m1 = parser.consume(_REGEX_TS)
         if not m1:
             return None
-        parser.consume(_REGEX_OPTIONAL_WHITESPACE)
         m2 = parser.consume(cls._REGEX_SETTINGS)
+        parser.consume(_REGEX_OPTIONAL_WHITESPACE)
         if not parser.consume(_REGEX_NL):
             return None
 

From 6b5d93b0b0240e287389d1d43b2d5293e18aa4cc Mon Sep 17 00:00:00 2001
From: Pandey Ganesha <39578603+Ganesh910@users.noreply.github.com>
Date: Wed, 13 Dec 2023 12:34:12 +0530
Subject: [PATCH 735/871] [ie/youtube] Fix `like_count` extraction (#8763)

Closes #8759
Authored by: Ganesh910
---
 yt_dlp/extractor/youtube.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 86818a9dc..5b14b187a 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -4480,14 +4480,13 @@ def process_language(container, base_url, lang_code, sub_name, query):
                             if mobj:
                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
                                 break
-            sbr_tooltip = try_get(
-                vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
-            if sbr_tooltip:
-                like_count, dislike_count = sbr_tooltip.split(' / ')
-                info.update({
-                    'like_count': str_to_int(like_count),
-                    'dislike_count': str_to_int(dislike_count),
-                })
+
+            info['like_count'] = traverse_obj(vpir, (
+                'videoActions', 'menuRenderer', 'topLevelButtons', ...,
+                'segmentedLikeDislikeButtonViewModel', 'likeButtonViewModel', 'likeButtonViewModel',
+                'toggleButtonViewModel', 'toggleButtonViewModel', 'defaultButtonViewModel',
+                'buttonViewModel', 'accessibilityText', {parse_count}), get_all=False)
+
             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
             if vcr:
                 vc = self._get_count(vcr, 'viewCount')

From c91af948e43570025e4aa887e248fd025abae394 Mon Sep 17 00:00:00 2001
From: Tristan Charpentier <tristan_charpentier@hotmail.com>
Date: Sun, 17 Dec 2023 09:07:55 -0500
Subject: [PATCH 736/871] [ie/RinseFM] Add extractor (#8778)

Authored by: hashFactory
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/rinsefm.py     | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 yt_dlp/extractor/rinsefm.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 9b96bd5b4..94369ca66 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1590,6 +1590,7 @@
 from .reuters import ReutersIE
 from .reverbnation import ReverbNationIE
 from .rheinmaintv import RheinMainTVIE
+from .rinsefm import RinseFMIE
 from .rmcdecouverte import RMCDecouverteIE
 from .rockstargames import RockstarGamesIE
 from .rokfin import (
diff --git a/yt_dlp/extractor/rinsefm.py b/yt_dlp/extractor/rinsefm.py
new file mode 100644
index 000000000..760adf0eb
--- /dev/null
+++ b/yt_dlp/extractor/rinsefm.py
@@ -0,0 +1,33 @@
+from .common import InfoExtractor
+from ..utils import format_field, parse_iso8601
+
+
+class RinseFMIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?rinse\.fm/episodes/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://rinse.fm/episodes/club-glow-15-12-2023-2000/',
+        'md5': '76ee0b719315617df42e15e710f46c7b',
+        'info_dict': {
+            'id': '1536535',
+            'ext': 'mp3',
+            'title': 'Club Glow - 15/12/2023 - 20:00',
+            'thumbnail': r're:^https://.+\.(?:jpg|JPG)$',
+            'release_timestamp': 1702598400,
+            'release_date': '20231215'
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        entry = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['entry']
+
+        return {
+            'id': entry['id'],
+            'title': entry.get('title'),
+            'url': entry['fileUrl'],
+            'vcodec': 'none',
+            'release_timestamp': parse_iso8601(entry.get('episodeDate')),
+            'thumbnail': format_field(
+                entry, [('featuredImage', 0, 'filename')], 'https://rinse.imgix.net/media/%s', default=None),
+        }

From c5f01bf7d4b9426c87c3f8248de23934a56579e0 Mon Sep 17 00:00:00 2001
From: "Amir Y. Perehodnik" <myrprhwdnyq@gmail.com>
Date: Mon, 18 Dec 2023 17:52:43 +0200
Subject: [PATCH 737/871] [ie/Maariv] Add extractor (#8331)

Authored by: amir16yp
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/maariv.py      | 62 +++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 yt_dlp/extractor/maariv.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 94369ca66..b3c411394 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -991,6 +991,7 @@
     LyndaIE,
     LyndaCourseIE
 )
+from .maariv import MaarivIE
 from .magellantv import MagellanTVIE
 from .magentamusik360 import MagentaMusik360IE
 from .mailru import (
diff --git a/yt_dlp/extractor/maariv.py b/yt_dlp/extractor/maariv.py
new file mode 100644
index 000000000..425a8b3b4
--- /dev/null
+++ b/yt_dlp/extractor/maariv.py
@@ -0,0 +1,62 @@
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_resolution,
+    unified_timestamp,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class MaarivIE(InfoExtractor):
+    IE_NAME = 'maariv.co.il'
+    _VALID_URL = r'https?://player\.maariv\.co\.il/public/player\.html\?(?:[^#]+&)?media=(?P<id>\d+)'
+    _EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL})']
+    _TESTS = [{
+        'url': 'https://player.maariv.co.il/public/player.html?player=maariv-desktop&media=3611585',
+        'info_dict': {
+            'id': '3611585',
+            'duration': 75,
+            'ext': 'mp4',
+            'upload_date': '20231009',
+            'title': 'מבצע חרבות ברזל',
+            'timestamp': 1696851301,
+        },
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://www.maariv.co.il/news/law/Article-1044008',
+        'info_dict': {
+            'id': '3611585',
+            'duration': 75,
+            'ext': 'mp4',
+            'upload_date': '20231009',
+            'title': 'מבצע חרבות ברזל',
+            'timestamp': 1696851301,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._download_json(
+            f'https://dal.walla.co.il/media/{video_id}?origin=player.maariv.co.il', video_id)['data']
+
+        formats = []
+        if hls_url := traverse_obj(data, ('video', 'url', {url_or_none})):
+            formats.extend(self._extract_m3u8_formats(hls_url, video_id, m3u8_id='hls', fatal=False))
+
+        for http_format in traverse_obj(data, ('video', 'stream_urls', ..., 'stream_url', {url_or_none})):
+            formats.append({
+                'url': http_format,
+                'format_id': 'http',
+                **parse_resolution(http_format),
+            })
+
+        return {
+            'id': video_id,
+            **traverse_obj(data, {
+                'title': 'title',
+                'duration': ('video', 'duration', {int_or_none}),
+                'timestamp': ('upload_date', {unified_timestamp}),
+            }),
+            'formats': formats,
+        }

From 00a3e47bf5440c96025a76e08337ff2a475ed83e Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Mon, 18 Dec 2023 21:32:08 +0100
Subject: [PATCH 738/871] [ie/bundestag] Add extractor (#8783)

Authored by: Grub4K
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/bundestag.py   | 123 ++++++++++++++++++++++++++++++++
 2 files changed, 124 insertions(+)
 create mode 100644 yt_dlp/extractor/bundestag.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b3c411394..572d79fba 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -276,6 +276,7 @@
 )
 from .businessinsider import BusinessInsiderIE
 from .bundesliga import BundesligaIE
+from .bundestag import BundestagIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
diff --git a/yt_dlp/extractor/bundestag.py b/yt_dlp/extractor/bundestag.py
new file mode 100644
index 000000000..9fd7c7de1
--- /dev/null
+++ b/yt_dlp/extractor/bundestag.py
@@ -0,0 +1,123 @@
+import re
+from functools import partial
+
+from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
+from ..utils import (
+    ExtractorError,
+    bug_reports_message,
+    clean_html,
+    format_field,
+    get_element_text_and_html_by_tag,
+    int_or_none,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class BundestagIE(InfoExtractor):
+    _VALID_URL = [
+        r'https?://dbtg\.tv/[cf]vid/(?P<id>\d+)',
+        r'https?://www\.bundestag\.de/mediathek/?\?(?:[^#]+&)?videoid=(?P<id>\d+)',
+    ]
+    _TESTS = [{
+        'url': 'https://dbtg.tv/cvid/7605304',
+        'info_dict': {
+            'id': '7605304',
+            'ext': 'mp4',
+            'title': '145. Sitzung vom 15.12.2023, TOP 24 Barrierefreiheit',
+            'description': 'md5:321a9dc6bdad201264c0045efc371561',
+        },
+    }, {
+        'url': 'https://www.bundestag.de/mediathek?videoid=7602120&url=L21lZGlhdGhla292ZXJsYXk=&mod=mediathek',
+        'info_dict': {
+            'id': '7602120',
+            'ext': 'mp4',
+            'title': '130. Sitzung vom 18.10.2023, TOP 1 Befragung der Bundesregierung',
+            'description': 'Befragung der Bundesregierung',
+        },
+    }, {
+        'url': 'https://www.bundestag.de/mediathek?videoid=7604941#url=L21lZGlhdGhla292ZXJsYXk/dmlkZW9pZD03NjA0OTQx&mod=mediathek',
+        'only_matching': True,
+    }, {
+        'url': 'http://dbtg.tv/fvid/3594346',
+        'only_matching': True,
+    }]
+
+    _OVERLAY_URL = 'https://www.bundestag.de/mediathekoverlay'
+    _INSTANCE_FORMAT = 'https://cldf-wzw-od.r53.cdn.tv1.eu/13014bundestagod/_definst_/13014bundestag/ondemand/3777parlamentsfernsehen/archiv/app144277506/145293313/{0}/{0}_playlist.smil/playlist.m3u8'
+
+    _SHARE_URL = 'https://webtv.bundestag.de/player/macros/_x_s-144277506/shareData.json?contentId='
+    _SHARE_AUDIO_REGEX = r'/\d+_(?P<codec>\w+)_(?P<bitrate>\d+)kb_(?P<channels>\w+)_\w+_\d+\.(?P<ext>\w+)'
+    _SHARE_VIDEO_REGEX = r'/\d+_(?P<codec>\w+)_(?P<width>\w+)_(?P<height>\w+)_(?P<bitrate>\d+)kb_\w+_\w+_\d+\.(?P<ext>\w+)'
+
+    def _bt_extract_share_formats(self, video_id):
+        share_data = self._download_json(
+            f'{self._SHARE_URL}{video_id}', video_id, note='Downloading share format JSON')
+        if traverse_obj(share_data, ('status', 'code', {int})) != 1:
+            self.report_warning(format_field(
+                share_data, [('status', 'message', {str})],
+                'Share API response: %s', default='Unknown Share API Error')
+                + bug_reports_message())
+            return
+
+        for name, url in share_data.items():
+            if not isinstance(name, str) or not url_or_none(url):
+                continue
+
+            elif name.startswith('audio'):
+                match = re.search(self._SHARE_AUDIO_REGEX, url)
+                yield {
+                    'format_id': name,
+                    'url': url,
+                    'vcodec': 'none',
+                    **traverse_obj(match, {
+                        'acodec': 'codec',
+                        'audio_channels': ('channels', {{'mono': 1, 'stereo': 2}.get}),
+                        'abr': ('bitrate', {int_or_none}),
+                        'ext': 'ext',
+                    }),
+                }
+
+            elif name.startswith('download'):
+                match = re.search(self._SHARE_VIDEO_REGEX, url)
+                yield {
+                    'format_id': name,
+                    'url': url,
+                    **traverse_obj(match, {
+                        'vcodec': 'codec',
+                        'tbr': ('bitrate', {int_or_none}),
+                        'width': ('width', {int_or_none}),
+                        'height': ('height', {int_or_none}),
+                        'ext': 'ext',
+                    }),
+                }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        formats = []
+        result = {'id': video_id, 'formats': formats}
+
+        try:
+            formats.extend(self._extract_m3u8_formats(
+                self._INSTANCE_FORMAT.format(video_id), video_id, m3u8_id='instance'))
+        except ExtractorError as error:
+            if isinstance(error.cause, HTTPError) and error.cause.status == 404:
+                raise ExtractorError('Could not find video id', expected=True)
+            self.report_warning(f'Error extracting hls formats: {error}', video_id)
+        formats.extend(self._bt_extract_share_formats(video_id))
+        if not formats:
+            self.raise_no_formats('Could not find suitable formats', video_id=video_id)
+
+        result.update(traverse_obj(self._download_webpage(
+            self._OVERLAY_URL, video_id,
+            query={'videoid': video_id, 'view': 'main'},
+            note='Downloading metadata overlay', fatal=False,
+        ), {
+            'title': (
+                {partial(get_element_text_and_html_by_tag, 'h3')}, 0,
+                {partial(re.sub, r'<span[^>]*>[^<]+</span>', '')}, {clean_html}),
+            'description': ({partial(get_element_text_and_html_by_tag, 'p')}, 0, {clean_html}),
+        }))
+
+        return result

From 1c54a98e19d047e7c15184237b6ef8ad50af489c Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Tue, 19 Dec 2023 07:24:55 -0600
Subject: [PATCH 739/871] [ie/twitter] Extract stale tweets (#8724)

Closes #8691
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 85 ++++++++++++++++++++++++++-----------
 1 file changed, 61 insertions(+), 24 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index d7609bc81..932b478d4 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -479,9 +479,9 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': [],
             'age_limit': 18,
+            '_old_archive_ids': ['twitter 643211948184596480'],
         },
     }, {
         'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
@@ -515,6 +515,7 @@ class TwitterIE(TwitterBaseIE):
             'like_count': int,
             'tags': ['TV', 'StarWars', 'TheForceAwakens'],
             'age_limit': 0,
+            '_old_archive_ids': ['twitter 665052190608723968'],
         },
     }, {
         'url': 'https://twitter.com/BTNBrentYarina/status/705235433198714880',
@@ -558,9 +559,9 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': ['Damndaniel'],
             'age_limit': 0,
+            '_old_archive_ids': ['twitter 700207533655363584'],
         },
     }, {
         'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
@@ -599,9 +600,9 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': [],
             'age_limit': 0,
+            '_old_archive_ids': ['twitter 719944021058060289'],
         },
     }, {
         'url': 'https://twitter.com/OPP_HSD/status/779210622571536384',
@@ -616,6 +617,7 @@ class TwitterIE(TwitterBaseIE):
             'thumbnail': r're:^https?://.*\.jpg',
         },
         'add_ie': ['Periscope'],
+        'skip': 'Broadcast not found',
     }, {
         # has mp4 formats via mobile API
         'url': 'https://twitter.com/news_al3alm/status/852138619213144067',
@@ -635,9 +637,9 @@ class TwitterIE(TwitterBaseIE):
             'thumbnail': r're:^https?://.*\.jpg',
             'tags': [],
             'repost_count': int,
-            'view_count': int,
             'like_count': int,
             'comment_count': int,
+            '_old_archive_ids': ['twitter 852138619213144067'],
         },
     }, {
         'url': 'https://twitter.com/i/web/status/910031516746514432',
@@ -657,9 +659,9 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': ['Maria'],
             'age_limit': 0,
+            '_old_archive_ids': ['twitter 910031516746514432'],
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -683,9 +685,9 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': [],
             'age_limit': 0,
+            '_old_archive_ids': ['twitter 1001551623938805763'],
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -749,6 +751,7 @@ class TwitterIE(TwitterBaseIE):
             'like_count': int,
             'tags': [],
             'age_limit': 0,
+            '_old_archive_ids': ['twitter 1349794411333394432'],
         },
         'params': {
             'skip_download': True,
@@ -771,18 +774,18 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': [],
             'age_limit': 0,
+            '_old_archive_ids': ['twitter 1577855540407197696'],
         },
         'params': {'skip_download': True},
     }, {
         'url': 'https://twitter.com/UltimaShadowX/status/1577719286659006464',
         'info_dict': {
             'id': '1577719286659006464',
-            'title': 'Ultima📛| New Era - Test',
+            'title': 'Ultima - Test',
             'description': 'Test https://t.co/Y3KEZD7Dad',
-            'uploader': 'Ultima📛| New Era',
+            'uploader': 'Ultima',
             'uploader_id': 'UltimaShadowX',
             'uploader_url': 'https://twitter.com/UltimaShadowX',
             'upload_date': '20221005',
@@ -813,9 +816,9 @@ class TwitterIE(TwitterBaseIE):
             'comment_count': int,
             'repost_count': int,
             'like_count': int,
-            'view_count': int,
             'tags': ['HurricaneIan'],
             'age_limit': 0,
+            '_old_archive_ids': ['twitter 1575560063510810624'],
         },
     }, {
         # Adult content, fails if not logged in
@@ -951,10 +954,10 @@ class TwitterIE(TwitterBaseIE):
             'uploader_url': 'https://twitter.com/CTVJLaidlaw',
             'display_id': '1600649710662213632',
             'like_count': int,
-            'view_count': int,
             'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
             'upload_date': '20221208',
             'age_limit': 0,
+            '_old_archive_ids': ['twitter 1600649710662213632'],
         },
         'params': {'noplaylist': True},
     }, {
@@ -979,7 +982,7 @@ class TwitterIE(TwitterBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
-            'view_count': int,
+            '_old_archive_ids': ['twitter 1621117700482416640'],
         },
     }, {
         'url': 'https://twitter.com/hlo_again/status/1599108751385972737/video/2',
@@ -995,13 +998,13 @@ class TwitterIE(TwitterBaseIE):
             'repost_count': int,
             'duration': 9.531,
             'comment_count': int,
-            'view_count': int,
             'upload_date': '20221203',
             'age_limit': 0,
             'timestamp': 1670092210.0,
             'tags': [],
             'uploader': '\u06ea',
             'description': '\U0001F48B https://t.co/bTj9Qz7vQP',
+            '_old_archive_ids': ['twitter 1599108751385972737'],
         },
         'params': {'noplaylist': True},
     }, {
@@ -1012,7 +1015,6 @@ class TwitterIE(TwitterBaseIE):
             'ext': 'mp4',
             'uploader_url': 'https://twitter.com/MunTheShinobi',
             'description': 'This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525 https://t.co/cNsA0MoOml',
-            'view_count': int,
             'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1600009362759733248/pu/img/XVhFQivj75H_YxxV.jpg?name=orig',
             'age_limit': 0,
             'uploader': 'Mün',
@@ -1025,6 +1027,7 @@ class TwitterIE(TwitterBaseIE):
             'uploader_id': 'MunTheShinobi',
             'duration': 139.987,
             'timestamp': 1670306984.0,
+            '_old_archive_ids': ['twitter 1600009574919962625'],
         },
     }, {
         # retweeted_status (private)
@@ -1068,8 +1071,8 @@ class TwitterIE(TwitterBaseIE):
             'thumbnail': r're:https://pbs\.twimg\.com/amplify_video_thumb/.+',
             'like_count': int,
             'repost_count': int,
-            'view_count': int,
             'comment_count': int,
+            '_old_archive_ids': ['twitter 1695424220702888009'],
         },
     }, {
         # retweeted_status w/ legacy API
@@ -1091,18 +1094,24 @@ class TwitterIE(TwitterBaseIE):
             'thumbnail': r're:https://pbs\.twimg\.com/amplify_video_thumb/.+',
             'like_count': int,
             'repost_count': int,
+            '_old_archive_ids': ['twitter 1695424220702888009'],
         },
         'params': {'extractor_args': {'twitter': {'api': ['legacy']}}},
     }, {
         # Broadcast embedded in tweet
-        'url': 'https://twitter.com/JessicaDobsonWX/status/1693057346933600402',
+        'url': 'https://twitter.com/JessicaDobsonWX/status/1731121063248175384',
         'info_dict': {
-            'id': '1yNGaNLjEblJj',
+            'id': '1rmxPMjLzAXKN',
             'ext': 'mp4',
-            'title': 'Jessica Dobson - WAVE Weather Now - Saturday 8/19/23 Update',
+            'title': 'WAVE Weather Now - Saturday 12/2/23 Update',
             'uploader': 'Jessica Dobson',
-            'uploader_id': '1DZEoDwDovRQa',
-            'thumbnail': r're:^https?://.*\.jpg',
+            'uploader_id': 'JessicaDobsonWX',
+            'uploader_url': 'https://twitter.com/JessicaDobsonWX',
+            'timestamp': 1701566398,
+            'upload_date': '20231203',
+            'live_status': 'was_live',
+            'thumbnail': r're:https://[^/]+pscp\.tv/.+\.jpg',
+            'concurrent_view_count': int,
             'view_count': int,
         },
         'add_ie': ['TwitterBroadcast'],
@@ -1125,6 +1134,30 @@ class TwitterIE(TwitterBaseIE):
         },
         'params': {'extractor_args': {'twitter': {'api': ['syndication']}}},
         'expected_warnings': ['Not all metadata'],
+    }, {
+        # "stale tweet" with typename "TweetWithVisibilityResults"
+        'url': 'https://twitter.com/RobertKennedyJr/status/1724884212803834154',
+        'md5': '62b1e11cdc2cdd0e527f83adb081f536',
+        'info_dict': {
+            'id': '1724883339285544960',
+            'ext': 'mp4',
+            'title': 'md5:cc56716f9ed0b368de2ba54c478e493c',
+            'description': 'md5:9dc14f5b0f1311fc7caf591ae253a164',
+            'display_id': '1724884212803834154',
+            'uploader': 'Robert F. Kennedy Jr',
+            'uploader_id': 'RobertKennedyJr',
+            'uploader_url': 'https://twitter.com/RobertKennedyJr',
+            'upload_date': '20231115',
+            'timestamp': 1700079417.0,
+            'duration': 341.048,
+            'thumbnail': r're:https://pbs\.twimg\.com/amplify_video_thumb/.+',
+            'tags': ['Kennedy24'],
+            'repost_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'age_limit': 0,
+            '_old_archive_ids': ['twitter 1724884212803834154'],
+        },
     }, {
         # onion route
         'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -1179,19 +1212,23 @@ def _graphql_to_legacy(self, data, twid):
         ), default={}, get_all=False) if self.is_logged_in else traverse_obj(
             data, ('tweetResult', 'result', {dict}), default={})
 
-        if result.get('__typename') not in ('Tweet', 'TweetTombstone', 'TweetUnavailable', None):
-            self.report_warning(f'Unknown typename: {result.get("__typename")}', twid, only_once=True)
+        typename = result.get('__typename')
+        if typename not in ('Tweet', 'TweetWithVisibilityResults', 'TweetTombstone', 'TweetUnavailable', None):
+            self.report_warning(f'Unknown typename: {typename}', twid, only_once=True)
 
         if 'tombstone' in result:
             cause = remove_end(traverse_obj(result, ('tombstone', 'text', 'text', {str})), '. Learn more')
             raise ExtractorError(f'Twitter API says: {cause or "Unknown error"}', expected=True)
-        elif result.get('__typename') == 'TweetUnavailable':
+        elif typename == 'TweetUnavailable':
             reason = result.get('reason')
             if reason == 'NsfwLoggedOut':
                 self.raise_login_required('NSFW tweet requires authentication')
             elif reason == 'Protected':
                 self.raise_login_required('You are not authorized to view this protected tweet')
             raise ExtractorError(reason or 'Requested tweet is unavailable', expected=True)
+        # Result for "stale tweet" needs additional transformation
+        elif typename == 'TweetWithVisibilityResults':
+            result = traverse_obj(result, ('tweet', {dict})) or {}
 
         status = result.get('legacy', {})
         status.update(traverse_obj(result, {
@@ -1377,7 +1414,7 @@ def add_thumbnail(name, size):
                 'formats': formats,
                 'subtitles': subtitles,
                 'thumbnails': thumbnails,
-                'view_count': traverse_obj(media, ('mediaStats', 'viewCount', {int_or_none})),
+                'view_count': traverse_obj(media, ('mediaStats', 'viewCount', {int_or_none})),  # No longer available
                 'duration': float_or_none(traverse_obj(media, ('video_info', 'duration_millis')), 1000),
                 # The codec of http formats are unknown
                 '_format_sort_fields': ('res', 'br', 'size', 'proto'),

From db8b4edc7d0bd27da462f6fe82ff6e13e3d68a04 Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Tue, 19 Dec 2023 22:21:47 +0800
Subject: [PATCH 740/871] [ie/JoqrAg] Add extractor (#8384)

Authored by: pzhlkj6612
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/joqrag.py      | 112 ++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 yt_dlp/extractor/joqrag.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 572d79fba..d5f030c6b 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -865,6 +865,7 @@
 )
 from .jove import JoveIE
 from .joj import JojIE
+from .joqrag import JoqrAgIE
 from .jstream import JStreamIE
 from .jtbc import (
     JTBCIE,
diff --git a/yt_dlp/extractor/joqrag.py b/yt_dlp/extractor/joqrag.py
new file mode 100644
index 000000000..3bb28af94
--- /dev/null
+++ b/yt_dlp/extractor/joqrag.py
@@ -0,0 +1,112 @@
+import datetime
+import urllib.parse
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    datetime_from_str,
+    unified_timestamp,
+    urljoin,
+)
+
+
+class JoqrAgIE(InfoExtractor):
+    IE_DESC = '超!A&G+ 文化放送 (f.k.a. AGQR) Nippon Cultural Broadcasting, Inc. (JOQR)'
+    _VALID_URL = [r'https?://www\.uniqueradio\.jp/agplayer5/(?:player|inc-player-hls)\.php',
+                  r'https?://(?:www\.)?joqr\.co\.jp/ag/',
+                  r'https?://(?:www\.)?joqr\.co\.jp/qr/ag(?:daily|regular)program/?(?:$|[#?])']
+    _TESTS = [{
+        'url': 'https://www.uniqueradio.jp/agplayer5/player.php',
+        'info_dict': {
+            'id': 'live',
+            'title': str,
+            'channel': '超!A&G+',
+            'description': str,
+            'live_status': 'is_live',
+            'release_timestamp': int,
+        },
+        'params': {
+            'skip_download': True,
+            'ignore_no_formats_error': True,
+        },
+    }, {
+        'url': 'https://www.uniqueradio.jp/agplayer5/inc-player-hls.php',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.joqr.co.jp/ag/article/103760/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.joqr.co.jp/qr/agdailyprogram/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.joqr.co.jp/qr/agregularprogram/',
+        'only_matching': True,
+    }]
+
+    def _extract_metadata(self, variable, html):
+        return clean_html(urllib.parse.unquote_plus(self._search_regex(
+            rf'var\s+{variable}\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
+            html, 'metadata', group='value', default=''))) or None
+
+    def _extract_start_timestamp(self, video_id, is_live):
+        def extract_start_time_from(date_str):
+            dt = datetime_from_str(date_str) + datetime.timedelta(hours=9)
+            date = dt.strftime('%Y%m%d')
+            start_time = self._search_regex(
+                r'<h3[^>]+\bclass="dailyProgram-itemHeaderTime"[^>]*>[\s\d:]+–\s*(\d{1,2}:\d{1,2})',
+                self._download_webpage(
+                    f'https://www.joqr.co.jp/qr/agdailyprogram/?date={date}', video_id,
+                    note=f'Downloading program list of {date}', fatal=False,
+                    errnote=f'Failed to download program list of {date}') or '',
+                'start time', default=None)
+            if start_time:
+                return unified_timestamp(f'{dt.strftime("%Y/%m/%d")} {start_time} +09:00')
+            return None
+
+        start_timestamp = extract_start_time_from('today')
+        if not start_timestamp:
+            return None
+
+        if not is_live or start_timestamp < datetime_from_str('now').timestamp():
+            return start_timestamp
+        else:
+            return extract_start_time_from('yesterday')
+
+    def _real_extract(self, url):
+        video_id = 'live'
+
+        metadata = self._download_webpage(
+            'https://www.uniqueradio.jp/aandg', video_id,
+            note='Downloading metadata', errnote='Failed to download metadata')
+        title = self._extract_metadata('Program_name', metadata)
+
+        if title == '放送休止':
+            formats = []
+            live_status = 'is_upcoming'
+            release_timestamp = self._extract_start_timestamp(video_id, False)
+            msg = 'This stream is not currently live'
+            if release_timestamp:
+                msg += (' and will start at '
+                        + datetime.datetime.fromtimestamp(release_timestamp).strftime('%Y-%m-%d %H:%M:%S'))
+            self.raise_no_formats(msg, expected=True)
+        else:
+            m3u8_path = self._search_regex(
+                r'<source\s[^>]*\bsrc="([^"]+)"',
+                self._download_webpage(
+                    'https://www.uniqueradio.jp/agplayer5/inc-player-hls.php', video_id,
+                    note='Downloading player data', errnote='Failed to download player data'),
+                'm3u8 url')
+            formats = self._extract_m3u8_formats(
+                urljoin('https://www.uniqueradio.jp/', m3u8_path), video_id)
+            live_status = 'is_live'
+            release_timestamp = self._extract_start_timestamp(video_id, True)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'channel': '超!A&G+',
+            'description': self._extract_metadata('Program_text', metadata),
+            'formats': formats,
+            'live_status': live_status,
+            'release_timestamp': release_timestamp,
+        }

From 196eb0fe77b78e2e5ca02c506c3837c2b1a7964c Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Wed, 20 Dec 2023 19:15:38 +1300
Subject: [PATCH 741/871] [networking] Strip whitespace around header values
 (#8802)

Fixes https://github.com/yt-dlp/yt-dlp/issues/8729
Authored by: coletdjnz
---
 test/test_utils.py         | 5 +++++
 yt_dlp/utils/networking.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 100f11788..6c8571f98 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2370,6 +2370,11 @@ def test_http_header_dict(self):
         headers4 = HTTPHeaderDict({'ytdl-test': 'data;'})
         self.assertEqual(set(headers4.items()), {('Ytdl-Test', 'data;')})
 
+        # common mistake: strip whitespace from values
+        # https://github.com/yt-dlp/yt-dlp/issues/8729
+        headers5 = HTTPHeaderDict({'ytdl-test': ' data; '})
+        self.assertEqual(set(headers5.items()), {('Ytdl-Test', 'data;')})
+
     def test_extract_basic_auth(self):
         assert extract_basic_auth('http://:foo.bar') == ('http://:foo.bar', None)
         assert extract_basic_auth('http://foo.bar') == ('http://foo.bar', None)
diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py
index ed0250011..4b73252cb 100644
--- a/yt_dlp/utils/networking.py
+++ b/yt_dlp/utils/networking.py
@@ -67,7 +67,7 @@ def __init__(self, *args, **kwargs):
     def __setitem__(self, key, value):
         if isinstance(value, bytes):
             value = value.decode('latin-1')
-        super().__setitem__(key.title(), str(value))
+        super().__setitem__(key.title(), str(value).strip())
 
     def __getitem__(self, key):
         return super().__getitem__(key.title())

From 37755a037e612bfc608c3d4722e8ef2ce6a022ee Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Wed, 20 Dec 2023 13:03:54 -0600
Subject: [PATCH 742/871] [test:networking] Update tests for OpenSSL 3.2
 (#8814)

Authored by: bashonly
---
 test/test_networking.py | 2 +-
 test/test_websockets.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index 64af6e459..dc60ca699 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -328,7 +328,7 @@ def test_ssl_error(self, handler):
         https_server_thread.start()
 
         with handler(verify=False) as rh:
-            with pytest.raises(SSLError, match='sslv3 alert handshake failure') as exc_info:
+            with pytest.raises(SSLError, match=r'ssl(?:v3|/tls) alert handshake failure') as exc_info:
                 validate_and_send(rh, Request(f'https://127.0.0.1:{https_port}/headers'))
             assert not issubclass(exc_info.type, CertificateVerifyError)
 
diff --git a/test/test_websockets.py b/test/test_websockets.py
index 39d3c7d72..af6142ea3 100644
--- a/test/test_websockets.py
+++ b/test/test_websockets.py
@@ -148,7 +148,7 @@ def test_verify_cert(self, handler):
     @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
     def test_ssl_error(self, handler):
         with handler(verify=False) as rh:
-            with pytest.raises(SSLError, match='sslv3 alert handshake failure') as exc_info:
+            with pytest.raises(SSLError, match=r'ssl(?:v3|/tls) alert handshake failure') as exc_info:
                 validate_and_send(rh, Request(self.bad_wss_host))
             assert not issubclass(exc_info.type, CertificateVerifyError)
 

From 19741ab8a401ec64d5e84fdbfcfb141d105e7bc8 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Thu, 21 Dec 2023 14:46:00 -0600
Subject: [PATCH 743/871] [ie/bbc] Fix JSON parsing bug

Authored by: bashonly
---
 yt_dlp/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/bbc.py b/yt_dlp/extractor/bbc.py
index d1d6e04fa..c94184bf0 100644
--- a/yt_dlp/extractor/bbc.py
+++ b/yt_dlp/extractor/bbc.py
@@ -1188,7 +1188,7 @@ def _real_extract(self, url):
         if initial_data is None:
             initial_data = self._search_regex(
                 r'window\.__INITIAL_DATA__\s*=\s*({.+?})\s*;', webpage,
-                'preload state', default={})
+                'preload state', default='{}')
         else:
             initial_data = self._parse_json(initial_data or '"{}"', playlist_id, fatal=False)
         initial_data = self._parse_json(initial_data, playlist_id, fatal=False)

From c919b68f7e79ea5010f75f648d3c9e45405a8011 Mon Sep 17 00:00:00 2001
From: barsnick <barsnick@users.noreply.github.com>
Date: Thu, 21 Dec 2023 21:47:32 +0100
Subject: [PATCH 744/871] [ie/bbc] Extract more formats (#8321)

Closes #4902
Authored by: barsnick, dirkf
---
 yt_dlp/extractor/bbc.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/bbc.py b/yt_dlp/extractor/bbc.py
index c94184bf0..015af9e1d 100644
--- a/yt_dlp/extractor/bbc.py
+++ b/yt_dlp/extractor/bbc.py
@@ -317,16 +317,25 @@ def _raise_extractor_error(self, media_selection_error):
 
     def _download_media_selector(self, programme_id):
         last_exception = None
+        formats, subtitles = [], {}
         for media_set in self._MEDIA_SETS:
             try:
-                return self._download_media_selector_url(
+                fmts, subs = self._download_media_selector_url(
                     self._MEDIA_SELECTOR_URL_TEMPL % (media_set, programme_id), programme_id)
+                formats.extend(fmts)
+                if subs:
+                    self._merge_subtitles(subs, target=subtitles)
             except BBCCoUkIE.MediaSelectionError as e:
                 if e.id in ('notukerror', 'geolocation', 'selectionunavailable'):
                     last_exception = e
                     continue
                 self._raise_extractor_error(e)
-        self._raise_extractor_error(last_exception)
+        if last_exception:
+            if formats or subtitles:
+                self.report_warning(f'{self.IE_NAME} returned error: {last_exception.id}')
+            else:
+                self._raise_extractor_error(last_exception)
+        return formats, subtitles
 
     def _download_media_selector_url(self, url, programme_id=None):
         media_selection = self._download_json(

From 632b8ee54eb2df8ac6e20746a0bd95b7ebb053aa Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 21 Dec 2023 15:06:26 -0600
Subject: [PATCH 745/871] [core] Release workflow and Updater cleanup (#8640)

- Only use trusted publishing with PyPI and remove support for PyPI tokens from release workflow
- Clean up improper actions syntax in the build workflow inputs
- Refactor Updater to allow for consistent unit testing with `UPDATE_SOURCES`

Authored by: bashonly
---
 .github/workflows/build.yml   |  8 ++++----
 .github/workflows/release.yml | 24 +-----------------------
 test/test_update.py           |  9 +++++++++
 yt_dlp/update.py              |  7 ++++---
 4 files changed, 18 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d944659b8..036ce4348 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -80,12 +80,12 @@ on:
         default: true
         type: boolean
       origin:
-        description: .
+        description: Origin
         required: false
-        default: ''
+        default: 'current repo'
         type: choice
         options:
-        - ''
+        - 'current repo'
 
 permissions:
   contents: read
@@ -99,7 +99,7 @@ jobs:
       - name: Process origin
         id: process_origin
         run: |
-          echo "origin=${{ inputs.origin || github.repository }}" >> "$GITHUB_OUTPUT"
+          echo "origin=${{ inputs.origin == 'current repo' && github.repository || inputs.origin }}" | tee "$GITHUB_OUTPUT"
 
   unix:
     needs: process
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 84e892ffe..69b5e3152 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -64,7 +64,6 @@ jobs:
       target_tag: ${{ steps.setup_variables.outputs.target_tag }}
       pypi_project: ${{ steps.setup_variables.outputs.pypi_project }}
       pypi_suffix: ${{ steps.setup_variables.outputs.pypi_suffix }}
-      pypi_token: ${{ steps.setup_variables.outputs.pypi_token }}
       head_sha: ${{ steps.get_target.outputs.head_sha }}
 
     steps:
@@ -153,7 +152,6 @@ jobs:
               ${{ !!secrets[format('{0}_archive_repo_token', env.target_repo)] }} || fallback_token
               pypi_project='${{ vars[format('{0}_pypi_project', env.target_repo)] }}'
               pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.target_repo)] }}'
-              ${{ !secrets[format('{0}_pypi_token', env.target_repo)] }} || pypi_token='${{ env.target_repo }}_pypi_token'
             fi
           else
             target_tag="${source_tag:-${version}}"
@@ -163,7 +161,6 @@ jobs:
               ${{ !!secrets[format('{0}_archive_repo_token', env.source_repo)] }} || fallback_token
               pypi_project='${{ vars[format('{0}_pypi_project', env.source_repo)] }}'
               pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.source_repo)] }}'
-              ${{ !secrets[format('{0}_pypi_token', env.source_repo)] }} || pypi_token='${{ env.source_repo }}_pypi_token'
             else
               target_repo='${{ github.repository }}'
             fi
@@ -172,13 +169,6 @@ jobs:
           if [[ "${target_repo}" == '${{ github.repository }}' ]] && ${{ !inputs.prerelease }}; then
             pypi_project='${{ vars.PYPI_PROJECT }}'
           fi
-          if [[ -z "${pypi_token}" && "${pypi_project}" ]]; then
-            if ${{ !secrets.PYPI_TOKEN }}; then
-              pypi_token=OIDC
-            else
-              pypi_token=PYPI_TOKEN
-            fi
-          fi
 
           echo "::group::Output variables"
           cat << EOF | tee -a "$GITHUB_OUTPUT"
@@ -189,7 +179,6 @@ jobs:
           target_tag=${target_tag}
           pypi_project=${pypi_project}
           pypi_suffix=${pypi_suffix}
-          pypi_token=${pypi_token}
           EOF
           echo "::endgroup::"
 
@@ -286,18 +275,7 @@ jobs:
           python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
           python setup.py sdist bdist_wheel
 
-      - name: Publish to PyPI via token
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets[needs.prepare.outputs.pypi_token] }}
-        if: |
-          needs.prepare.outputs.pypi_token != 'OIDC' && env.TWINE_PASSWORD
-        run: |
-          twine upload dist/*
-
-      - name: Publish to PyPI via trusted publishing
-        if: |
-          needs.prepare.outputs.pypi_token == 'OIDC'
+      - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           verbose: true
diff --git a/test/test_update.py b/test/test_update.py
index 2a5647e44..a5a388c10 100644
--- a/test/test_update.py
+++ b/test/test_update.py
@@ -11,6 +11,14 @@
 from test.helper import FakeYDL, report_warning
 from yt_dlp.update import Updater, UpdateInfo
 
+
+# XXX: Keep in sync with yt_dlp.update.UPDATE_SOURCES
+TEST_UPDATE_SOURCES = {
+    'stable': 'yt-dlp/yt-dlp',
+    'nightly': 'yt-dlp/yt-dlp-nightly-builds',
+    'master': 'yt-dlp/yt-dlp-master-builds',
+}
+
 TEST_API_DATA = {
     'yt-dlp/yt-dlp/latest': {
         'tag_name': '2023.12.31',
@@ -104,6 +112,7 @@ class FakeUpdater(Updater):
 
     _channel = 'stable'
     _origin = 'yt-dlp/yt-dlp'
+    _update_sources = TEST_UPDATE_SOURCES
 
     def _download_update_spec(self, *args, **kwargs):
         return TEST_LOCKFILE_ACTUAL
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index f99583b08..ba7eadf81 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -206,13 +206,14 @@ class Updater:
     # XXX: use class variables to simplify testing
     _channel = CHANNEL
     _origin = ORIGIN
+    _update_sources = UPDATE_SOURCES
 
     def __init__(self, ydl, target: str | None = None):
         self.ydl = ydl
         # For backwards compat, target needs to be treated as if it could be None
         self.requested_channel, sep, self.requested_tag = (target or self._channel).rpartition('@')
         # Check if requested_tag is actually the requested repo/channel
-        if not sep and ('/' in self.requested_tag or self.requested_tag in UPDATE_SOURCES):
+        if not sep and ('/' in self.requested_tag or self.requested_tag in self._update_sources):
             self.requested_channel = self.requested_tag
             self.requested_tag: str = None  # type: ignore (we set it later)
         elif not self.requested_channel:
@@ -237,11 +238,11 @@ def __init__(self, ydl, target: str | None = None):
                 self._block_restart('Automatically restarting into custom builds is disabled for security reasons')
         else:
             # Check if requested_channel resolves to a known repository or else raise
-            self.requested_repo = UPDATE_SOURCES.get(self.requested_channel)
+            self.requested_repo = self._update_sources.get(self.requested_channel)
             if not self.requested_repo:
                 self._report_error(
                     f'Invalid update channel {self.requested_channel!r} requested. '
-                    f'Valid channels are {", ".join(UPDATE_SOURCES)}', True)
+                    f'Valid channels are {", ".join(self._update_sources)}', True)
 
         self._identifier = f'{detect_variant()} {system_identifier()}'
 

From bc4ab17b38f01000d99c5c2bedec89721fee65ec Mon Sep 17 00:00:00 2001
From: barsnick <barsnick@users.noreply.github.com>
Date: Fri, 22 Dec 2023 02:32:29 +0100
Subject: [PATCH 746/871] [cleanup] Fix spelling of `IE_NAME` (#8810)

Authored by: barsnick
---
 yt_dlp/extractor/iheart.py | 2 +-
 yt_dlp/extractor/kinja.py  | 2 +-
 yt_dlp/extractor/nba.py    | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/iheart.py b/yt_dlp/extractor/iheart.py
index 2c6a5b6a1..fb6f51e2c 100644
--- a/yt_dlp/extractor/iheart.py
+++ b/yt_dlp/extractor/iheart.py
@@ -23,7 +23,7 @@ def _extract_episode(self, episode):
 
 
 class IHeartRadioIE(IHeartRadioBaseIE):
-    IENAME = 'iheartradio'
+    IE_NAME = 'iheartradio'
     _VALID_URL = r'(?:https?://(?:www\.)?iheart\.com/podcast/[^/]+/episode/(?P<display_id>[^/?&#]+)-|iheartradio:)(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.iheart.com/podcast/105-behind-the-bastards-29236323/episode/part-one-alexander-lukashenko-the-dictator-70346499/?embed=true',
diff --git a/yt_dlp/extractor/kinja.py b/yt_dlp/extractor/kinja.py
index a225d0a0d..f4e5c4c47 100644
--- a/yt_dlp/extractor/kinja.py
+++ b/yt_dlp/extractor/kinja.py
@@ -12,7 +12,7 @@
 
 
 class KinjaEmbedIE(InfoExtractor):
-    IENAME = 'kinja:embed'
+    IE_NAME = 'kinja:embed'
     _DOMAIN_REGEX = r'''(?:[^.]+\.)?
         (?:
             avclub|
diff --git a/yt_dlp/extractor/nba.py b/yt_dlp/extractor/nba.py
index d8fc82488..81d11e3a5 100644
--- a/yt_dlp/extractor/nba.py
+++ b/yt_dlp/extractor/nba.py
@@ -97,7 +97,7 @@ def _extract_video(self, filter_key, filter_value):
 
 
 class NBAWatchEmbedIE(NBAWatchBaseIE):
-    IENAME = 'nba:watch:embed'
+    IE_NAME = 'nba:watch:embed'
     _VALID_URL = NBAWatchBaseIE._VALID_URL_BASE + r'embed\?.*?\bid=(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://watch.nba.com/embed?id=659395',
@@ -339,7 +339,7 @@ def _real_extract(self, url):
 
 
 class NBAEmbedIE(NBABaseIE):
-    IENAME = 'nba:embed'
+    IE_NAME = 'nba:embed'
     _VALID_URL = r'https?://secure\.nba\.com/assets/amp/include/video/(?:topI|i)frame\.html\?.*?\bcontentId=(?P<id>[^?#&]+)'
     _TESTS = [{
         'url': 'https://secure.nba.com/assets/amp/include/video/topIframe.html?contentId=teams/bulls/2020/12/04/3478774/1607105587854-20201204_SCHEDULE_RELEASE_FINAL_DRUPAL-3478774&team=bulls&adFree=false&profile=71&videoPlayerName=TAMPCVP&baseUrl=&videoAdsection=nba.com_mobile_web_teamsites_chicagobulls&ampEnv=',
@@ -361,7 +361,7 @@ def _real_extract(self, url):
 
 
 class NBAIE(NBABaseIE):
-    IENAME = 'nba'
+    IE_NAME = 'nba'
     _VALID_URL = NBABaseIE._VALID_URL_BASE + '(?!%s)video/(?P<id>(?:[^/]+/)*[^/?#&]+)' % NBABaseIE._CHANNEL_PATH_REGEX
     _TESTS = [{
         'url': 'https://www.nba.com/bulls/video/teams/bulls/2020/12/04/3478774/1607105587854-20201204schedulereleasefinaldrupal-3478774',
@@ -388,7 +388,7 @@ def _extract_url_results(self, team, content_id):
 
 
 class NBAChannelIE(NBABaseIE):
-    IENAME = 'nba:channel'
+    IE_NAME = 'nba:channel'
     _VALID_URL = NBABaseIE._VALID_URL_BASE + '(?:%s)/(?P<id>[^/?#&]+)' % NBABaseIE._CHANNEL_PATH_REGEX
     _TESTS = [{
         'url': 'https://www.nba.com/blazers/video/channel/summer_league',

From 0d531c35eca4c2eb36e160530a7a333edbc727cc Mon Sep 17 00:00:00 2001
From: Nicolas Dato <67328748+nicodato@users.noreply.github.com>
Date: Fri, 22 Dec 2023 18:52:07 -0300
Subject: [PATCH 747/871] [ie/RudoVideo] Add extractor (#8664)

Authored by: nicodato
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/rudovideo.py   | 135 ++++++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+)
 create mode 100644 yt_dlp/extractor/rudovideo.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index d5f030c6b..5c34bb7f4 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1647,6 +1647,7 @@
     RumbleIE,
     RumbleChannelIE,
 )
+from .rudovideo import RudoVideoIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
diff --git a/yt_dlp/extractor/rudovideo.py b/yt_dlp/extractor/rudovideo.py
new file mode 100644
index 000000000..1b8595593
--- /dev/null
+++ b/yt_dlp/extractor/rudovideo.py
@@ -0,0 +1,135 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    js_to_json,
+    traverse_obj,
+    update_url_query,
+    url_or_none,
+)
+
+
+class RudoVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://rudo\.video/(?P<type>vod|podcast|live)/(?P<id>[^/?&#]+)'
+    _EMBED_REGEX = [r'<iframe[^>]+src=[\'"](?P<url>(?:https?:)//rudo\.video/(?:vod|podcast|live)/[^\'"]+)']
+    _TESTS = [{
+        'url': 'https://rudo.video/podcast/cz2wrUy8l0o',
+        'md5': '28ed82b477708dc5e12e072da2449221',
+        'info_dict': {
+            'id': 'cz2wrUy8l0o',
+            'title': 'Diego Cabot',
+            'ext': 'mp4',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/podcast/bQkt07',
+        'md5': '36b22a9863de0f47f00fc7532a32a898',
+        'info_dict': {
+            'id': 'bQkt07',
+            'title': 'Tubular Bells',
+            'ext': 'mp4',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/podcast/b42ZUznHX0',
+        'md5': 'b91c70d832938871367f8ad10c895821',
+        'info_dict': {
+            'id': 'b42ZUznHX0',
+            'title': 'Columna Ruperto Concha',
+            'ext': 'mp3',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/vod/bN5AaJ',
+        'md5': '01324a329227e2591530ecb4f555c881',
+        'info_dict': {
+            'id': 'bN5AaJ',
+            'title': 'Ucrania 19.03',
+            'creator': 'La Tercera',
+            'ext': 'mp4',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/live/bbtv',
+        'info_dict': {
+            'id': 'bbtv',
+            'ext': 'mp4',
+            'creator': 'BioBioTV',
+            'live_status': 'is_live',
+            'title': r're:^LIVE BBTV\s\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}$',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/live/c13',
+        'info_dict': {
+            'id': 'c13',
+            'title': 'CANAL13',
+            'ext': 'mp4',
+        },
+        'skip': 'Geo-restricted to Chile',
+    }, {
+        'url': 'https://rudo.video/live/t13-13cl',
+        'info_dict': {
+            'id': 't13-13cl',
+            'title': 'T13',
+            'ext': 'mp4',
+        },
+        'skip': 'Geo-restricted to Chile',
+    }]
+
+    def _real_extract(self, url):
+        video_id, type_ = self._match_valid_url(url).group('id', 'type')
+        is_live = type_ == 'live'
+
+        webpage = self._download_webpage(url, video_id)
+        if 'Streaming is not available in your area' in webpage:
+            self.raise_geo_restricted()
+
+        media_url = (
+            self._search_regex(
+                r'var\s+streamURL\s*=\s*[\'"]([^?\'"]+)', webpage, 'stream url', default=None)
+            # Source URL must be used only if streamURL is unavailable
+            or self._search_regex(
+                r'<source[^>]+src=[\'"]([^\'"]+)', webpage, 'source url', default=None))
+        if not media_url:
+            youtube_url = self._search_regex(r'file:\s*[\'"]((?:https?:)//(?:www\.)?youtube\.com[^\'"]+)',
+                                             webpage, 'youtube url', default=None)
+            if youtube_url:
+                return self.url_result(youtube_url, 'Youtube')
+            raise ExtractorError('Unable to extract stream url')
+
+        token_array = self._search_json(
+            r'<script>var\s+_\$_[a-zA-Z0-9]+\s*=', webpage, 'access token array', video_id,
+            contains_pattern=r'\[(?s:.+)\]', default=None, transform_source=js_to_json)
+        if token_array:
+            token_url = traverse_obj(token_array, (..., {url_or_none}), get_all=False)
+            if not token_url:
+                raise ExtractorError('Invalid access token array')
+            access_token = self._download_json(
+                token_url, video_id, note='Downloading access token')['data']['authToken']
+            media_url = update_url_query(media_url, {'auth-token': access_token})
+
+        ext = determine_ext(media_url)
+        if ext == 'm3u8':
+            formats = self._extract_m3u8_formats(media_url, video_id, live=is_live)
+        elif ext == 'mp3':
+            formats = [{
+                'url': media_url,
+                'vcodec': 'none',
+            }]
+        else:
+            formats = [{'url': media_url}]
+
+        return {
+            'id': video_id,
+            'title': (self._search_regex(r'var\s+titleVideo\s*=\s*[\'"]([^\'"]+)',
+                                         webpage, 'title', default=None)
+                      or self._og_search_title(webpage)),
+            'creator': self._search_regex(r'var\s+videoAuthor\s*=\s*[\'"]([^?\'"]+)',
+                                          webpage, 'videoAuthor', default=None),
+            'thumbnail': (self._search_regex(r'var\s+posterIMG\s*=\s*[\'"]([^?\'"]+)',
+                                             webpage, 'thumbnail', default=None)
+                          or self._og_search_thumbnail(webpage)),
+            'formats': formats,
+            'is_live': is_live,
+        }

From 13b3cb3c2b7169a1e17d6fc62593bf744170521c Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 24 Dec 2023 00:11:10 +0100
Subject: [PATCH 748/871] [ci] Run core tests only for core changes (#8841)

Authored by: Grub4K
---
 .github/workflows/core.yml | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index b1ae4ae76..b22adb1b9 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -1,5 +1,25 @@
 name: Core Tests
-on: [push, pull_request]
+on:
+  push:
+    paths:
+      - .github/**
+      - devscripts/**
+      - test/**
+      - yt_dlp/**.py
+      - '!yt_dlp/extractor/*.py'
+      - yt_dlp/extractor/__init__.py
+      - yt_dlp/extractor/common.py
+      - yt_dlp/extractor/extractors.py
+  pull_request:
+    paths:
+      - .github/**
+      - devscripts/**
+      - test/**
+      - yt_dlp/**.py
+      - '!yt_dlp/extractor/*.py'
+      - yt_dlp/extractor/__init__.py
+      - yt_dlp/extractor/common.py
+      - yt_dlp/extractor/extractors.py
 permissions:
   contents: read
 

From f45c4efcd928a173e1300a8f1ce4258e70c969b1 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 24 Dec 2023 10:33:16 -0600
Subject: [PATCH 749/871] [ie/litv] Fix premium content extraction (#8842)

Closes #8654
Authored by: bashonly
---
 yt_dlp/extractor/litv.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/litv.py b/yt_dlp/extractor/litv.py
index 2c7c7175e..1003fb2fd 100644
--- a/yt_dlp/extractor/litv.py
+++ b/yt_dlp/extractor/litv.py
@@ -6,6 +6,7 @@
     int_or_none,
     smuggle_url,
     traverse_obj,
+    try_call,
     unsmuggle_url,
 )
 
@@ -96,13 +97,22 @@ def _real_extract(self, url):
             r'uiHlsUrl\s*=\s*testBackendData\(([^;]+)\);',
             webpage, 'video data', default='{}'), video_id)
         if not video_data:
-            payload = {
-                'assetId': program_info['assetId'],
-                'watchDevices': program_info['watchDevices'],
-                'contentType': program_info['contentType'],
-            }
+            payload = {'assetId': program_info['assetId']}
+            puid = try_call(lambda: self._get_cookies('https://www.litv.tv/')['PUID'].value)
+            if puid:
+                payload.update({
+                    'type': 'auth',
+                    'puid': puid,
+                })
+                endpoint = 'getUrl'
+            else:
+                payload.update({
+                    'watchDevices': program_info['watchDevices'],
+                    'contentType': program_info['contentType'],
+                })
+                endpoint = 'getMainUrlNoAuth'
             video_data = self._download_json(
-                'https://www.litv.tv/vod/ajax/getMainUrlNoAuth', video_id,
+                f'https://www.litv.tv/vod/ajax/{endpoint}', video_id,
                 data=json.dumps(payload).encode('utf-8'),
                 headers={'Content-Type': 'application/json'})
 

From 50eaea9fd7787546b53660e736325fa31c77765d Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 24 Dec 2023 10:40:03 -0600
Subject: [PATCH 750/871] [ie/instagram] Fix stories extraction (#8843)

Closes #8290
Authored by: bashonly
---
 yt_dlp/extractor/instagram.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/instagram.py b/yt_dlp/extractor/instagram.py
index bfc4b7b88..dbaa332c2 100644
--- a/yt_dlp/extractor/instagram.py
+++ b/yt_dlp/extractor/instagram.py
@@ -10,6 +10,7 @@
     ExtractorError,
     decode_base_n,
     encode_base_n,
+    filter_dict,
     float_or_none,
     format_field,
     get_element_by_attribute,
@@ -703,28 +704,31 @@ def _real_extract(self, url):
         user_info = self._search_json(r'"user":', story_info, 'user info', story_id, fatal=False)
         if not user_info:
             self.raise_login_required('This content is unreachable')
-        user_id = user_info.get('id')
 
+        user_id = traverse_obj(user_info, 'pk', 'id', expected_type=str)
         story_info_url = user_id if username != 'highlights' else f'highlight:{story_id}'
+        if not story_info_url:  # user id is only mandatory for non-highlights
+            raise ExtractorError('Unable to extract user id')
+
         videos = traverse_obj(self._download_json(
             f'{self._API_BASE_URL}/feed/reels_media/?reel_ids={story_info_url}',
             story_id, errnote=False, fatal=False, headers=self._API_HEADERS), 'reels')
         if not videos:
             self.raise_login_required('You need to log in to access this content')
 
-        full_name = traverse_obj(videos, (f'highlight:{story_id}', 'user', 'full_name'), (str(user_id), 'user', 'full_name'))
+        full_name = traverse_obj(videos, (f'highlight:{story_id}', 'user', 'full_name'), (user_id, 'user', 'full_name'))
         story_title = traverse_obj(videos, (f'highlight:{story_id}', 'title'))
         if not story_title:
             story_title = f'Story by {username}'
 
-        highlights = traverse_obj(videos, (f'highlight:{story_id}', 'items'), (str(user_id), 'items'))
+        highlights = traverse_obj(videos, (f'highlight:{story_id}', 'items'), (user_id, 'items'))
         info_data = []
         for highlight in highlights:
             highlight_data = self._extract_product(highlight)
             if highlight_data.get('formats'):
                 info_data.append({
-                    **highlight_data,
                     'uploader': full_name,
                     'uploader_id': user_id,
+                    **filter_dict(highlight_data),
                 })
         return self.playlist_result(info_data, playlist_id=story_id, playlist_title=story_title)

From e7d22348e77367740da78a3db27167ecf894b7c9 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 24 Dec 2023 10:40:50 -0600
Subject: [PATCH 751/871] [ie/twitter] Prioritize m3u8 formats (#8826)

Closes #8117
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 932b478d4..5e8504aa1 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1416,8 +1416,8 @@ def add_thumbnail(name, size):
                 'thumbnails': thumbnails,
                 'view_count': traverse_obj(media, ('mediaStats', 'viewCount', {int_or_none})),  # No longer available
                 'duration': float_or_none(traverse_obj(media, ('video_info', 'duration_millis')), 1000),
-                # The codec of http formats are unknown
-                '_format_sort_fields': ('res', 'br', 'size', 'proto'),
+                # Prioritize m3u8 formats for compat, see https://github.com/yt-dlp/yt-dlp/issues/8117
+                '_format_sort_fields': ('res', 'proto:m3u8', 'br', 'size'),  # http format codec is unknown
             }
 
         def extract_from_card_info(card):

From 116c268438ea4d3738f6fa502c169081ca8f0ee7 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 24 Dec 2023 10:41:28 -0600
Subject: [PATCH 752/871] [ie/twitter] Work around API rate-limit (#8825)

Closes #8762
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 73 +++++++++++++++++++++----------------
 1 file changed, 42 insertions(+), 31 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 5e8504aa1..c3a6e406c 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -10,6 +10,7 @@
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
 )
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     dict_get,
@@ -1317,41 +1318,51 @@ def _build_graphql_query(self, media_id):
             }
         }
 
-    def _extract_status(self, twid):
-        if self.is_logged_in or self._selected_api == 'graphql':
-            status = self._graphql_to_legacy(self._call_graphql_api(self._GRAPHQL_ENDPOINT, twid), twid)
-
-        elif self._selected_api == 'legacy':
-            status = self._call_api(f'statuses/show/{twid}.json', twid, {
-                'cards_platform': 'Web-12',
-                'include_cards': 1,
-                'include_reply_count': 1,
-                'include_user_entities': 0,
-                'tweet_mode': 'extended',
+    def _call_syndication_api(self, twid):
+        self.report_warning(
+            'Not all metadata or media is available via syndication endpoint', twid, only_once=True)
+        status = self._download_json(
+            'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
+            headers={'User-Agent': 'Googlebot'}, query={
+                'id': twid,
+                # TODO: token = ((Number(twid) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '')
+                'token': ''.join(random.choices('123456789abcdefghijklmnopqrstuvwxyz', k=10)),
             })
+        if not status:
+            raise ExtractorError('Syndication endpoint returned empty JSON response')
+        # Transform the result so its structure matches that of legacy/graphql
+        media = []
+        for detail in traverse_obj(status, ((None, 'quoted_tweet'), 'mediaDetails', ..., {dict})):
+            detail['id_str'] = traverse_obj(detail, (
+                'video_info', 'variants', ..., 'url', {self._MEDIA_ID_RE.search}, 1), get_all=False) or twid
+            media.append(detail)
+        status['extended_entities'] = {'media': media}
 
-        elif self._selected_api == 'syndication':
-            self.report_warning(
-                'Not all metadata or media is available via syndication endpoint', twid, only_once=True)
-            status = self._download_json(
-                'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
-                headers={'User-Agent': 'Googlebot'}, query={
-                    'id': twid,
-                    # TODO: token = ((Number(twid) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '')
-                    'token': ''.join(random.choices('123456789abcdefghijklmnopqrstuvwxyz', k=10)),
+        return status
+
+    def _extract_status(self, twid):
+        if self._selected_api not in ('graphql', 'legacy', 'syndication'):
+            raise ExtractorError(f'{self._selected_api!r} is not a valid API selection', expected=True)
+
+        try:
+            if self.is_logged_in or self._selected_api == 'graphql':
+                status = self._graphql_to_legacy(self._call_graphql_api(self._GRAPHQL_ENDPOINT, twid), twid)
+            elif self._selected_api == 'legacy':
+                status = self._call_api(f'statuses/show/{twid}.json', twid, {
+                    'cards_platform': 'Web-12',
+                    'include_cards': 1,
+                    'include_reply_count': 1,
+                    'include_user_entities': 0,
+                    'tweet_mode': 'extended',
                 })
-            if not status:
-                raise ExtractorError('Syndication endpoint returned empty JSON response')
-            # Transform the result so its structure matches that of legacy/graphql
-            media = []
-            for detail in traverse_obj(status, ((None, 'quoted_tweet'), 'mediaDetails', ..., {dict})):
-                detail['id_str'] = traverse_obj(detail, (
-                    'video_info', 'variants', ..., 'url', {self._MEDIA_ID_RE.search}, 1), get_all=False) or twid
-                media.append(detail)
-            status['extended_entities'] = {'media': media}
+        except ExtractorError as e:
+            if not isinstance(e.cause, HTTPError) or not e.cause.status == 429:
+                raise
+            self.report_warning('Rate-limit exceeded; falling back to syndication endpoint')
+            status = self._call_syndication_api(twid)
 
-        else:
-            raise ExtractorError(f'"{self._selected_api}" is not a valid API selection', expected=True)
+        if self._selected_api == 'syndication':
+            status = self._call_syndication_api(twid)
 
         return traverse_obj(status, 'retweeted_status', None, expected_type=dict) or {}
 

From 00cdda4f6fe18712ced13dbc64b7ea10f323e268 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 24 Dec 2023 22:09:01 +0100
Subject: [PATCH 753/871] [core] Fix format selection parse error for CPython
 3.12 (#8797)

Authored by: Grub4K
---
 test/test_YoutubeDL.py |  2 ++
 yt_dlp/YoutubeDL.py    | 11 +++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 48c710e00..8bff08314 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -140,6 +140,8 @@ def test(inp, *expected, multi=False):
         test('example-with-dashes', 'example-with-dashes')
         test('all', '2', '47', '45', 'example-with-dashes', '35')
         test('mergeall', '2+47+45+example-with-dashes+35', multi=True)
+        # See: https://github.com/yt-dlp/yt-dlp/pulls/8797
+        test('7_a/worst', '35')
 
     def test_format_selection_audio(self):
         formats = [
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 0c07866e4..5e28fd0e2 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2465,9 +2465,16 @@ def final_selector(ctx):
                 return selector_function(ctx_copy)
             return final_selector
 
-        stream = io.BytesIO(format_spec.encode())
+        # HACK: Python 3.12 changed the underlying parser, rendering '7_a' invalid
+        #       Prefix numbers with random letters to avoid it being classified as a number
+        #       See: https://github.com/yt-dlp/yt-dlp/pulls/8797
+        # TODO: Implement parser not reliant on tokenize.tokenize
+        prefix = ''.join(random.choices(string.ascii_letters, k=32))
+        stream = io.BytesIO(re.sub(r'\d[_\d]*', rf'{prefix}\g<0>', format_spec).encode())
         try:
-            tokens = list(_remove_unused_ops(tokenize.tokenize(stream.readline)))
+            tokens = list(_remove_unused_ops(
+                token._replace(string=token.string.replace(prefix, ''))
+                for token in tokenize.tokenize(stream.readline)))
         except tokenize.TokenError:
             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
 

From 1f8bd8eba82ba10ddb49ee7cc0be4540dab103d5 Mon Sep 17 00:00:00 2001
From: Lars Strojny <lars@strojny.net>
Date: Sun, 24 Dec 2023 23:38:21 +0100
Subject: [PATCH 754/871] [ie/ARDBetaMediathek] Fix series extraction (#8687)

Closes #7666
Authored by: lstrojny
---
 yt_dlp/extractor/ard.py | 50 ++++++++++++++++++++++-------------------
 1 file changed, 27 insertions(+), 23 deletions(-)

diff --git a/yt_dlp/extractor/ard.py b/yt_dlp/extractor/ard.py
index ca1faa7d0..8ac926c91 100644
--- a/yt_dlp/extractor/ard.py
+++ b/yt_dlp/extractor/ard.py
@@ -292,7 +292,7 @@ class ARDIE(InfoExtractor):
     _TESTS = [{
         # available till 7.12.2023
         'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-video-424.html',
-        'md5': 'a438f671e87a7eba04000336a119ccc4',
+        'md5': '94812e6438488fb923c361a44469614b',
         'info_dict': {
             'id': 'maischberger-video-424',
             'display_id': 'maischberger-video-424',
@@ -403,26 +403,25 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
     _VALID_URL = r'''(?x)https://
         (?:(?:beta|www)\.)?ardmediathek\.de/
         (?:(?P<client>[^/]+)/)?
-        (?:player|live|video|(?P<playlist>sendung|sammlung))/
+        (?:player|live|video|(?P<playlist>sendung|serie|sammlung))/
         (?:(?P<display_id>(?(playlist)[^?#]+?|[^?#]+))/)?
         (?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)
         (?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
 
     _TESTS = [{
-        'url': 'https://www.ardmediathek.de/video/filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy8xZGY0ZGJmZS00ZWQwLTRmMGItYjhhYy0wOGQ4ZmYxNjVhZDI',
-        'md5': '3fd5fead7a370a819341129c8d713136',
+        'url': 'https://www.ardmediathek.de/video/filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen/Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
+        'md5': 'b6e8ab03f2bcc6e1f9e6cef25fcc03c4',
         'info_dict': {
-            'display_id': 'filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen',
-            'id': '12172961',
-            'title': 'Wolfsland - Die traurigen Schwestern',
-            'description': r're:^Als der Polizeiobermeister Raaben',
-            'duration': 5241,
-            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:efa186f7b0054957',
-            'timestamp': 1670710500,
-            'upload_date': '20221210',
+            'display_id': 'filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen',
+            'id': '12939099',
+            'title': 'Liebe auf vier Pfoten',
+            'description': r're:^Claudia Schmitt, Anwältin in Salzburg',
+            'duration': 5222,
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:aee7cbf8f06de976?w=960&ch=ae4d0f2ee47d8b9b',
+            'timestamp': 1701343800,
+            'upload_date': '20231130',
             'ext': 'mp4',
-            'age_limit': 12,
-            'episode': 'Wolfsland - Die traurigen Schwestern',
+            'episode': 'Liebe auf vier Pfoten',
             'series': 'Filme im MDR'
         },
     }, {
@@ -454,7 +453,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'duration': 915,
             'episode': 'tagesschau, 20:00 Uhr',
             'series': 'tagesschau',
-            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49',
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49?w=960&ch=ee69108ae344f678',
         },
     }, {
         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
@@ -475,6 +474,10 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         # playlist of type 'sendung'
         'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/',
         'only_matching': True,
+    }, {
+        # playlist of type 'serie'
+        'url': 'https://www.ardmediathek.de/serie/nachtstreife/staffel-1/Y3JpZDovL3N3ci5kZS9zZGIvc3RJZC8xMjQy/1',
+        'only_matching': True,
     }, {
         # playlist of type 'sammlung'
         'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
@@ -487,10 +490,11 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         'only_matching': True,
     }]
 
-    def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, pageNumber):
+    def _ARD_load_playlist_snippet(self, playlist_id, display_id, client, mode, page_number):
         """ Query the ARD server for playlist information
         and returns the data in "raw" format """
-        if mode == 'sendung':
+        assert mode in ('sendung', 'serie', 'sammlung')
+        if mode in ('sendung', 'serie'):
             graphQL = json.dumps({
                 'query': '''{
                     showPage(
@@ -507,7 +511,7 @@ def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, page
                             links { target { id href title } }
                             type
                         }
-                    }}''' % (client, playlist_id, pageNumber),
+                    }}''' % (client, playlist_id, page_number),
             }).encode()
         else:  # mode == 'sammlung'
             graphQL = json.dumps({
@@ -528,7 +532,7 @@ def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, page
                                 type
                             }
                         }
-                    }}''' % (client, playlist_id, pageNumber),
+                    }}''' % (client, playlist_id, page_number),
             }).encode()
         # Ressources for ARD graphQL debugging:
         # https://api-test.ardmediathek.de/public-gateway
@@ -538,7 +542,7 @@ def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, page
             data=graphQL,
             headers={'Content-Type': 'application/json'})['data']
         # align the structure of the returned data:
-        if mode == 'sendung':
+        if mode in ('sendung', 'serie'):
             show_page = show_page['showPage']
         else:  # mode == 'sammlung'
             show_page = show_page['morePage']['widget']
@@ -546,12 +550,12 @@ def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, page
 
     def _ARD_extract_playlist(self, url, playlist_id, display_id, client, mode):
         """ Collects all playlist entries and returns them as info dict.
-        Supports playlists of mode 'sendung' and 'sammlung', and also nested
-        playlists. """
+        Supports playlists of mode 'sendung', 'serie', and 'sammlung',
+        as well as nested playlists. """
         entries = []
         pageNumber = 0
         while True:  # iterate by pageNumber
-            show_page = self._ARD_load_playlist_snipped(
+            show_page = self._ARD_load_playlist_snippet(
                 playlist_id, display_id, client, mode, pageNumber)
             for teaser in show_page['teasers']:  # process playlist items
                 if '/compilation/' in teaser['links']['target']['href']:

From c39358a54bc6675ae0c50b81024e5a086e41656a Mon Sep 17 00:00:00 2001
From: kclauhk <78251477+kclauhk@users.noreply.github.com>
Date: Mon, 25 Dec 2023 06:43:35 +0800
Subject: [PATCH 755/871] [ie/Facebook] Fix Memories extraction (#8681)

- Support group /posts/ URLs
- Raise a proper error message if no formats are found

Closes #8669
Authored by: kclauhk
---
 yt_dlp/extractor/facebook.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 58162cc5f..a07a0d344 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -52,7 +52,7 @@ class FacebookIE(InfoExtractor):
                             )\?(?:.*?)(?:v|video_id|story_fbid)=|
                             [^/]+/videos/(?:[^/]+/)?|
                             [^/]+/posts/|
-                            groups/[^/]+/permalink/|
+                            groups/[^/]+/(?:permalink|posts)/|
                             watchparty/
                         )|
                     facebook:
@@ -232,6 +232,21 @@ class FacebookIE(InfoExtractor):
             'uploader_id': '100013949973717',
         },
         'skip': 'Requires logging in',
+    }, {
+        # data.node.comet_sections.content.story.attachments[].throwbackStyles.attachment_target_renderer.attachment.target.attachments[].styles.attachment.media
+        'url': 'https://www.facebook.com/groups/1645456212344334/posts/3737828833107051/',
+        'info_dict': {
+            'id': '1569199726448814',
+            'ext': 'mp4',
+            'title': 'Pence MUST GO!',
+            'description': 'Vickie Gentry shared a memory.',
+            'timestamp': 1511548260,
+            'upload_date': '20171124',
+            'uploader': 'Vickie Gentry',
+            'uploader_id': 'pfbid0FuZhHCeWDAxWxEbr3yKPFaRstXvRxgsp9uCPG6GjD4J2AitB35NUAuJ4Q75KcjiDl',
+            'thumbnail': r're:^https?://.*',
+            'duration': 148.435,
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -612,9 +627,11 @@ def parse_attachment(attachment, key='media'):
                 nodes = variadic(traverse_obj(data, 'nodes', 'node') or [])
                 attachments = traverse_obj(nodes, (
                     ..., 'comet_sections', 'content', 'story', (None, 'attached_story'), 'attachments',
-                    ..., ('styles', 'style_type_renderer'), 'attachment'), expected_type=dict) or []
+                    ..., ('styles', 'style_type_renderer', ('throwbackStyles', 'attachment_target_renderer')),
+                    'attachment', {dict}))
                 for attachment in attachments:
-                    ns = try_get(attachment, lambda x: x['all_subattachments']['nodes'], list) or []
+                    ns = traverse_obj(attachment, ('all_subattachments', 'nodes', ..., {dict}),
+                                      ('target', 'attachments', ..., 'styles', 'attachment', {dict}))
                     for n in ns:
                         parse_attachment(n)
                     parse_attachment(attachment)
@@ -637,7 +654,7 @@ def parse_attachment(attachment, key='media'):
                 if len(entries) > 1:
                     return self.playlist_result(entries, video_id)
 
-                video_info = entries[0]
+                video_info = entries[0] if entries else {'id': video_id}
                 webpage_info = extract_metadata(webpage)
                 # honor precise duration in video info
                 if video_info.get('duration'):

From 65de7d204ce88c0225df1321060304baab85dbd8 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 26 Dec 2023 01:40:24 +0100
Subject: [PATCH 756/871] Update to ytdl-commit-be008e6 (#8836)

- [utils] Make restricted filenames ignore some Unicode categories (by dirkf)
- [ie/telewebion] Fix extraction (by Grub4K)
- [ie/imgur] Overhaul extractor (by bashonly, Grub4K)
- [ie/EpidemicSound] Add extractor (by Grub4K)

Authored by: bashonly, dirkf, Grub4K

Co-authored-by: bashonly <bashonly@protonmail.com>
---
 yt_dlp/extractor/_extractors.py   |   1 +
 yt_dlp/extractor/epidemicsound.py | 107 +++++++++
 yt_dlp/extractor/imgur.py         | 364 ++++++++++++++++++++++++------
 yt_dlp/extractor/telewebion.py    | 157 +++++++++----
 yt_dlp/utils/_utils.py            |   2 +-
 5 files changed, 518 insertions(+), 113 deletions(-)
 create mode 100644 yt_dlp/extractor/epidemicsound.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 5c34bb7f4..62103f13c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -548,6 +548,7 @@
     EpiconIE,
     EpiconSeriesIE,
 )
+from .epidemicsound import EpidemicSoundIE
 from .eplus import EplusIbIE
 from .epoch import EpochIE
 from .eporner import EpornerIE
diff --git a/yt_dlp/extractor/epidemicsound.py b/yt_dlp/extractor/epidemicsound.py
new file mode 100644
index 000000000..0d81b11c8
--- /dev/null
+++ b/yt_dlp/extractor/epidemicsound.py
@@ -0,0 +1,107 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    orderedSet,
+    parse_iso8601,
+    parse_qs,
+    parse_resolution,
+    str_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class EpidemicSoundIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?epidemicsound\.com/track/(?P<id>[0-9a-zA-Z]+)'
+    _TESTS = [{
+        'url': 'https://www.epidemicsound.com/track/yFfQVRpSPz/',
+        'md5': 'd98ff2ddb49e8acab9716541cbc9dfac',
+        'info_dict': {
+            'id': '45014',
+            'display_id': 'yFfQVRpSPz',
+            'ext': 'mp3',
+            'title': 'Door Knock Door 1',
+            'alt_title': 'Door Knock Door 1',
+            'tags': ['foley', 'door', 'knock', 'glass', 'window', 'glass door knock'],
+            'categories': ['Misc. Door'],
+            'duration': 1,
+            'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/default-sfx/3000x3000.jpg',
+            'timestamp': 1415320353,
+            'upload_date': '20141107',
+        },
+    }, {
+        'url': 'https://www.epidemicsound.com/track/mj8GTTwsZd/',
+        'md5': 'c82b745890f9baf18dc2f8d568ee3830',
+        'info_dict': {
+            'id': '148700',
+            'display_id': 'mj8GTTwsZd',
+            'ext': 'mp3',
+            'title': 'Noplace',
+            'tags': ['liquid drum n bass', 'energetic'],
+            'categories': ['drum and bass'],
+            'duration': 237,
+            'timestamp': 1694426482,
+            'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/11138/3000x3000.jpg',
+            'upload_date': '20230911',
+            'release_timestamp': 1700535606,
+            'release_date': '20231121',
+        },
+    }]
+
+    @staticmethod
+    def _epidemic_parse_thumbnail(url: str):
+        if not url_or_none(url):
+            return None
+
+        return {
+            'url': url,
+            **(traverse_obj(url, ({parse_qs}, {
+                'width': ('width', 0, {int_or_none}),
+                'height': ('height', 0, {int_or_none}),
+            })) or parse_resolution(url)),
+        }
+
+    @staticmethod
+    def _epidemic_fmt_or_none(f):
+        if not f.get('format'):
+            f['format'] = f.get('format_id')
+        elif not f.get('format_id'):
+            f['format_id'] = f['format']
+        if not f['url'] or not f['format']:
+            return None
+        if f.get('format_note'):
+            f['format_note'] = f'track ID {f["format_note"]}'
+        if f['format'] != 'full':
+            f['preference'] = -2
+        return f
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json(f'https://www.epidemicsound.com/json/track/{video_id}', video_id)
+
+        thumbnails = traverse_obj(json_data, [('imageUrl', 'cover')])
+        thumb_base_url = traverse_obj(json_data, ('coverArt', 'baseUrl', {url_or_none}))
+        if thumb_base_url:
+            thumbnails.extend(traverse_obj(json_data, (
+                'coverArt', 'sizes', ..., {thumb_base_url.__add__})))
+
+        return traverse_obj(json_data, {
+            'id': ('id', {str_or_none}),
+            'display_id': ('publicSlug', {str}),
+            'title': ('title', {str}),
+            'alt_title': ('oldTitle', {str}),
+            'duration': ('length', {float_or_none}),
+            'timestamp': ('added', {parse_iso8601}),
+            'release_timestamp': ('releaseDate', {parse_iso8601}),
+            'categories': ('genres', ..., 'tag', {str}),
+            'tags': ('metadataTags', ..., {str}),
+            'age_limit': ('isExplicit', {lambda b: 18 if b else None}),
+            'thumbnails': ({lambda _: thumbnails}, {orderedSet}, ..., {self._epidemic_parse_thumbnail}),
+            'formats': ('stems', {dict.items}, ..., {
+                'format': (0, {str_or_none}),
+                'format_note': (1, 's3TrackId', {str_or_none}),
+                'format_id': (1, 'stemType', {str}),
+                'url': (1, 'lqMp3Url', {url_or_none}),
+            }, {self._epidemic_fmt_or_none}),
+        })
diff --git a/yt_dlp/extractor/imgur.py b/yt_dlp/extractor/imgur.py
index bff6ed57f..1fa0a2a79 100644
--- a/yt_dlp/extractor/imgur.py
+++ b/yt_dlp/extractor/imgur.py
@@ -1,99 +1,243 @@
+import functools
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
+    determine_ext,
+    float_or_none,
     int_or_none,
     js_to_json,
     mimetype2ext,
-    ExtractorError,
+    parse_iso8601,
+    str_or_none,
+    strip_or_none,
+    traverse_obj,
+    url_or_none,
 )
 
 
-class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|(?:t(?:opic)?|r)/[^/]+)/)(?P<id>[a-zA-Z0-9]+)'
+class ImgurBaseIE(InfoExtractor):
+    _CLIENT_ID = '546c25a59c58ad7'
+
+    @classmethod
+    def _imgur_result(cls, item_id):
+        return cls.url_result(f'https://imgur.com/{item_id}', ImgurIE, item_id)
+
+    def _call_api(self, endpoint, video_id, **kwargs):
+        return self._download_json(
+            f'https://api.imgur.com/post/v1/{endpoint}/{video_id}?client_id={self._CLIENT_ID}&include=media,account',
+            video_id, **kwargs)
+
+    @staticmethod
+    def get_description(s):
+        if 'Discover the magic of the internet at Imgur' in s:
+            return None
+        return s or None
+
+
+class ImgurIE(ImgurBaseIE):
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|t|topic|r)/)(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
-        'url': 'https://i.imgur.com/A61SaA1.gifv',
+        'url': 'https://imgur.com/A61SaA1',
         'info_dict': {
             'id': 'A61SaA1',
             'ext': 'mp4',
-            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
+            'title': 'MRW gifv is up and running without any bugs',
+            'timestamp': 1416446068,
+            'upload_date': '20141120',
+            'dislike_count': int,
+            'comment_count': int,
+            'release_timestamp': 1416446068,
+            'release_date': '20141120',
+            'like_count': int,
+            'thumbnail': 'https://i.imgur.com/A61SaA1h.jpg',
         },
     }, {
-        'url': 'https://imgur.com/A61SaA1',
+        'url': 'https://i.imgur.com/A61SaA1.gifv',
         'only_matching': True,
     }, {
         'url': 'https://i.imgur.com/crGpqCV.mp4',
         'only_matching': True,
     }, {
-        # no title
         'url': 'https://i.imgur.com/jxBXAMC.gifv',
-        'only_matching': True,
+        'info_dict': {
+            'id': 'jxBXAMC',
+            'ext': 'mp4',
+            'title': 'Fahaka puffer feeding',
+            'timestamp': 1533835503,
+            'upload_date': '20180809',
+            'release_date': '20180809',
+            'like_count': int,
+            'duration': 30.0,
+            'comment_count': int,
+            'release_timestamp': 1533835503,
+            'thumbnail': 'https://i.imgur.com/jxBXAMCh.jpg',
+            'dislike_count': int,
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        data = self._call_api('media', video_id)
+        if not traverse_obj(data, ('media', 0, (
+                ('type', {lambda t: t == 'video' or None}),
+                ('metadata', 'is_animated'))), get_all=False):
+            raise ExtractorError(f'{video_id} is not a video or animated image', expected=True)
         webpage = self._download_webpage(
-            'https://i.imgur.com/{id}.gifv'.format(id=video_id), video_id)
+            f'https://i.imgur.com/{video_id}.gifv', video_id, fatal=False) or ''
+        formats = []
 
-        width = int_or_none(self._og_search_property(
-            'video:width', webpage, default=None))
-        height = int_or_none(self._og_search_property(
-            'video:height', webpage, default=None))
+        media_fmt = traverse_obj(data, ('media', 0, {
+            'url': ('url', {url_or_none}),
+            'ext': ('ext', {str}),
+            'width': ('width', {int_or_none}),
+            'height': ('height', {int_or_none}),
+            'filesize': ('size', {int_or_none}),
+            'acodec': ('metadata', 'has_sound', {lambda b: None if b else 'none'}),
+        }))
+        media_url = media_fmt.get('url')
+        if media_url:
+            if not media_fmt.get('ext'):
+                media_fmt['ext'] = mimetype2ext(traverse_obj(
+                    data, ('media', 0, 'mime_type'))) or determine_ext(media_url)
+            if traverse_obj(data, ('media', 0, 'type')) == 'image':
+                media_fmt['acodec'] = 'none'
+                media_fmt.setdefault('preference', -10)
+            formats.append(media_fmt)
 
         video_elements = self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',
             webpage, 'video elements', default=None)
-        if not video_elements:
-            raise ExtractorError(
-                'No sources found for video %s. Maybe an image?' % video_id,
-                expected=True)
 
-        formats = []
-        for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
-            formats.append({
-                'format_id': m.group('type').partition('/')[2],
-                'url': self._proto_relative_url(m.group('src')),
-                'ext': mimetype2ext(m.group('type')),
-                'width': width,
-                'height': height,
-                'http_headers': {
-                    'User-Agent': 'yt-dlp (like wget)',
-                },
-            })
+        if video_elements:
+            def og_get_size(media_type):
+                return {
+                    p: int_or_none(self._og_search_property(f'{media_type}:{p}', webpage, default=None))
+                    for p in ('width', 'height')
+                }
 
-        gif_json = self._search_regex(
-            r'(?s)var\s+videoItem\s*=\s*(\{.*?\})',
-            webpage, 'GIF code', fatal=False)
-        if gif_json:
-            gifd = self._parse_json(
-                gif_json, video_id, transform_source=js_to_json)
-            formats.append({
-                'format_id': 'gif',
-                'preference': -10,  # gifs are worse than videos
-                'width': width,
-                'height': height,
-                'ext': 'gif',
-                'acodec': 'none',
-                'vcodec': 'gif',
-                'container': 'gif',
-                'url': self._proto_relative_url(gifd['gifUrl']),
-                'filesize': gifd.get('size'),
-                'http_headers': {
-                    'User-Agent': 'yt-dlp (like wget)',
-                },
+            size = og_get_size('video')
+            if not any(size.values()):
+                size = og_get_size('image')
+
+            formats = traverse_obj(
+                re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements),
+                (..., {
+                    'format_id': ('type', {lambda s: s.partition('/')[2]}),
+                    'url': ('src', {self._proto_relative_url}),
+                    'ext': ('type', {mimetype2ext}),
+                }))
+            for f in formats:
+                f.update(size)
+
+            # We can get the original gif format from the webpage as well
+            gif_json = traverse_obj(self._search_json(
+                r'var\s+videoItem\s*=', webpage, 'GIF info', video_id,
+                transform_source=js_to_json, fatal=False), {
+                    'url': ('gifUrl', {self._proto_relative_url}),
+                    'filesize': ('size', {int_or_none}),
             })
+            if gif_json:
+                gif_json.update(size)
+                gif_json.update({
+                    'format_id': 'gif',
+                    'preference': -10,  # gifs < videos
+                    'ext': 'gif',
+                    'acodec': 'none',
+                    'vcodec': 'gif',
+                    'container': 'gif',
+                })
+                formats.append(gif_json)
+
+        search = functools.partial(self._html_search_meta, html=webpage, default=None)
+
+        twitter_fmt = {
+            'format_id': 'twitter',
+            'url': url_or_none(search('twitter:player:stream')),
+            'ext': mimetype2ext(search('twitter:player:stream:content_type')),
+            'width': int_or_none(search('twitter:width')),
+            'height': int_or_none(search('twitter:height')),
+        }
+        if twitter_fmt['url']:
+            formats.append(twitter_fmt)
+
+        if not formats:
+            self.raise_no_formats(
+                f'No sources found for video {video_id}. Maybe a plain image?', expected=True)
+        self._remove_duplicate_formats(formats)
 
         return {
+            'title': self._og_search_title(webpage, default=None),
+            'description': self.get_description(self._og_search_description(webpage, default='')),
+            **traverse_obj(data, {
+                'uploader_id': ('account_id', {lambda a: str(a) if int_or_none(a) else None}),
+                'uploader': ('account', 'username', {lambda x: strip_or_none(x) or None}),
+                'uploader_url': ('account', 'avatar_url', {url_or_none}),
+                'like_count': ('upvote_count', {int_or_none}),
+                'dislike_count': ('downvote_count', {int_or_none}),
+                'comment_count': ('comment_count', {int_or_none}),
+                'age_limit': ('is_mature', {lambda x: 18 if x else None}),
+                'timestamp': (('updated_at', 'created_at'), {parse_iso8601}),
+                'release_timestamp': ('created_at', {parse_iso8601}),
+            }, get_all=False),
+            **traverse_obj(data, ('media', 0, 'metadata', {
+                'title': ('title', {lambda x: strip_or_none(x) or None}),
+                'description': ('description', {self.get_description}),
+                'duration': ('duration', {float_or_none}),
+                'timestamp': (('updated_at', 'created_at'), {parse_iso8601}),
+                'release_timestamp': ('created_at', {parse_iso8601}),
+            }), get_all=False),
             'id': video_id,
             'formats': formats,
-            'title': self._og_search_title(webpage, default=video_id),
+            'thumbnail': url_or_none(search('thumbnailUrl')),
         }
 
 
-class ImgurGalleryIE(InfoExtractor):
+class ImgurGalleryBaseIE(ImgurBaseIE):
+    _GALLERY = True
+
+    def _real_extract(self, url):
+        gallery_id = self._match_id(url)
+
+        data = self._call_api('albums', gallery_id, fatal=False, expected_status=404)
+
+        info = traverse_obj(data, {
+            'title': ('title', {lambda x: strip_or_none(x) or None}),
+            'description': ('description', {self.get_description}),
+        })
+
+        if traverse_obj(data, 'is_album'):
+
+            def yield_media_ids():
+                for m_id in traverse_obj(data, (
+                        'media', lambda _, v: v.get('type') == 'video' or v['metadata']['is_animated'],
+                        'id', {lambda x: str_or_none(x) or None})):
+                    yield m_id
+
+            # if a gallery with exactly one video, apply album metadata to video
+            media_id = (
+                self._GALLERY
+                and traverse_obj(data, ('image_count', {lambda c: c == 1}))
+                and next(yield_media_ids(), None))
+
+            if not media_id:
+                result = self.playlist_result(
+                    map(self._imgur_result, yield_media_ids()), gallery_id)
+                result.update(info)
+                return result
+            gallery_id = media_id
+
+        result = self._imgur_result(gallery_id)
+        info['_type'] = 'url_transparent'
+        result.update(info)
+        return result
+
+
+class ImgurGalleryIE(ImgurGalleryBaseIE):
     IE_NAME = 'imgur:gallery'
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/]+)/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/?#]+)/(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
         'url': 'http://imgur.com/gallery/Q95ko',
@@ -102,49 +246,121 @@ class ImgurGalleryIE(InfoExtractor):
             'title': 'Adding faces make every GIF better',
         },
         'playlist_count': 25,
+        'skip': 'Zoinks! You\'ve taken a wrong turn.',
     }, {
+        # TODO: static images - replace with animated/video gallery
         'url': 'http://imgur.com/topic/Aww/ll5Vk',
         'only_matching': True,
     }, {
         'url': 'https://imgur.com/gallery/YcAQlkx',
+        'add_ies': ['Imgur'],
         'info_dict': {
             'id': 'YcAQlkx',
             'ext': 'mp4',
             'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
-        }
+            'timestamp': 1358554297,
+            'upload_date': '20130119',
+            'uploader_id': '1648642',
+            'uploader': 'wittyusernamehere',
+            'release_timestamp': 1358554297,
+            'thumbnail': 'https://i.imgur.com/YcAQlkxh.jpg',
+            'release_date': '20130119',
+            'uploader_url': 'https://i.imgur.com/u3R4I2S_d.png?maxwidth=290&fidelity=grand',
+            'comment_count': int,
+            'dislike_count': int,
+            'like_count': int,
+        },
     }, {
+        # TODO: static image - replace with animated/video gallery
         'url': 'http://imgur.com/topic/Funny/N8rOudd',
         'only_matching': True,
     }, {
         'url': 'http://imgur.com/r/aww/VQcQPhM',
-        'only_matching': True,
+        'add_ies': ['Imgur'],
+        'info_dict': {
+            'id': 'VQcQPhM',
+            'ext': 'mp4',
+            'title': 'The boss is here',
+            'timestamp': 1476494751,
+            'upload_date': '20161015',
+            'uploader_id': '19138530',
+            'uploader': 'thematrixcam',
+            'comment_count': int,
+            'dislike_count': int,
+            'uploader_url': 'https://i.imgur.com/qCjr5Pi_d.png?maxwidth=290&fidelity=grand',
+            'release_timestamp': 1476494751,
+            'like_count': int,
+            'release_date': '20161015',
+            'thumbnail': 'https://i.imgur.com/VQcQPhMh.jpg',
+        },
+    },
+        # from https://github.com/ytdl-org/youtube-dl/pull/16674
+        {
+        'url': 'https://imgur.com/t/unmuted/6lAn9VQ',
+        'info_dict': {
+            'id': '6lAn9VQ',
+            'title': 'Penguins !',
+        },
+        'playlist_count': 3,
+    }, {
+        'url': 'https://imgur.com/t/unmuted/kx2uD3C',
+        'add_ies': ['Imgur'],
+        'info_dict': {
+            'id': 'ZVMv45i',
+            'ext': 'mp4',
+            'title': 'Intruder',
+            'timestamp': 1528129683,
+            'upload_date': '20180604',
+            'release_timestamp': 1528129683,
+            'release_date': '20180604',
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'duration': 30.03,
+            'thumbnail': 'https://i.imgur.com/ZVMv45ih.jpg',
+        },
+    }, {
+        'url': 'https://imgur.com/t/unmuted/wXSK0YH',
+        'add_ies': ['Imgur'],
+        'info_dict': {
+            'id': 'JCAP4io',
+            'ext': 'mp4',
+            'title': 're:I got the blues$',
+            'description': 'Luka’s vocal stylings.\n\nFP edit: don’t encourage me. I’ll never stop posting Luka and friends.',
+            'timestamp': 1527809525,
+            'upload_date': '20180531',
+            'like_count': int,
+            'dislike_count': int,
+            'duration': 30.03,
+            'comment_count': int,
+            'release_timestamp': 1527809525,
+            'thumbnail': 'https://i.imgur.com/JCAP4ioh.jpg',
+            'release_date': '20180531',
+        },
     }]
 
-    def _real_extract(self, url):
-        gallery_id = self._match_id(url)
 
-        data = self._download_json(
-            'https://imgur.com/gallery/%s.json' % gallery_id,
-            gallery_id)['data']['image']
-
-        if data.get('is_album'):
-            entries = [
-                self.url_result('http://imgur.com/%s' % image['hash'], ImgurIE.ie_key(), image['hash'])
-                for image in data['album_images']['images'] if image.get('hash')]
-            return self.playlist_result(entries, gallery_id, data.get('title'), data.get('description'))
-
-        return self.url_result('http://imgur.com/%s' % gallery_id, ImgurIE.ie_key(), gallery_id)
-
-
-class ImgurAlbumIE(ImgurGalleryIE):  # XXX: Do not subclass from concrete IE
+class ImgurAlbumIE(ImgurGalleryBaseIE):
     IE_NAME = 'imgur:album'
     _VALID_URL = r'https?://(?:i\.)?imgur\.com/a/(?P<id>[a-zA-Z0-9]+)'
-
+    _GALLERY = False
     _TESTS = [{
+        # TODO: only static images - replace with animated/video gallery
         'url': 'http://imgur.com/a/j6Orj',
+        'only_matching': True,
+    },
+        # from https://github.com/ytdl-org/youtube-dl/pull/21693
+        {
+        'url': 'https://imgur.com/a/iX265HX',
         'info_dict': {
-            'id': 'j6Orj',
-            'title': 'A Literary Analysis of "Star Wars: The Force Awakens"',
+            'id': 'iX265HX',
+            'title': 'enen-no-shouboutai'
         },
-        'playlist_count': 12,
+        'playlist_count': 2,
+    }, {
+        'url': 'https://imgur.com/a/8pih2Ed',
+        'info_dict': {
+            'id': '8pih2Ed'
+        },
+        'playlist_mincount': 1,
     }]
diff --git a/yt_dlp/extractor/telewebion.py b/yt_dlp/extractor/telewebion.py
index 550549f05..9378ed021 100644
--- a/yt_dlp/extractor/telewebion.py
+++ b/yt_dlp/extractor/telewebion.py
@@ -1,52 +1,133 @@
+from __future__ import annotations
+
+import json
+from functools import partial
+from textwrap import dedent
+
 from .common import InfoExtractor
+from ..utils import ExtractorError, format_field, int_or_none, parse_iso8601
+from ..utils.traversal import traverse_obj
+
+
+def _fmt_url(url):
+    return partial(format_field, template=url, default=None)
 
 
 class TelewebionIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?telewebion\.com/#!/episode/(?P<id>\d+)'
-
-    _TEST = {
-        'url': 'http://www.telewebion.com/#!/episode/1263668/',
+    _VALID_URL = r'https?://(?:www\.)?telewebion\.com/episode/(?P<id>(?:0x[a-fA-F\d]+|\d+))'
+    _TESTS = [{
+        'url': 'http://www.telewebion.com/episode/0x1b3139c/',
         'info_dict': {
-            'id': '1263668',
+            'id': '0x1b3139c',
             'ext': 'mp4',
-            'title': 'قرعه\u200cکشی لیگ قهرمانان اروپا',
-            'thumbnail': r're:^https?://.*\.jpg',
+            'title': 'قرعه‌کشی لیگ قهرمانان اروپا',
+            'series': '+ فوتبال',
+            'series_id': '0x1b2505c',
+            'channel': 'شبکه 3',
+            'channel_id': '0x1b1a761',
+            'channel_url': 'https://telewebion.com/live/tv3',
+            'timestamp': 1425522414,
+            'upload_date': '20150305',
+            'release_timestamp': 1425517020,
+            'release_date': '20150305',
+            'duration': 420,
             'view_count': int,
+            'tags': ['ورزشی', 'لیگ اروپا', 'اروپا'],
+            'thumbnail': 'https://static.telewebion.com/episodeImages/YjFhM2MxMDBkMDNiZTU0MjE5YjQ3ZDY0Mjk1ZDE0ZmUwZWU3OTE3OWRmMDAyODNhNzNkNjdmMWMzMWIyM2NmMA/default',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
+        'skip_download': 'm3u8',
+    }, {
+        'url': 'https://telewebion.com/episode/162175536',
+        'info_dict': {
+            'id': '0x9aa9a30',
+            'ext': 'mp4',
+            'title': 'کارما یعنی این !',
+            'series': 'پاورقی',
+            'series_id': '0x29a7426',
+            'channel': 'شبکه 2',
+            'channel_id': '0x1b1a719',
+            'channel_url': 'https://telewebion.com/live/tv2',
+            'timestamp': 1699979968,
+            'upload_date': '20231114',
+            'release_timestamp': 1699991638,
+            'release_date': '20231114',
+            'duration': 78,
+            'view_count': int,
+            'tags': ['کلیپ های منتخب', ' کلیپ طنز ', ' کلیپ سیاست ', 'پاورقی', 'ویژه فلسطین'],
+            'thumbnail': 'https://static.telewebion.com/episodeImages/871e9455-7567-49a5-9648-34c22c197f5f/default',
         },
-    }
+        'skip_download': 'm3u8',
+    }]
+
+    def _call_graphql_api(
+        self, operation, video_id, query,
+        variables: dict[str, tuple[str, str]] | None = None,
+        note='Downloading GraphQL JSON metadata',
+    ):
+        parameters = ''
+        if variables:
+            parameters = ', '.join(f'${name}: {type_}' for name, (type_, _) in variables.items())
+            parameters = f'({parameters})'
+
+        result = self._download_json('https://graph.telewebion.com/graphql', video_id, note, data=json.dumps({
+            'operationName': operation,
+            'query': f'query {operation}{parameters} @cacheControl(maxAge: 60) {{{query}\n}}\n',
+            'variables': {name: value for name, (_, value) in (variables or {}).items()}
+        }, separators=(',', ':')).encode(), headers={
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+        })
+        if not result or traverse_obj(result, 'errors'):
+            message = ', '.join(traverse_obj(result, ('errors', ..., 'message', {str})))
+            raise ExtractorError(message or 'Unknown GraphQL API error')
+
+        return result['data']
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        if not video_id.startswith('0x'):
+            video_id = hex(int(video_id))
 
-        secure_token = self._download_webpage(
-            'http://m.s2.telewebion.com/op/op?action=getSecurityToken', video_id)
-        episode_details = self._download_json(
-            'http://m.s2.telewebion.com/op/op', video_id,
-            query={'action': 'getEpisodeDetails', 'episode_id': video_id})
+        episode_data = self._call_graphql_api('getEpisodeDetail', video_id, dedent('''
+            queryEpisode(filter: {EpisodeID: $EpisodeId}, first: 1) {
+              title
+              program {
+                ProgramID
+                title
+              }
+              image
+              view_count
+              duration
+              started_at
+              created_at
+              channel {
+                ChannelID
+                name
+                descriptor
+              }
+              tags {
+                name
+              }
+            }
+        '''), {'EpisodeId': ('[ID!]', video_id)})
 
-        m3u8_url = 'http://m.s1.telewebion.com/smil/%s.m3u8?filepath=%s&m3u8=1&secure_token=%s' % (
-            video_id, episode_details['file_path'], secure_token)
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, ext='mp4', m3u8_id='hls')
-
-        picture_paths = [
-            episode_details.get('picture_path'),
-            episode_details.get('large_picture_path'),
-        ]
-
-        thumbnails = [{
-            'url': picture_path,
-            'preference': idx,
-        } for idx, picture_path in enumerate(picture_paths) if picture_path is not None]
-
-        return {
-            'id': video_id,
-            'title': episode_details['title'],
-            'formats': formats,
-            'thumbnails': thumbnails,
-            'view_count': episode_details.get('view_count'),
-        }
+        info_dict = traverse_obj(episode_data, ('queryEpisode', 0, {
+            'title': ('title', {str}),
+            'view_count': ('view_count', {int_or_none}),
+            'duration': ('duration', {int_or_none}),
+            'tags': ('tags', ..., 'name', {str}),
+            'release_timestamp': ('started_at', {parse_iso8601}),
+            'timestamp': ('created_at', {parse_iso8601}),
+            'series': ('program', 'title', {str}),
+            'series_id': ('program', 'ProgramID', {str}),
+            'channel': ('channel', 'name', {str}),
+            'channel_id': ('channel', 'ChannelID', {str}),
+            'channel_url': ('channel', 'descriptor', {_fmt_url('https://telewebion.com/live/%s')}),
+            'thumbnail': ('image', {_fmt_url('https://static.telewebion.com/episodeImages/%s/default')}),
+            'formats': (
+                'channel', 'descriptor', {str},
+                {_fmt_url(f'https://cdna.telewebion.com/%s/episode/{video_id}/playlist.m3u8')},
+                {partial(self._extract_m3u8_formats, video_id=video_id, ext='mp4', m3u8_id='hls')}),
+        }))
+        info_dict['id'] = video_id
+        return info_dict
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index f1d7cead6..361617c02 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -636,7 +636,7 @@ def replace_insane(char):
         elif char in '\\/|*<>':
             return '\0_'
         if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace() or ord(char) > 127):
-            return '\0_'
+            return '' if unicodedata.category(char)[0] in 'CM' else '\0_'
         return char
 
     # Replace look-alike Unicode glyphs

From 2d1d683a541d71f3d3bb999dfe8eeb1976fb91ce Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 26 Dec 2023 18:30:04 +0100
Subject: [PATCH 757/871] [devscripts] `run_tests`: Create Python script
 (#8720)

Authored by: Grub4K
---
 .github/workflows/core.yml       |  6 +--
 .github/workflows/download.yml   |  7 +---
 .github/workflows/quick-test.yml |  2 +-
 CONTRIBUTING.md                  | 31 +++++++-------
 devscripts/run_tests.bat         | 17 +-------
 devscripts/run_tests.py          | 70 ++++++++++++++++++++++++++++++++
 devscripts/run_tests.sh          | 14 +------
 7 files changed, 95 insertions(+), 52 deletions(-)
 create mode 100755 devscripts/run_tests.py

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index b22adb1b9..ded7e6d61 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -38,18 +38,14 @@ jobs:
         os: [ubuntu-latest]
         # CPython 3.11 is in quick-test
         python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.8, pypy-3.10]
-        run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
         - os: windows-latest
           python-version: '3.8'
-          run-tests-ext: bat
         - os: windows-latest
           python-version: '3.12'
-          run-tests-ext: bat
         - os: windows-latest
           python-version: pypy-3.9
-          run-tests-ext: bat
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -62,4 +58,4 @@ jobs:
       continue-on-error: False
       run: |
         python3 -m yt_dlp -v || true  # Print debug head
-        ./devscripts/run_tests.${{ matrix.run-tests-ext }} core
+        python3 ./devscripts/run_tests.py core
diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml
index 73b2f9ca3..9f47d6718 100644
--- a/.github/workflows/download.yml
+++ b/.github/workflows/download.yml
@@ -18,7 +18,7 @@ jobs:
       run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: true
-      run: ./devscripts/run_tests.sh download
+      run: python3 ./devscripts/run_tests.py download
 
   full:
     name: Full Download Tests
@@ -29,15 +29,12 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ['3.10', '3.11', '3.12', pypy-3.8, pypy-3.10]
-        run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
         - os: windows-latest
           python-version: '3.8'
-          run-tests-ext: bat
         - os: windows-latest
           python-version: pypy-3.9
-          run-tests-ext: bat
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -48,4 +45,4 @@ jobs:
       run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: true
-      run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} download
+      run: python3 ./devscripts/run_tests.py download
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index edbdaffd7..1ccfbe836 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -19,7 +19,7 @@ jobs:
     - name: Run tests
       run: |
         python3 -m yt_dlp -v || true
-        ./devscripts/run_tests.sh core
+        python3 ./devscripts/run_tests.py core
   flake8:
     name: Linter
     if: "!contains(github.event.head_commit.message, 'ci skip all')"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c472f3251..248917bf5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -140,12 +140,9 @@ # DEVELOPER INSTRUCTIONS
 
     python -m yt_dlp
 
-To run the test, simply invoke your favorite test runner, or execute a test file directly; any of the following work:
+To run all the available core tests, use:
 
-    python -m unittest discover
-    python test/test_download.py
-    nosetests
-    pytest
+    python devscripts/run_tests.py
 
 See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases.
 
@@ -187,15 +184,21 @@ ## Adding support for a new site
             'url': 'https://yourextractor.com/watch/42',
             'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
             'info_dict': {
+                # For videos, only the 'id' and 'ext' fields are required to RUN the test:
                 'id': '42',
                 'ext': 'mp4',
-                'title': 'Video title goes here',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                # TODO more properties, either as:
-                # * A value
-                # * MD5 checksum; start the string with md5:
-                # * A regular expression; start the string with re:
-                # * Any Python type, e.g. int or float
+                # Then if the test run fails, it will output the missing/incorrect fields.
+                # Properties can be added as:
+                # * A value, e.g.
+                #     'title': 'Video title goes here',
+                # * MD5 checksum; start the string with 'md5:', e.g.
+                #     'description': 'md5:098f6bcd4621d373cade4e832627b4f6',
+                # * A regular expression; start the string with 're:', e.g.
+                #     'thumbnail': r're:^https?://.*\.jpg$',
+                # * A count of elements in a list; start the string with 'count:', e.g.
+                #     'tags': 'count:10',
+                # * Any Python type, e.g.
+                #     'view_count': int,
             }
         }]
 
@@ -215,8 +218,8 @@ ## Adding support for a new site
             }
     ```
 1. Add an import in [`yt_dlp/extractor/_extractors.py`](yt_dlp/extractor/_extractors.py). Note that the class name must end with `IE`.
-1. Run `python test/test_download.py TestDownload.test_YourExtractor` (note that `YourExtractor` doesn't end with `IE`). This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, the tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. You can also run all the tests in one go with `TestDownload.test_YourExtractor_all`
-1. Make sure you have atleast one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
+1. Run `python devscripts/run_tests.py YourExtractor`. This *may fail* at first, but you can continually re-run it until you're done. Upon failure, it will output the missing fields and/or correct values which you can copy. If you decide to add more than one test, the tests will then be named `YourExtractor`, `YourExtractor_1`, `YourExtractor_2`, etc. Note that tests with an `only_matching` key in the test's dict are not included in the count. You can also run all the tests in one go with `YourExtractor_all`
+1. Make sure you have at least one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
 1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L119-L440). Add tests and code for as many as you want.
 1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
 
diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat
index 190d23918..57b1f4bf4 100644
--- a/devscripts/run_tests.bat
+++ b/devscripts/run_tests.bat
@@ -1,17 +1,4 @@
-@setlocal
 @echo off
-cd /d %~dp0..
 
-if ["%~1"]==[""] (
-    set "test_set="test""
-) else if ["%~1"]==["core"] (
-    set "test_set="-m not download""
-) else if ["%~1"]==["download"] (
-    set "test_set="-m "download""
-) else (
-    echo.Invalid test type "%~1". Use "core" ^| "download"
-    exit /b 1
-)
-
-set PYTHONWARNINGS=error
-pytest %test_set%
+>&2 echo run_tests.bat is deprecated. Please use `devscripts/run_tests.py` instead
+python %~dp0run_tests.py %~1
diff --git a/devscripts/run_tests.py b/devscripts/run_tests.py
new file mode 100755
index 000000000..b0c6ee67a
--- /dev/null
+++ b/devscripts/run_tests.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import argparse
+import functools
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+fix_test_name = functools.partial(re.compile(r'IE(_all|_\d+)?$').sub, r'\1')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Run selected yt-dlp tests')
+    parser.add_argument(
+        'test', help='a extractor tests, or one of "core" or "download"', nargs='*')
+    parser.add_argument(
+        '-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
+    return parser.parse_args()
+
+
+def run_tests(*tests, pattern=None):
+    run_core = 'core' in tests or (not pattern and not tests)
+    run_download = 'download' in tests
+    tests = list(map(fix_test_name, tests))
+
+    arguments = ['pytest', '-Werror', '--tb', 'short']
+    if run_core:
+        arguments.extend(['-m', 'not download'])
+    elif run_download:
+        arguments.extend(['-m', 'download'])
+    elif pattern:
+        arguments.extend(['-k', pattern])
+    else:
+        arguments.extend(
+            f'test/test_download.py::TestDownload::test_{test}' for test in tests)
+
+    print(f'Running {arguments}')
+    try:
+        subprocess.run(arguments)
+        return
+    except FileNotFoundError:
+        pass
+
+    arguments = [sys.executable, '-Werror', '-m', 'unittest']
+    if run_core:
+        print('"pytest" needs to be installed to run core tests', file=sys.stderr)
+        return
+    elif run_download:
+        arguments.append('test.test_download')
+    elif pattern:
+        arguments.extend(['-k', pattern])
+    else:
+        arguments.extend(
+            f'test.test_download.TestDownload.test_{test}' for test in tests)
+
+    print(f'Running {arguments}')
+    subprocess.run(arguments)
+
+
+if __name__ == '__main__':
+    try:
+        args = parse_args()
+
+        os.chdir(Path(__file__).parent.parent)
+        run_tests(*args.test, pattern=args.k)
+    except KeyboardInterrupt:
+        pass
diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
index faa642e96..123ceb1ee 100755
--- a/devscripts/run_tests.sh
+++ b/devscripts/run_tests.sh
@@ -1,14 +1,4 @@
 #!/usr/bin/env sh
 
-if [ -z "$1" ]; then
-    test_set='test'
-elif [ "$1" = 'core' ]; then
-    test_set="-m not download"
-elif [ "$1" = 'download' ]; then
-    test_set="-m download"
-else
-    echo 'Invalid test type "'"$1"'". Use "core" | "download"'
-    exit 1
-fi
-
-python3 -bb -Werror -m pytest "$test_set"
+>&2 echo 'run_tests.sh is deprecated. Please use `devscripts/run_tests.py` instead'
+python3 devscripts/run_tests.py "$1"

From 225cf2b830a1de2c5eacd257edd2a01aed1e1114 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 26 Dec 2023 19:55:30 +0100
Subject: [PATCH 758/871] Fix 2d1d683a541d71f3d3bb999dfe8eeb1976fb91ce

Authored by: Grub4K
---
 devscripts/run_tests.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/devscripts/run_tests.py b/devscripts/run_tests.py
index b0c6ee67a..6d638a974 100755
--- a/devscripts/run_tests.py
+++ b/devscripts/run_tests.py
@@ -21,12 +21,14 @@ def parse_args():
     return parser.parse_args()
 
 
-def run_tests(*tests, pattern=None):
+def run_tests(*tests, pattern=None, ci=False):
     run_core = 'core' in tests or (not pattern and not tests)
     run_download = 'download' in tests
     tests = list(map(fix_test_name, tests))
 
-    arguments = ['pytest', '-Werror', '--tb', 'short']
+    arguments = ['pytest', '-Werror', '--tb=short']
+    if ci:
+        arguments.append('--color=yes')
     if run_core:
         arguments.extend(['-m', 'not download'])
     elif run_download:
@@ -37,17 +39,16 @@ def run_tests(*tests, pattern=None):
         arguments.extend(
             f'test/test_download.py::TestDownload::test_{test}' for test in tests)
 
-    print(f'Running {arguments}')
+    print(f'Running {arguments}', flush=True)
     try:
-        subprocess.run(arguments)
-        return
+        return subprocess.call(arguments)
     except FileNotFoundError:
         pass
 
     arguments = [sys.executable, '-Werror', '-m', 'unittest']
     if run_core:
-        print('"pytest" needs to be installed to run core tests', file=sys.stderr)
-        return
+        print('"pytest" needs to be installed to run core tests', file=sys.stderr, flush=True)
+        return 1
     elif run_download:
         arguments.append('test.test_download')
     elif pattern:
@@ -56,8 +57,8 @@ def run_tests(*tests, pattern=None):
         arguments.extend(
             f'test.test_download.TestDownload.test_{test}' for test in tests)
 
-    print(f'Running {arguments}')
-    subprocess.run(arguments)
+    print(f'Running {arguments}', flush=True)
+    return subprocess.call(arguments)
 
 
 if __name__ == '__main__':
@@ -65,6 +66,6 @@ def run_tests(*tests, pattern=None):
         args = parse_args()
 
         os.chdir(Path(__file__).parent.parent)
-        run_tests(*args.test, pattern=args.k)
+        sys.exit(run_tests(*args.test, pattern=args.k, ci=bool(os.getenv('CI'))))
     except KeyboardInterrupt:
         pass

From 5f009a094f0e8450792b097c4c8273622778052d Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sat, 30 Dec 2023 21:44:32 +0100
Subject: [PATCH 759/871] [ie/ARD] Overhaul extractors (#8878)

Closes #8731, Closes #6784, Closes #2366, Closes #2975, Closes #8760
Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py |   2 +-
 yt_dlp/extractor/ard.py         | 623 +++++++++++++-------------------
 2 files changed, 250 insertions(+), 375 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 62103f13c..6f7a1e4f1 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -134,8 +134,8 @@
 from .arkena import ArkenaIE
 from .ard import (
     ARDBetaMediathekIE,
+    ARDMediathekCollectionIE,
     ARDIE,
-    ARDMediathekIE,
 )
 from .arte import (
     ArteTVIE,
diff --git a/yt_dlp/extractor/ard.py b/yt_dlp/extractor/ard.py
index 8ac926c91..91d297e8b 100644
--- a/yt_dlp/extractor/ard.py
+++ b/yt_dlp/extractor/ard.py
@@ -1,24 +1,23 @@
-import json
 import re
+from functools import partial
 
 from .common import InfoExtractor
-from .generic import GenericIE
 from ..utils import (
+    OnDemandPagedList,
     determine_ext,
-    ExtractorError,
     int_or_none,
+    join_nonempty,
+    make_archive_id,
     parse_duration,
-    qualities,
+    parse_iso8601,
+    remove_start,
     str_or_none,
-    try_get,
     unified_strdate,
-    unified_timestamp,
-    update_url,
     update_url_query,
     url_or_none,
     xpath_text,
 )
-from ..compat import compat_etree_fromstring
+from ..utils.traversal import traverse_obj
 
 
 class ARDMediathekBaseIE(InfoExtractor):
@@ -61,45 +60,6 @@ def _parse_media_info(self, media_info, video_id, fsk):
             'subtitles': subtitles,
         }
 
-    def _ARD_extract_episode_info(self, title):
-        """Try to extract season/episode data from the title."""
-        res = {}
-        if not title:
-            return res
-
-        for pattern in [
-            # Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
-            # from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
-            r'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
-            # E.g.: title="Fritjof aus Norwegen (2) (AD)"
-            # from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
-            r'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
-            r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
-            # E.g.: title="Folge 25/42: Symmetrie"
-            # from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
-            # E.g.: title="Folge 1063 - Vertrauen"
-            # from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
-            r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
-        ]:
-            m = re.match(pattern, title)
-            if m:
-                groupdict = m.groupdict()
-                res['season_number'] = int_or_none(groupdict.get('season_number'))
-                res['episode_number'] = int_or_none(groupdict.get('episode_number'))
-                res['episode'] = str_or_none(groupdict.get('episode'))
-                # Build the episode title by removing numeric episode information:
-                if groupdict.get('ep_info') and not res['episode']:
-                    res['episode'] = str_or_none(
-                        title.replace(groupdict.get('ep_info'), ''))
-                if res['episode']:
-                    res['episode'] = res['episode'].strip()
-                break
-
-        # As a fallback use the whole title as the episode name:
-        if not res.get('episode'):
-            res['episode'] = title.strip()
-        return res
-
     def _extract_formats(self, media_info, video_id):
         type_ = media_info.get('_type')
         media_array = media_info.get('_mediaArray', [])
@@ -155,138 +115,6 @@ def _extract_formats(self, media_info, video_id):
         return formats
 
 
-class ARDMediathekIE(ARDMediathekBaseIE):
-    IE_NAME = 'ARD:mediathek'
-    _VALID_URL = r'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
-
-    _TESTS = [{
-        # available till 26.07.2022
-        'url': 'http://www.ardmediathek.de/tv/S%C3%9CDLICHT/Was-ist-die-Kunst-der-Zukunft-liebe-Ann/BR-Fernsehen/Video?bcastId=34633636&documentId=44726822',
-        'info_dict': {
-            'id': '44726822',
-            'ext': 'mp4',
-            'title': 'Was ist die Kunst der Zukunft, liebe Anna McCarthy?',
-            'description': 'md5:4ada28b3e3b5df01647310e41f3a62f5',
-            'duration': 1740,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
-    }, {
-        'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872',
-        'only_matching': True,
-    }, {
-        # audio
-        'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
-        'only_matching': True,
-    }, {
-        'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
-        'only_matching': True,
-    }, {
-        # audio
-        'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
-        'only_matching': True,
-    }, {
-        'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698',
-        'only_matching': True,
-    }]
-
-    @classmethod
-    def suitable(cls, url):
-        return False if ARDBetaMediathekIE.suitable(url) else super(ARDMediathekIE, cls).suitable(url)
-
-    def _real_extract(self, url):
-        # determine video id from url
-        m = self._match_valid_url(url)
-
-        document_id = None
-
-        numid = re.search(r'documentId=([0-9]+)', url)
-        if numid:
-            document_id = video_id = numid.group(1)
-        else:
-            video_id = m.group('video_id')
-
-        webpage = self._download_webpage(url, video_id)
-
-        ERRORS = (
-            ('>Leider liegt eine Störung vor.', 'Video %s is unavailable'),
-            ('>Der gewünschte Beitrag ist nicht mehr verfügbar.<',
-             'Video %s is no longer available'),
-        )
-
-        for pattern, message in ERRORS:
-            if pattern in webpage:
-                raise ExtractorError(message % video_id, expected=True)
-
-        if re.search(r'[\?&]rss($|[=&])', url):
-            doc = compat_etree_fromstring(webpage.encode('utf-8'))
-            if doc.tag == 'rss':
-                return GenericIE()._extract_rss(url, video_id, doc)
-
-        title = self._og_search_title(webpage, default=None) or self._html_search_regex(
-            [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
-             r'<meta name="dcterms\.title" content="(.*?)"/>',
-             r'<h4 class="headline">(.*?)</h4>',
-             r'<title[^>]*>(.*?)</title>'],
-            webpage, 'title')
-        description = self._og_search_description(webpage, default=None) or self._html_search_meta(
-            'dcterms.abstract', webpage, 'description', default=None)
-        if description is None:
-            description = self._html_search_meta(
-                'description', webpage, 'meta description', default=None)
-        if description is None:
-            description = self._html_search_regex(
-                r'<p\s+class="teasertext">(.+?)</p>',
-                webpage, 'teaser text', default=None)
-
-        # Thumbnail is sometimes not present.
-        # It is in the mobile version, but that seems to use a different URL
-        # structure altogether.
-        thumbnail = self._og_search_thumbnail(webpage, default=None)
-
-        media_streams = re.findall(r'''(?x)
-            mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
-            "([^"]+)"''', webpage)
-
-        if media_streams:
-            QUALITIES = qualities(['lo', 'hi', 'hq'])
-            formats = []
-            for furl in set(media_streams):
-                if furl.endswith('.f4m'):
-                    fid = 'f4m'
-                else:
-                    fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
-                    fid = fid_m.group(1) if fid_m else None
-                formats.append({
-                    'quality': QUALITIES(fid),
-                    'format_id': fid,
-                    'url': furl,
-                })
-            info = {
-                'formats': formats,
-            }
-        else:  # request JSON file
-            if not document_id:
-                video_id = self._search_regex(
-                    (r'/play/(?:config|media|sola)/(\d+)', r'contentId["\']\s*:\s*(\d+)'),
-                    webpage, 'media id', default=None)
-            info = self._extract_media_info(
-                'http://www.ardmediathek.de/play/media/%s' % video_id,
-                webpage, video_id)
-
-        info.update({
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-        })
-        info.update(self._ARD_extract_episode_info(info['title']))
-
-        return info
-
-
 class ARDIE(InfoExtractor):
     _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html'
     _TESTS = [{
@@ -399,21 +227,23 @@ def _real_extract(self, url):
         }
 
 
-class ARDBetaMediathekIE(ARDMediathekBaseIE):
+class ARDBetaMediathekIE(InfoExtractor):
+    IE_NAME = 'ARDMediathek'
     _VALID_URL = r'''(?x)https://
         (?:(?:beta|www)\.)?ardmediathek\.de/
-        (?:(?P<client>[^/]+)/)?
-        (?:player|live|video|(?P<playlist>sendung|serie|sammlung))/
-        (?:(?P<display_id>(?(playlist)[^?#]+?|[^?#]+))/)?
-        (?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)
-        (?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
+        (?:[^/]+/)?
+        (?:player|live|video)/
+        (?:(?P<display_id>[^?#]+)/)?
+        (?P<id>[a-zA-Z0-9]+)
+        /?(?:[?#]|$)'''
+    _GEO_COUNTRIES = ['DE']
 
     _TESTS = [{
         'url': 'https://www.ardmediathek.de/video/filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen/Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
         'md5': 'b6e8ab03f2bcc6e1f9e6cef25fcc03c4',
         'info_dict': {
             'display_id': 'filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen',
-            'id': '12939099',
+            'id': 'Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
             'title': 'Liebe auf vier Pfoten',
             'description': r're:^Claudia Schmitt, Anwältin in Salzburg',
             'duration': 5222,
@@ -422,7 +252,10 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'upload_date': '20231130',
             'ext': 'mp4',
             'episode': 'Liebe auf vier Pfoten',
-            'series': 'Filme im MDR'
+            'series': 'Filme im MDR',
+            'age_limit': 0,
+            'channel': 'MDR',
+            '_old_archive_ids': ['ardbetamediathek 12939099'],
         },
     }, {
         'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
@@ -443,7 +276,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
         'md5': '1e73ded21cb79bac065117e80c81dc88',
         'info_dict': {
-            'id': '10049223',
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
             'ext': 'mp4',
             'title': 'tagesschau, 20:00 Uhr',
             'timestamp': 1636398000,
@@ -454,6 +287,26 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'episode': 'tagesschau, 20:00 Uhr',
             'series': 'tagesschau',
             'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49?w=960&ch=ee69108ae344f678',
+            'channel': 'ARD-Aktuell',
+            '_old_archive_ids': ['ardbetamediathek 10049223'],
+        },
+    }, {
+        'url': 'https://www.ardmediathek.de/video/7-tage/7-tage-unter-harten-jungs/hr-fernsehen/N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3',
+        'md5': 'c428b9effff18ff624d4f903bda26315',
+        'info_dict': {
+            'id': 'N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3',
+            'ext': 'mp4',
+            'duration': 2700,
+            'episode': '7 Tage ... unter harten Jungs',
+            'description': 'md5:0f215470dcd2b02f59f4bd10c963f072',
+            'upload_date': '20231005',
+            'timestamp': 1696491171,
+            'display_id': '7-tage/7-tage-unter-harten-jungs/hr-fernsehen',
+            'series': '7 Tage ...',
+            'channel': 'HR',
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:f6e6d5ffac41925c?w=960&ch=fa32ba69bc87989a',
+            'title': '7 Tage ... unter harten Jungs',
+            '_old_archive_ids': ['ardbetamediathek 94834686'],
         },
     }, {
         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
@@ -470,6 +323,176 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
     }, {
         'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
+        'only_matching': True,
+    }]
+
+    def _extract_episode_info(self, title):
+        patterns = [
+            # Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
+            # from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
+            r'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
+            # E.g.: title="Fritjof aus Norwegen (2) (AD)"
+            # from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
+            r'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
+            r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
+            # E.g.: title="Folge 25/42: Symmetrie"
+            # from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
+            # E.g.: title="Folge 1063 - Vertrauen"
+            # from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
+            r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
+            # As a fallback use the full title
+            r'(?P<title>.*)',
+        ]
+
+        return traverse_obj(patterns, (..., {partial(re.match, string=title)}, {
+            'season_number': ('season_number', {int_or_none}),
+            'episode_number': ('episode_number', {int_or_none}),
+            'episode': ((
+                ('episode', {str_or_none}),
+                ('ep_info', {lambda x: title.replace(x, '')}),
+                ('title', {str}),
+            ), {str.strip}),
+        }), get_all=False)
+
+    def _real_extract(self, url):
+        video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
+
+        page_data = self._download_json(
+            f'https://api.ardmediathek.de/page-gateway/pages/ard/item/{video_id}', video_id, query={
+                'embedded': 'false',
+                'mcV6': 'true',
+            })
+
+        player_data = traverse_obj(
+            page_data, ('widgets', lambda _, v: v['type'] in ('player_ondemand', 'player_live'), {dict}), get_all=False)
+        is_live = player_data.get('type') == 'player_live'
+        media_data = traverse_obj(player_data, ('mediaCollection', 'embedded', {dict}))
+
+        if player_data.get('blockedByFsk'):
+            self.raise_no_formats('This video is only available after 22:00', expected=True)
+
+        formats = []
+        subtitles = {}
+        for stream in traverse_obj(media_data, ('streams', ..., {dict})):
+            kind = stream.get('kind')
+            # Prioritize main stream over sign language and others
+            preference = 1 if kind == 'main' else None
+            for media in traverse_obj(stream, ('media', lambda _, v: url_or_none(v['url']))):
+                media_url = media['url']
+
+                audio_kind = traverse_obj(media, (
+                    'audios', 0, 'kind', {str}), default='').replace('standard', '')
+                lang_code = traverse_obj(media, ('audios', 0, 'languageCode', {str})) or 'deu'
+                lang = join_nonempty(lang_code, audio_kind)
+                language_preference = 10 if lang == 'deu' else -10
+
+                if determine_ext(media_url) == 'm3u8':
+                    fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                        media_url, video_id, m3u8_id=f'hls-{kind}', preference=preference, fatal=False, live=is_live)
+                    for f in fmts:
+                        f['language'] = lang
+                        f['language_preference'] = language_preference
+                    formats.extend(fmts)
+                    self._merge_subtitles(subs, target=subtitles)
+                else:
+                    formats.append({
+                        'url': media_url,
+                        'format_id': f'http-{kind}',
+                        'preference': preference,
+                        'language': lang,
+                        'language_preference': language_preference,
+                        **traverse_obj(media, {
+                            'format_note': ('forcedLabel', {str}),
+                            'width': ('maxHResolutionPx', {int_or_none}),
+                            'height': ('maxVResolutionPx', {int_or_none}),
+                            'vcodec': ('videoCodec', {str}),
+                        }),
+                    })
+
+        for sub in traverse_obj(media_data, ('subtitles', ..., {dict})):
+            for sources in traverse_obj(sub, ('sources', lambda _, v: url_or_none(v['url']))):
+                subtitles.setdefault(sub.get('languageCode') or 'deu', []).append({
+                    'url': sources['url'],
+                    'ext': {'webvtt': 'vtt', 'ebutt': 'ttml'}.get(sources.get('kind')),
+                })
+
+        age_limit = traverse_obj(page_data, ('fskRating', {lambda x: remove_start(x, 'FSK')}, {int_or_none}))
+        old_id = traverse_obj(page_data, ('tracking', 'atiCustomVars', 'contentId'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': is_live,
+            'age_limit': age_limit,
+            **traverse_obj(media_data, ('meta', {
+                'title': 'title',
+                'description': 'synopsis',
+                'timestamp': ('broadcastedOnDateTime', {parse_iso8601}),
+                'series': 'seriesTitle',
+                'thumbnail': ('images', 0, 'url', {url_or_none}),
+                'duration': ('durationSeconds', {int_or_none}),
+                'channel': 'clipSourceName',
+            })),
+            **self._extract_episode_info(page_data.get('title')),
+            '_old_archive_ids': [make_archive_id(ARDBetaMediathekIE, old_id)],
+        }
+
+
+class ARDMediathekCollectionIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https://
+        (?:(?:beta|www)\.)?ardmediathek\.de/
+        (?:[^/?#]+/)?
+        (?P<playlist>sendung|serie|sammlung)/
+        (?:(?P<display_id>[^?#]+?)/)?
+        (?P<id>[a-zA-Z0-9]+)
+        (?:/(?P<season>\d+)(?:/(?P<version>OV|AD))?)?/?(?:[?#]|$)'''
+    _GEO_COUNTRIES = ['DE']
+
+    _TESTS = [{
+        'url': 'https://www.ardmediathek.de/serie/quiz/staffel-1-originalversion/Y3JpZDovL3dkci5kZS9vbmUvcXVpeg/1/OV',
+        'info_dict': {
+            'id': 'Y3JpZDovL3dkci5kZS9vbmUvcXVpeg_1_OV',
+            'display_id': 'quiz/staffel-1-originalversion',
+            'title': 'Staffel 1 Originalversion',
+        },
+        'playlist_count': 3,
+    }, {
+        'url': 'https://www.ardmediathek.de/serie/babylon-berlin/staffel-4-mit-audiodeskription/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu/4/AD',
+        'info_dict': {
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu_4_AD',
+            'display_id': 'babylon-berlin/staffel-4-mit-audiodeskription',
+            'title': 'Staffel 4 mit Audiodeskription',
+        },
+        'playlist_count': 12,
+    }, {
+        'url': 'https://www.ardmediathek.de/serie/babylon-berlin/staffel-1/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu/1/',
+        'info_dict': {
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu_1',
+            'display_id': 'babylon-berlin/staffel-1',
+            'title': 'Staffel 1',
+        },
+        'playlist_count': 8,
+    }, {
+        'url': 'https://www.ardmediathek.de/sendung/tatort/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydA',
+        'info_dict': {
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydA',
+            'display_id': 'tatort',
+            'title': 'Tatort',
+        },
+        'playlist_mincount': 500,
+    }, {
+        'url': 'https://www.ardmediathek.de/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2',
+        'info_dict': {
+            'id': '5eOHzt8XB2sqeFXbIoJlg2',
+            'display_id': 'die-kirche-bleibt-im-dorf',
+            'title': 'Die Kirche bleibt im Dorf',
+            'description': 'Die Kirche bleibt im Dorf',
+        },
+        'playlist_count': 4,
     }, {
         # playlist of type 'sendung'
         'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/',
@@ -482,196 +505,48 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         # playlist of type 'sammlung'
         'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
         'only_matching': True,
-    }, {
-        'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3dkci5kZS9CZWl0cmFnLWQ2NDJjYWEzLTMwZWYtNGI4NS1iMTI2LTU1N2UxYTcxOGIzOQ/tatort-duo-koeln-leipzig-ihr-kinderlein-kommet',
-        'only_matching': True,
     }]
 
-    def _ARD_load_playlist_snippet(self, playlist_id, display_id, client, mode, page_number):
-        """ Query the ARD server for playlist information
-        and returns the data in "raw" format """
-        assert mode in ('sendung', 'serie', 'sammlung')
-        if mode in ('sendung', 'serie'):
-            graphQL = json.dumps({
-                'query': '''{
-                    showPage(
-                        client: "%s"
-                        showId: "%s"
-                        pageNumber: %d
-                    ) {
-                        pagination {
-                            pageSize
-                            totalElements
-                        }
-                        teasers {        # Array
-                            mediumTitle
-                            links { target { id href title } }
-                            type
-                        }
-                    }}''' % (client, playlist_id, page_number),
-            }).encode()
-        else:  # mode == 'sammlung'
-            graphQL = json.dumps({
-                'query': '''{
-                    morePage(
-                        client: "%s"
-                        compilationId: "%s"
-                        pageNumber: %d
-                    ) {
-                        widget {
-                            pagination {
-                                pageSize
-                                totalElements
-                            }
-                            teasers {        # Array
-                                mediumTitle
-                                links { target { id href title } }
-                                type
-                            }
-                        }
-                    }}''' % (client, playlist_id, page_number),
-            }).encode()
-        # Ressources for ARD graphQL debugging:
-        # https://api-test.ardmediathek.de/public-gateway
-        show_page = self._download_json(
-            'https://api.ardmediathek.de/public-gateway',
-            '[Playlist] %s' % display_id,
-            data=graphQL,
-            headers={'Content-Type': 'application/json'})['data']
-        # align the structure of the returned data:
-        if mode in ('sendung', 'serie'):
-            show_page = show_page['showPage']
-        else:  # mode == 'sammlung'
-            show_page = show_page['morePage']['widget']
-        return show_page
-
-    def _ARD_extract_playlist(self, url, playlist_id, display_id, client, mode):
-        """ Collects all playlist entries and returns them as info dict.
-        Supports playlists of mode 'sendung', 'serie', and 'sammlung',
-        as well as nested playlists. """
-        entries = []
-        pageNumber = 0
-        while True:  # iterate by pageNumber
-            show_page = self._ARD_load_playlist_snippet(
-                playlist_id, display_id, client, mode, pageNumber)
-            for teaser in show_page['teasers']:  # process playlist items
-                if '/compilation/' in teaser['links']['target']['href']:
-                    # alternativ cond.: teaser['type'] == "compilation"
-                    # => This is an nested compilation, e.g. like:
-                    # https://www.ardmediathek.de/ard/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2/
-                    link_mode = 'sammlung'
-                else:
-                    link_mode = 'video'
-
-                item_url = 'https://www.ardmediathek.de/%s/%s/%s/%s/%s' % (
-                    client, link_mode, display_id,
-                    # perform HTLM quoting of episode title similar to ARD:
-                    re.sub('^-|-$', '',  # remove '-' from begin/end
-                           re.sub('[^a-zA-Z0-9]+', '-',  # replace special chars by -
-                                  teaser['links']['target']['title'].lower()
-                                  .replace('ä', 'ae').replace('ö', 'oe')
-                                  .replace('ü', 'ue').replace('ß', 'ss'))),
-                    teaser['links']['target']['id'])
-                entries.append(self.url_result(
-                    item_url,
-                    ie=ARDBetaMediathekIE.ie_key()))
-
-            if (show_page['pagination']['pageSize'] * (pageNumber + 1)
-               >= show_page['pagination']['totalElements']):
-                # we've processed enough pages to get all playlist entries
-                break
-            pageNumber = pageNumber + 1
-
-        return self.playlist_result(entries, playlist_id, playlist_title=display_id)
+    _PAGE_SIZE = 100
 
     def _real_extract(self, url):
-        video_id, display_id, playlist_type, client, season_number = self._match_valid_url(url).group(
-            'id', 'display_id', 'playlist', 'client', 'season')
-        display_id, client = display_id or video_id, client or 'ard'
+        playlist_id, display_id, playlist_type, season_number, version = self._match_valid_url(url).group(
+            'id', 'display_id', 'playlist', 'season', 'version')
 
-        if playlist_type:
-            # TODO: Extract only specified season
-            return self._ARD_extract_playlist(url, video_id, display_id, client, playlist_type)
+        def call_api(page_num):
+            api_path = 'compilations/ard' if playlist_type == 'sammlung' else 'widgets/ard/asset'
+            return self._download_json(
+                f'https://api.ardmediathek.de/page-gateway/{api_path}/{playlist_id}', playlist_id,
+                f'Downloading playlist page {page_num}', query={
+                    'pageNumber': page_num,
+                    'pageSize': self._PAGE_SIZE,
+                    **({
+                        'seasoned': 'true',
+                        'seasonNumber': season_number,
+                        'withOriginalversion': 'true' if version == 'OV' else 'false',
+                        'withAudiodescription': 'true' if version == 'AD' else 'false',
+                    } if season_number else {}),
+                })
 
-        player_page = self._download_json(
-            'https://api.ardmediathek.de/public-gateway',
-            display_id, data=json.dumps({
-                'query': '''{
-  playerPage(client:"%s", clipId: "%s") {
-    blockedByFsk
-    broadcastedOn
-    maturityContentRating
-    mediaCollection {
-      _duration
-      _geoblocked
-      _isLive
-      _mediaArray {
-        _mediaStreamArray {
-          _quality
-          _server
-          _stream
-        }
-      }
-      _previewImage
-      _subtitleUrl
-      _type
-    }
-    show {
-      title
-    }
-    image {
-      src
-    }
-    synopsis
-    title
-    tracking {
-      atiCustomVars {
-        contentId
-      }
-    }
-  }
-}''' % (client, video_id),
-            }).encode(), headers={
-                'Content-Type': 'application/json'
-            })['data']['playerPage']
-        title = player_page['title']
-        content_id = str_or_none(try_get(
-            player_page, lambda x: x['tracking']['atiCustomVars']['contentId']))
-        media_collection = player_page.get('mediaCollection') or {}
-        if not media_collection and content_id:
-            media_collection = self._download_json(
-                'https://www.ardmediathek.de/play/media/' + content_id,
-                content_id, fatal=False) or {}
-        info = self._parse_media_info(
-            media_collection, content_id or video_id,
-            player_page.get('blockedByFsk'))
-        age_limit = None
-        description = player_page.get('synopsis')
-        maturity_content_rating = player_page.get('maturityContentRating')
-        if maturity_content_rating:
-            age_limit = int_or_none(maturity_content_rating.lstrip('FSK'))
-        if not age_limit and description:
-            age_limit = int_or_none(self._search_regex(
-                r'\(FSK\s*(\d+)\)\s*$', description, 'age limit', default=None))
-        info.update({
-            'age_limit': age_limit,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'timestamp': unified_timestamp(player_page.get('broadcastedOn')),
-            'series': try_get(player_page, lambda x: x['show']['title']),
-            'thumbnail': (media_collection.get('_previewImage')
-                          or try_get(player_page, lambda x: update_url(x['image']['src'], query=None, fragment=None))
-                          or self.get_thumbnail_from_html(display_id, url)),
-        })
-        info.update(self._ARD_extract_episode_info(info['title']))
-        return info
+        def fetch_page(page_num):
+            for item in traverse_obj(call_api(page_num), ('teasers', ..., {dict})):
+                item_id = traverse_obj(item, ('links', 'target', ('urlId', 'id')), 'id', get_all=False)
+                if not item_id or item_id == playlist_id:
+                    continue
+                item_mode = 'sammlung' if item.get('type') == 'compilation' else 'video'
+                yield self.url_result(
+                    f'https://www.ardmediathek.de/{item_mode}/{item_id}',
+                    ie=(ARDMediathekCollectionIE if item_mode == 'sammlung' else ARDBetaMediathekIE),
+                    **traverse_obj(item, {
+                        'id': ('id', {str}),
+                        'title': ('longTitle', {str}),
+                        'duration': ('duration', {int_or_none}),
+                        'timestamp': ('broadcastedOn', {parse_iso8601}),
+                    }))
 
-    def get_thumbnail_from_html(self, display_id, url):
-        webpage = self._download_webpage(url, display_id, fatal=False) or ''
-        return (
-            self._og_search_thumbnail(webpage, default=None)
-            or self._html_search_meta('thumbnailUrl', webpage, default=None))
+        page_data = call_api(0)
+        full_id = join_nonempty(playlist_id, season_number, version, delim='_')
+
+        return self.playlist_result(
+            OnDemandPagedList(fetch_page, self._PAGE_SIZE), full_id, display_id=display_id,
+            title=page_data.get('title'), description=page_data.get('synopsis'))

From f9fb3ce86e3c6a0c3c33b45392b8d7288bceba76 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 30 Dec 2023 22:27:36 +0100
Subject: [PATCH 760/871] [cleanup] Misc (#8598)

Authored by: bashonly, pukkandan, seproDev, Grub4K

Co-authored-by: bashonly <bashonly@protonmail.com>
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 .github/workflows/core.yml         |  4 ++--
 .github/workflows/quick-test.yml   |  4 ++--
 Collaborators.md                   | 18 ++++++++++--------
 README.md                          |  3 ++-
 devscripts/changelog_override.json |  6 ++++++
 devscripts/make_changelog.py       | 18 ++----------------
 devscripts/make_issue_template.py  |  6 +-----
 requirements.txt                   |  1 -
 test/test_YoutubeDL.py             |  2 +-
 test/test_update.py                |  2 +-
 test/test_utils.py                 |  2 ++
 yt-dlp.cmd                         |  2 +-
 yt-dlp.sh                          |  2 +-
 yt_dlp/YoutubeDL.py                |  8 +++++++-
 yt_dlp/extractor/banbye.py         |  2 +-
 yt_dlp/extractor/duoplay.py        | 15 ---------------
 yt_dlp/extractor/floatplane.py     | 10 +++++-----
 yt_dlp/extractor/generic.py        |  2 +-
 yt_dlp/extractor/panopto.py        |  2 +-
 yt_dlp/extractor/radiofrance.py    |  6 +++---
 yt_dlp/extractor/wordpress.py      |  2 +-
 yt_dlp/extractor/youtube.py        |  1 +
 yt_dlp/networking/__init__.py      |  1 -
 yt_dlp/networking/_helper.py       |  8 ++++----
 yt_dlp/networking/_requests.py     |  4 +++-
 yt_dlp/networking/_websockets.py   | 14 ++++++++++----
 yt_dlp/networking/websocket.py     |  2 +-
 yt_dlp/socks.py                    |  2 +-
 yt_dlp/utils/_utils.py             |  8 ++++++--
 yt_dlp/utils/traversal.py          |  2 +-
 30 files changed, 77 insertions(+), 82 deletions(-)

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index ded7e6d61..eaaf03dee 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -36,8 +36,8 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        # CPython 3.11 is in quick-test
-        python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.8, pypy-3.10]
+        # CPython 3.8 is in quick-test
+        python-version: ['3.9', '3.10', '3.11', '3.12', pypy-3.8, pypy-3.10]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
         - os: windows-latest
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index 1ccfbe836..84fca62d4 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.11
+    - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:
-        python-version: '3.11'
+        python-version: '3.8'
     - name: Install test requirements
       run: pip install pytest -r requirements.txt
     - name: Run tests
diff --git a/Collaborators.md b/Collaborators.md
index 70ab616f1..894a853c9 100644
--- a/Collaborators.md
+++ b/Collaborators.md
@@ -29,6 +29,7 @@ ## [coletdjnz](https://github.com/coletdjnz)
 [![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/coletdjnz)
 
 * Improved plugin architecture
+* Rewrote the networking infrastructure, implemented support for `requests`
 * YouTube improvements including: age-gate bypass, private playlists, multiple-clients (to avoid throttling) and a lot of under-the-hood improvements
 * Added support for new websites YoutubeWebArchive, MainStreaming, PRX, nzherald, Mediaklikk, StarTV etc
 * Improved/fixed support for Patreon, panopto, gfycat, itv, pbs, SouthParkDE etc
@@ -46,16 +47,17 @@ ## [Ashish0804](https://github.com/Ashish0804) <sub><sup>[Inactive]</sup></sub>
 
 ## [bashonly](https://github.com/bashonly)
 
-* `--update-to`, automated release, nightly builds
-* `--cookies-from-browser` support for Firefox containers
-* Added support for new websites Genius, Kick, NBCStations, Triller, VideoKen etc
-* Improved/fixed support for Anvato, Brightcove, Instagram, ParamountPlus, Reddit, SlidesLive, TikTok, Twitter, Vimeo etc
+* `--update-to`, self-updater rewrite, automated/nightly/master releases
+* `--cookies-from-browser` support for Firefox containers, external downloader cookie handling overhaul
+* Added support for new websites like Dacast, Kick, NBCStations, Triller, VideoKen, Weverse, WrestleUniverse etc
+* Improved/fixed support for Anvato, Brightcove, Reddit, SlidesLive, TikTok, Twitter, Vimeo etc
 
 
 ## [Grub4K](https://github.com/Grub4K)
 
-[![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/Grub4K) [![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/Grub4K)
+[![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/Grub4K) [![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/Grub4K)
 
-* `--update-to`, automated release, nightly builds
-* Rework internals like `traverse_obj`, various core refactors and bugs fixes
-* Helped fix crunchyroll, Twitter, wrestleuniverse, wistia, slideslive etc
+* `--update-to`, self-updater rewrite, automated/nightly/master releases
+* Reworked internals like `traverse_obj`, various core refactors and bugs fixes
+* Implemented proper progress reporting for parallel downloads
+* Improved/fixed/added Bundestag, crunchyroll, pr0gramm, Twitter, WrestleUniverse etc
diff --git a/README.md b/README.md
index 06aceec02..4f8fcb6c9 100644
--- a/README.md
+++ b/README.md
@@ -159,6 +159,7 @@ ### Differences in default behavior
 * yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filter` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this
 * yt-dlp versions between 2021.11.10 and 2023.06.21 estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values
 * yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests.
+* The sub-module `swfinterp` is removed.
 
 For ease of use, a few more compat options are available:
 
@@ -299,7 +300,7 @@ ### Misc
 
 * [**pycryptodomex**](https://github.com/Legrandin/pycryptodome)\* - For decrypting AES-128 HLS streams and various other data. Licensed under [BSD-2-Clause](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
 * [**phantomjs**](https://github.com/ariya/phantomjs) - Used in extractors where javascript needs to be run. Licensed under [BSD-3-Clause](https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD)
-* [**secretstorage**](https://github.com/mitya57/secretstorage) - For `--cookies-from-browser` to access the **Gnome** keyring while decrypting cookies of **Chromium**-based browsers on **Linux**. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
+* [**secretstorage**](https://github.com/mitya57/secretstorage)\* - For `--cookies-from-browser` to access the **Gnome** keyring while decrypting cookies of **Chromium**-based browsers on **Linux**. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
 * Any external downloader that you want to use with `--downloader`
 
 ### Deprecated
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index ca811cb65..8c5286432 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -114,5 +114,11 @@
         "action": "add",
         "when": "f04b5bedad7b281bee9814686bba1762bae092eb",
         "short": "[priority] Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)\n\t- Disallow smuggling of arbitrary `http_headers`; extractors now only use specific headers"
+    },
+    {
+        "action": "change",
+        "when": "15f22b4880b6b3f71f350c64d70976ae65b9f1ca",
+        "short": "[webvtt] Allow spaces before newlines for CueBlock (#7681)",
+        "authors": ["TSRBerry"]
     }
 ]
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index d0e893e58..123eebc2a 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -40,20 +40,6 @@ def subgroup_lookup(cls):
         return {
             name: group
             for group, names in {
-                cls.CORE: {
-                    'aes',
-                    'cache',
-                    'compat_utils',
-                    'compat',
-                    'cookies',
-                    'dependencies',
-                    'formats',
-                    'jsinterp',
-                    'outtmpl',
-                    'plugins',
-                    'update',
-                    'utils',
-                },
                 cls.MISC: {
                     'build',
                     'ci',
@@ -404,9 +390,9 @@ def groups(self):
             if not group:
                 if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
                     group = CommitGroup.EXTRACTOR
+                    logger.error(f'Assuming [ie] group for {commit.short!r}')
                 else:
-                    group = CommitGroup.POSTPROCESSOR
-                logger.warning(f'Failed to map {commit.short!r}, selected {group.name.lower()}')
+                    group = CommitGroup.CORE
 
             commit_info = CommitInfo(
                 details, sub_details, message.strip(),
diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index 6c85e200f..a5d59f3c0 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -9,11 +9,7 @@
 
 import re
 
-from devscripts.utils import (
-    get_filename_args,
-    read_file,
-    write_file,
-)
+from devscripts.utils import get_filename_args, read_file, write_file
 
 VERBOSE_TMPL = '''
   - type: checkboxes
diff --git a/requirements.txt b/requirements.txt
index d983fa03f..06ff82a80 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
 mutagen
 pycryptodomex
-websockets
 brotli; implementation_name=='cpython'
 brotlicffi; implementation_name!='cpython'
 certifi
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 8bff08314..0087cbc94 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -730,7 +730,7 @@ def expect_same_infodict(out):
                 self.assertEqual(got_dict.get(info_field), expected, info_field)
             return True
 
-        test('%()j', (expect_same_infodict, str))
+        test('%()j', (expect_same_infodict, None))
 
         # NA placeholder
         NA_TEST_OUTTMPL = '%(uploader_date)s-%(width)d-%(x|def)s-%(id)s.%(ext)s'
diff --git a/test/test_update.py b/test/test_update.py
index a5a388c10..bc139562f 100644
--- a/test/test_update.py
+++ b/test/test_update.py
@@ -9,7 +9,7 @@
 
 
 from test.helper import FakeYDL, report_warning
-from yt_dlp.update import Updater, UpdateInfo
+from yt_dlp.update import UpdateInfo, Updater
 
 
 # XXX: Keep in sync with yt_dlp.update.UPDATE_SOURCES
diff --git a/test/test_utils.py b/test/test_utils.py
index 6c8571f98..c3e387cd0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2110,6 +2110,8 @@ def test_traverse_obj(self):
         self.assertEqual(traverse_obj(_TEST_DATA, (..., {str_or_none})),
                          [item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
                          msg='Function in set should be a transformation')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('fail', {lambda _: 'const'})), 'const',
+                         msg='Function in set should always be called')
         if __debug__:
             with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
                 traverse_obj(_TEST_DATA, set())
diff --git a/yt-dlp.cmd b/yt-dlp.cmd
index aa4500f9f..5537e0ea9 100644
--- a/yt-dlp.cmd
+++ b/yt-dlp.cmd
@@ -1 +1 @@
-@py -bb -Werror -Xdev "%~dp0yt_dlp\__main__.py" %*
+@py -Werror -Xdev "%~dp0yt_dlp\__main__.py" %*
diff --git a/yt-dlp.sh b/yt-dlp.sh
index 22a69250c..ce74df801 100755
--- a/yt-dlp.sh
+++ b/yt-dlp.sh
@@ -1,2 +1,2 @@
 #!/usr/bin/env sh
-exec "${PYTHON:-python3}" -bb -Werror -Xdev "$(dirname "$(realpath "$0")")/yt_dlp/__main__.py" "$@"
+exec "${PYTHON:-python3}" -Werror -Xdev "$(dirname "$(realpath "$0")")/yt_dlp/__main__.py" "$@"
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 5e28fd0e2..8d96498a6 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -60,7 +60,13 @@
     get_postprocessor,
 )
 from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
-from .update import REPOSITORY, _get_system_deprecation, _make_label, current_git_head, detect_variant
+from .update import (
+    REPOSITORY,
+    _get_system_deprecation,
+    _make_label,
+    current_git_head,
+    detect_variant,
+)
 from .utils import (
     DEFAULT_OUTTMPL,
     IDENTITY,
diff --git a/yt_dlp/extractor/banbye.py b/yt_dlp/extractor/banbye.py
index dfcc82f02..67af29a96 100644
--- a/yt_dlp/extractor/banbye.py
+++ b/yt_dlp/extractor/banbye.py
@@ -152,7 +152,7 @@ def page_func(page_num):
                 'sort': 'new',
                 'limit': self._PAGE_SIZE,
                 'offset': page_num * self._PAGE_SIZE,
-            }, note=f'Downloading page {page_num+1}')
+            }, note=f'Downloading page {page_num + 1}')
             return [
                 self.url_result(f"{self._VIDEO_BASE}/{video['_id']}", BanByeIE)
                 for video in data['items']
diff --git a/yt_dlp/extractor/duoplay.py b/yt_dlp/extractor/duoplay.py
index e57fa7924..7d3f39942 100644
--- a/yt_dlp/extractor/duoplay.py
+++ b/yt_dlp/extractor/duoplay.py
@@ -53,21 +53,6 @@ class DuoplayIE(InfoExtractor):
             'episode_id': 14,
             'release_year': 2010,
         },
-    }, {
-        'note': 'Movie',
-        'url': 'https://duoplay.ee/4325/naljamangud',
-        'md5': '2b0bcac4159a08b1844c2bfde06b1199',
-        'info_dict': {
-            'id': '4325',
-            'ext': 'mp4',
-            'title': 'Näljamängud',
-            'thumbnail': r're:https://.+\.jpg(?:\?c=\d+)?$',
-            'description': 'md5:fb35f5eb2ff46cdb82e4d5fbe7b49a13',
-            'cast': ['Jennifer Lawrence', 'Josh Hutcherson', 'Liam Hemsworth'],
-            'upload_date': '20231109',
-            'timestamp': 1699552800,
-            'release_year': 2012,
-        },
     }, {
         'note': 'Movie without expiry',
         'url': 'https://duoplay.ee/5501/pilvede-all.-neljas-ode',
diff --git a/yt_dlp/extractor/floatplane.py b/yt_dlp/extractor/floatplane.py
index 09abb40bf..2cf4d4e64 100644
--- a/yt_dlp/extractor/floatplane.py
+++ b/yt_dlp/extractor/floatplane.py
@@ -173,8 +173,8 @@ def format_path(params):
                 'formats': formats,
             })
 
-        uploader_url = format_field(traverse_obj(
-            post_data, 'creator'), 'urlname', 'https://www.floatplane.com/channel/%s/home', default=None)
+        uploader_url = format_field(
+            post_data, [('creator', 'urlname')], 'https://www.floatplane.com/channel/%s/home') or None
         channel_url = urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname')))
 
         post_info = {
@@ -248,7 +248,7 @@ def _fetch_page(self, display_id, creator_id, channel_id, page):
         for post in page_data or []:
             yield self.url_result(
                 f'https://www.floatplane.com/post/{post["id"]}',
-                ie=FloatplaneIE, video_id=post['id'], video_title=post.get('title'),
+                FloatplaneIE, id=post['id'], title=post.get('title'),
                 release_timestamp=parse_iso8601(post.get('releaseDate')))
 
     def _real_extract(self, url):
@@ -264,5 +264,5 @@ def _real_extract(self, url):
 
         return self.playlist_result(OnDemandPagedList(functools.partial(
             self._fetch_page, display_id, creator_data['id'], channel_data.get('id')), self._PAGE_SIZE),
-            display_id, playlist_title=channel_data.get('title') or creator_data.get('title'),
-            playlist_description=channel_data.get('about') or creator_data.get('about'))
+            display_id, title=channel_data.get('title') or creator_data.get('title'),
+            description=channel_data.get('about') or creator_data.get('about'))
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 606b4f5d1..1f0011c09 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -35,8 +35,8 @@
     unified_timestamp,
     unsmuggle_url,
     update_url_query,
-    urlhandle_detect_ext,
     url_or_none,
+    urlhandle_detect_ext,
     urljoin,
     variadic,
     xpath_attr,
diff --git a/yt_dlp/extractor/panopto.py b/yt_dlp/extractor/panopto.py
index 5ab2b2bce..ddea32d70 100644
--- a/yt_dlp/extractor/panopto.py
+++ b/yt_dlp/extractor/panopto.py
@@ -536,7 +536,7 @@ def _fetch_page(self, base_url, query_params, display_id, page):
         }
 
         response = self._call_api(
-            base_url, '/Services/Data.svc/GetSessions', f'{display_id} page {page+1}',
+            base_url, '/Services/Data.svc/GetSessions', f'{display_id} page {page + 1}',
             data={'queryParameters': params}, fatal=False)
 
         for result in get_first(response, 'Results', default=[]):
diff --git a/yt_dlp/extractor/radiofrance.py b/yt_dlp/extractor/radiofrance.py
index ec1b97631..6bd6fe9b6 100644
--- a/yt_dlp/extractor/radiofrance.py
+++ b/yt_dlp/extractor/radiofrance.py
@@ -264,7 +264,7 @@ def _real_extract(self, url):
         }
 
 
-class RadioFrancePlaylistBase(RadioFranceBaseIE):
+class RadioFrancePlaylistBaseIE(RadioFranceBaseIE):
     """Subclasses must set _METADATA_KEY"""
 
     def _call_api(self, content_id, cursor, page_num):
@@ -308,7 +308,7 @@ def _real_extract(self, url):
             })})
 
 
-class RadioFrancePodcastIE(RadioFrancePlaylistBase):
+class RadioFrancePodcastIE(RadioFrancePlaylistBaseIE):
     _VALID_URL = rf'''(?x)
         {RadioFranceBaseIE._VALID_URL_BASE}
         /(?:{RadioFranceBaseIE._STATIONS_RE})
@@ -369,7 +369,7 @@ def _call_api(self, podcast_id, cursor, page_num):
             note=f'Downloading page {page_num}', query={'pageCursor': cursor})
 
 
-class RadioFranceProfileIE(RadioFrancePlaylistBase):
+class RadioFranceProfileIE(RadioFrancePlaylistBaseIE):
     _VALID_URL = rf'{RadioFranceBaseIE._VALID_URL_BASE}/personnes/(?P<id>[\w-]+)'
 
     _TESTS = [{
diff --git a/yt_dlp/extractor/wordpress.py b/yt_dlp/extractor/wordpress.py
index 53820b57a..378d99dbc 100644
--- a/yt_dlp/extractor/wordpress.py
+++ b/yt_dlp/extractor/wordpress.py
@@ -70,7 +70,7 @@ def _extract_from_webpage(self, url, webpage):
                 'height': int_or_none(traverse_obj(track, ('dimensions', 'original', 'height'))),
                 'width': int_or_none(traverse_obj(track, ('dimensions', 'original', 'width'))),
             } for track in traverse_obj(playlist_json, ('tracks', ...), expected_type=dict)]
-            yield self.playlist_result(entries, self._generic_id(url) + f'-wp-playlist-{i+1}', 'Wordpress Playlist')
+            yield self.playlist_result(entries, self._generic_id(url) + f'-wp-playlist-{i + 1}', 'Wordpress Playlist')
 
 
 class WordpressMiniAudioPlayerEmbedIE(InfoExtractor):
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 5b14b187a..88126d11f 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -5297,6 +5297,7 @@ def _extract_webpage(self, url, item_id, fatal=True):
             # See: https://github.com/yt-dlp/yt-dlp/issues/116
             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
                 retry.error = ExtractorError('Incomplete yt initial data received')
+                data = None
                 continue
 
         return webpage, data
diff --git a/yt_dlp/networking/__init__.py b/yt_dlp/networking/__init__.py
index 96c5a0678..acadc0147 100644
--- a/yt_dlp/networking/__init__.py
+++ b/yt_dlp/networking/__init__.py
@@ -28,4 +28,3 @@
     pass
 except Exception as e:
     warnings.warn(f'Failed to import "websockets" request handler: {e}' + bug_reports_message())
-
diff --git a/yt_dlp/networking/_helper.py b/yt_dlp/networking/_helper.py
index a6fa3550b..d79dd7953 100644
--- a/yt_dlp/networking/_helper.py
+++ b/yt_dlp/networking/_helper.py
@@ -219,7 +219,7 @@ def _socket_connect(ip_addr, timeout, source_address):
             sock.bind(source_address)
         sock.connect(sa)
         return sock
-    except socket.error:
+    except OSError:
         sock.close()
         raise
 
@@ -237,7 +237,7 @@ def create_socks_proxy_socket(dest_addr, proxy_args, proxy_ip_addr, timeout, sou
             sock.bind(source_address)
         sock.connect(dest_addr)
         return sock
-    except socket.error:
+    except OSError:
         sock.close()
         raise
 
@@ -255,7 +255,7 @@ def create_connection(
     host, port = address
     ip_addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
     if not ip_addrs:
-        raise socket.error('getaddrinfo returns an empty list')
+        raise OSError('getaddrinfo returns an empty list')
     if source_address is not None:
         af = socket.AF_INET if ':' not in source_address[0] else socket.AF_INET6
         ip_addrs = [addr for addr in ip_addrs if addr[0] == af]
@@ -272,7 +272,7 @@ def create_connection(
             # https://bugs.python.org/issue36820
             err = None
             return sock
-        except socket.error as e:
+        except OSError as e:
             err = e
 
     try:
diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py
index 9fb1d75f4..e129110ca 100644
--- a/yt_dlp/networking/_requests.py
+++ b/yt_dlp/networking/_requests.py
@@ -188,6 +188,7 @@ class RequestsSession(requests.sessions.Session):
     """
     Ensure unified redirect method handling with our urllib redirect handler.
     """
+
     def rebuild_method(self, prepared_request, response):
         new_method = get_redirect_method(prepared_request.method, response.status_code)
 
@@ -218,6 +219,7 @@ def filter(self, record):
 
 class Urllib3LoggingHandler(logging.Handler):
     """Redirect urllib3 logs to our logger"""
+
     def __init__(self, logger, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._logger = logger
@@ -367,7 +369,7 @@ def _new_conn(self):
                 self, f'Connection to {self.host} timed out. (connect timeout={self.timeout})') from e
         except SocksProxyError as e:
             raise urllib3.exceptions.ProxyError(str(e), e) from e
-        except (OSError, socket.error) as e:
+        except OSError as e:
             raise urllib3.exceptions.NewConnectionError(
                 self, f'Failed to establish a new connection: {e}') from e
 
diff --git a/yt_dlp/networking/_websockets.py b/yt_dlp/networking/_websockets.py
index ad85554e4..ed64080d6 100644
--- a/yt_dlp/networking/_websockets.py
+++ b/yt_dlp/networking/_websockets.py
@@ -5,20 +5,26 @@
 import ssl
 import sys
 
-from ._helper import create_connection, select_proxy, make_socks_proxy_opts, create_socks_proxy_socket
-from .common import Response, register_rh, Features
+from ._helper import (
+    create_connection,
+    create_socks_proxy_socket,
+    make_socks_proxy_opts,
+    select_proxy,
+)
+from .common import Features, Response, register_rh
 from .exceptions import (
     CertificateVerifyError,
     HTTPError,
+    ProxyError,
     RequestError,
     SSLError,
-    TransportError, ProxyError,
+    TransportError,
 )
 from .websocket import WebSocketRequestHandler, WebSocketResponse
 from ..compat import functools
 from ..dependencies import websockets
-from ..utils import int_or_none
 from ..socks import ProxyError as SocksProxyError
+from ..utils import int_or_none
 
 if not websockets:
     raise ImportError('websockets is not installed')
diff --git a/yt_dlp/networking/websocket.py b/yt_dlp/networking/websocket.py
index 09fcf78ac..0e7e73c9e 100644
--- a/yt_dlp/networking/websocket.py
+++ b/yt_dlp/networking/websocket.py
@@ -2,7 +2,7 @@
 
 import abc
 
-from .common import Response, RequestHandler
+from .common import RequestHandler, Response
 
 
 class WebSocketResponse(Response):
diff --git a/yt_dlp/socks.py b/yt_dlp/socks.py
index e7f41d7e2..b4957ac2e 100644
--- a/yt_dlp/socks.py
+++ b/yt_dlp/socks.py
@@ -49,7 +49,7 @@ class Socks5AddressType:
     ATYP_IPV6 = 0x04
 
 
-class ProxyError(socket.error):
+class ProxyError(OSError):
     ERR_SUCCESS = 0x00
 
     def __init__(self, code=None, msg=None):
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 361617c02..89a0d4cff 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -558,7 +558,7 @@ def decode(self, s):
                     s = self._close_object(e)
                     if s is not None:
                         continue
-                raise type(e)(f'{e.msg} in {s[e.pos-10:e.pos+10]!r}', s, e.pos)
+                raise type(e)(f'{e.msg} in {s[e.pos - 10:e.pos + 10]!r}', s, e.pos)
         assert False, 'Too many attempts to decode JSON'
 
 
@@ -1885,6 +1885,7 @@ def setproctitle(title):
     buf = ctypes.create_string_buffer(len(title_bytes))
     buf.value = title_bytes
     try:
+        # PR_SET_NAME = 15      Ref: /usr/include/linux/prctl.h
         libc.prctl(15, buf, 0, 0, 0)
     except AttributeError:
         return  # Strange libc, just skip this
@@ -2260,6 +2261,9 @@ def __getitem__(self, idx):
             raise self.IndexError()
         return entries[0]
 
+    def __bool__(self):
+        return bool(self.getslice(0, 1))
+
 
 class OnDemandPagedList(PagedList):
     """Download pages until a page with less than maximum results"""
@@ -5070,7 +5074,7 @@ def truncate_string(s, left, right=0):
     assert left > 3 and right >= 0
     if s is None or len(s) <= left + right:
         return s
-    return f'{s[:left-3]}...{s[-right:] if right else ""}'
+    return f'{s[:left - 3]}...{s[-right:] if right else ""}'
 
 
 def orderedSet_from_options(options, alias_dict, *, use_regex=False, start=None):
diff --git a/yt_dlp/utils/traversal.py b/yt_dlp/utils/traversal.py
index ff5703198..5a2f69fcc 100644
--- a/yt_dlp/utils/traversal.py
+++ b/yt_dlp/utils/traversal.py
@@ -23,7 +23,7 @@ def traverse_obj(
 
     >>> obj = [{}, {"key": "value"}]
     >>> traverse_obj(obj, (1, "key"))
-    "value"
+    'value'
 
     Each of the provided `paths` is tested and the first producing a valid result will be returned.
     The next path will also be tested if the path branched but no results could be found.

From f10589e3453009bb523f55849bba144c9b91cf2a Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sat, 30 Dec 2023 15:39:06 -0600
Subject: [PATCH 761/871] [docs] Update youtube-dl merge commit in `README.md`

Authored by: bashonly
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4f8fcb6c9..cfd861c56 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,7 @@
 
 # NEW FEATURES
 
-* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@66ab08**](https://github.com/ytdl-org/youtube-dl/commit/66ab0814c4baa2dc79c2dd5287bc0ad61a37c5b9) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
+* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@be008e6**](https://github.com/ytdl-org/youtube-dl/commit/be008e657d79832642e2158557c899249c9e31cd) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 

From 9f40cd289665b2fb8a05ccaf9721b3b2ca0f39c7 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Sat, 30 Dec 2023 21:43:13 +0000
Subject: [PATCH 762/871] Release 2023.12.30

Created by: bashonly

:ci skip all :ci run dl
---
 CONTRIBUTORS      |  14 ++++
 Changelog.md      |  87 ++++++++++++++++++++
 supportedsites.md | 203 ++++++++++++----------------------------------
 yt_dlp/version.py |   6 +-
 4 files changed, 154 insertions(+), 156 deletions(-)

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 8b6b3671e..adcc92144 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -528,3 +528,17 @@ almx
 elivinsky
 starius
 TravisDupes
+amir16yp
+Fymyte
+Ganesh910
+hashFactory
+kclauhk
+Kyraminol
+lstrojny
+middlingphys
+NickCis
+nicodato
+prettykool
+S-Aarab
+sonmezberkay
+TSRBerry
diff --git a/Changelog.md b/Changelog.md
index 6115446cb..30de9072e 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,93 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.12.30
+
+#### Core changes
+- [Fix format selection parse error for CPython 3.12](https://github.com/yt-dlp/yt-dlp/commit/00cdda4f6fe18712ced13dbc64b7ea10f323e268) ([#8797](https://github.com/yt-dlp/yt-dlp/issues/8797)) by [Grub4K](https://github.com/Grub4K)
+- [Let `read_stdin` obey `--quiet`](https://github.com/yt-dlp/yt-dlp/commit/a174c453ee1e853c584ceadeac17eef2bd433dc5) by [pukkandan](https://github.com/pukkandan)
+- [Merged with youtube-dl be008e6](https://github.com/yt-dlp/yt-dlp/commit/65de7d204ce88c0225df1321060304baab85dbd8) by [bashonly](https://github.com/bashonly), [dirkf](https://github.com/dirkf), [Grub4K](https://github.com/Grub4K)
+- [Parse `release_year` from `release_date`](https://github.com/yt-dlp/yt-dlp/commit/1732eccc0a40256e076bf0435a29f0f1d8419280) ([#8524](https://github.com/yt-dlp/yt-dlp/issues/8524)) by [seproDev](https://github.com/seproDev)
+- [Release workflow and Updater cleanup](https://github.com/yt-dlp/yt-dlp/commit/632b8ee54eb2df8ac6e20746a0bd95b7ebb053aa) ([#8640](https://github.com/yt-dlp/yt-dlp/issues/8640)) by [bashonly](https://github.com/bashonly)
+- [Remove Python 3.7 support](https://github.com/yt-dlp/yt-dlp/commit/f4b95acafcd69a50040730dfdf732e797278fdcc) ([#8361](https://github.com/yt-dlp/yt-dlp/issues/8361)) by [bashonly](https://github.com/bashonly)
+- [Support `NO_COLOR` environment variable](https://github.com/yt-dlp/yt-dlp/commit/a0b19d319a6ce8b7059318fa17a34b144fde1785) ([#8385](https://github.com/yt-dlp/yt-dlp/issues/8385)) by [Grub4K](https://github.com/Grub4K), [prettykool](https://github.com/prettykool)
+- **outtmpl**: [Support multiplication](https://github.com/yt-dlp/yt-dlp/commit/993edd3f6e17e966c763bc86dc34125445cec6b6) by [pukkandan](https://github.com/pukkandan)
+- **utils**: `traverse_obj`: [Move `is_user_input` into output template](https://github.com/yt-dlp/yt-dlp/commit/0b6f829b1dfda15d3c1d7d1fbe4ea6102c26dd24) ([#8673](https://github.com/yt-dlp/yt-dlp/issues/8673)) by [Grub4K](https://github.com/Grub4K)
+- **webvtt**: [Allow spaces before newlines for CueBlock](https://github.com/yt-dlp/yt-dlp/commit/15f22b4880b6b3f71f350c64d70976ae65b9f1ca) ([#7681](https://github.com/yt-dlp/yt-dlp/issues/7681)) by [TSRBerry](https://github.com/TSRBerry) (With fixes in [298230e](https://github.com/yt-dlp/yt-dlp/commit/298230e550886b746c266724dd701d842ca2696e) by [pukkandan](https://github.com/pukkandan))
+
+#### Extractor changes
+- [Add `media_type` field](https://github.com/yt-dlp/yt-dlp/commit/e370f9ec36972d06100a3db893b397bfc1b07b4d) by [trainman261](https://github.com/trainman261)
+- [Extract from `media` elements in SMIL manifests](https://github.com/yt-dlp/yt-dlp/commit/ddb2d7588bea48bae965dbfabe6df6550c9d3d43) ([#8504](https://github.com/yt-dlp/yt-dlp/issues/8504)) by [seproDev](https://github.com/seproDev)
+- **abematv**: [Fix season metadata](https://github.com/yt-dlp/yt-dlp/commit/cc07f5cc85d9e2a6cd0bedb9d961665eea0d6047) ([#8607](https://github.com/yt-dlp/yt-dlp/issues/8607)) by [middlingphys](https://github.com/middlingphys)
+- **allstar**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/3237f8ba29fe13bf95ff42b1e48b5b5109715feb) ([#8274](https://github.com/yt-dlp/yt-dlp/issues/8274)) by [S-Aarab](https://github.com/S-Aarab)
+- **altcensored**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/3f90813f0617e0d21302398010de7496c9ae36aa) ([#8291](https://github.com/yt-dlp/yt-dlp/issues/8291)) by [drzraf](https://github.com/drzraf)
+- **ard**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/5f009a094f0e8450792b097c4c8273622778052d) ([#8878](https://github.com/yt-dlp/yt-dlp/issues/8878)) by [seproDev](https://github.com/seproDev)
+- **ardbetamediathek**: [Fix series extraction](https://github.com/yt-dlp/yt-dlp/commit/1f8bd8eba82ba10ddb49ee7cc0be4540dab103d5) ([#8687](https://github.com/yt-dlp/yt-dlp/issues/8687)) by [lstrojny](https://github.com/lstrojny)
+- **bbc**
+    - [Extract more formats](https://github.com/yt-dlp/yt-dlp/commit/c919b68f7e79ea5010f75f648d3c9e45405a8011) ([#8321](https://github.com/yt-dlp/yt-dlp/issues/8321)) by [barsnick](https://github.com/barsnick), [dirkf](https://github.com/dirkf)
+    - [Fix JSON parsing bug](https://github.com/yt-dlp/yt-dlp/commit/19741ab8a401ec64d5e84fdbfcfb141d105e7bc8) by [bashonly](https://github.com/bashonly)
+- **bfmtv**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/4903f452b68efb62dadf22e81be8c7934fc743e7) ([#8651](https://github.com/yt-dlp/yt-dlp/issues/8651)) by [bashonly](https://github.com/bashonly)
+- **bilibili**: [Support courses and interactive videos](https://github.com/yt-dlp/yt-dlp/commit/9f09bdcfcb8e2b4b2decdc30d35d34b993bc7a94) ([#8343](https://github.com/yt-dlp/yt-dlp/issues/8343)) by [c-basalt](https://github.com/c-basalt)
+- **bitchute**: [Fix and improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/b1a1ec1540605d2ea7abdb63336ffb1c56bf6316) ([#8507](https://github.com/yt-dlp/yt-dlp/issues/8507)) by [SirElderling](https://github.com/SirElderling)
+- **box**: [Fix formats extraction](https://github.com/yt-dlp/yt-dlp/commit/5a230233d6fce06f4abd1fce0dc92b948e6f780b) ([#8649](https://github.com/yt-dlp/yt-dlp/issues/8649)) by [bashonly](https://github.com/bashonly)
+- **bundestag**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/00a3e47bf5440c96025a76e08337ff2a475ed83e) ([#8783](https://github.com/yt-dlp/yt-dlp/issues/8783)) by [Grub4K](https://github.com/Grub4K)
+- **drtv**: [Set default ext for m3u8 formats](https://github.com/yt-dlp/yt-dlp/commit/f96ab86cd837b1b5823baa87d144e15322ee9298) ([#8590](https://github.com/yt-dlp/yt-dlp/issues/8590)) by [seproDev](https://github.com/seproDev)
+- **duoplay**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/66a0127d45033c698bdbedf162cddc55d9e7b906) ([#8542](https://github.com/yt-dlp/yt-dlp/issues/8542)) by [glensc](https://github.com/glensc)
+- **eplus**: [Add login support and DRM detection](https://github.com/yt-dlp/yt-dlp/commit/d5d1517e7d838500800d193ac3234b06e89654cd) ([#8661](https://github.com/yt-dlp/yt-dlp/issues/8661)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **facebook**
+    - [Fix Memories extraction](https://github.com/yt-dlp/yt-dlp/commit/c39358a54bc6675ae0c50b81024e5a086e41656a) ([#8681](https://github.com/yt-dlp/yt-dlp/issues/8681)) by [kclauhk](https://github.com/kclauhk)
+    - [Improve subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/9cafb9ff17e14475a35c9a58b5bb010c86c9db4b) ([#8296](https://github.com/yt-dlp/yt-dlp/issues/8296)) by [kclauhk](https://github.com/kclauhk)
+- **floatplane**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/628fa244bbce2ad39775a5959e99588f30cac152) ([#8639](https://github.com/yt-dlp/yt-dlp/issues/8639)) by [seproDev](https://github.com/seproDev)
+- **francetv**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/71f28097fec1c9e029f74b68a4eadc8915399840) ([#8409](https://github.com/yt-dlp/yt-dlp/issues/8409)) by [Fymyte](https://github.com/Fymyte)
+- **instagram**: [Fix stories extraction](https://github.com/yt-dlp/yt-dlp/commit/50eaea9fd7787546b53660e736325fa31c77765d) ([#8843](https://github.com/yt-dlp/yt-dlp/issues/8843)) by [bashonly](https://github.com/bashonly)
+- **joqrag**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/db8b4edc7d0bd27da462f6fe82ff6e13e3d68a04) ([#8384](https://github.com/yt-dlp/yt-dlp/issues/8384)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **litv**: [Fix premium content extraction](https://github.com/yt-dlp/yt-dlp/commit/f45c4efcd928a173e1300a8f1ce4258e70c969b1) ([#8842](https://github.com/yt-dlp/yt-dlp/issues/8842)) by [bashonly](https://github.com/bashonly)
+- **maariv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/c5f01bf7d4b9426c87c3f8248de23934a56579e0) ([#8331](https://github.com/yt-dlp/yt-dlp/issues/8331)) by [amir16yp](https://github.com/amir16yp)
+- **mediastream**: [Fix authenticated format extraction](https://github.com/yt-dlp/yt-dlp/commit/b03c89309eb141be1a1eceeeb7475dd3b7529ad9) ([#8657](https://github.com/yt-dlp/yt-dlp/issues/8657)) by [NickCis](https://github.com/NickCis)
+- **nebula**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/45d82be65f71bb05506bd55376c6fdb36bc54142) ([#8566](https://github.com/yt-dlp/yt-dlp/issues/8566)) by [elyse0](https://github.com/elyse0), [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev)
+- **nintendo**: [Fix Nintendo Direct extraction](https://github.com/yt-dlp/yt-dlp/commit/1d24da6c899ef280d8b0a48a5e280ecd5d39cdf4) ([#8609](https://github.com/yt-dlp/yt-dlp/issues/8609)) by [Grub4K](https://github.com/Grub4K)
+- **ondemandkorea**: [Fix upgraded format extraction](https://github.com/yt-dlp/yt-dlp/commit/04a5e06350e3ef7c03f94f2f3f90dd96c6411152) ([#8677](https://github.com/yt-dlp/yt-dlp/issues/8677)) by [seproDev](https://github.com/seproDev)
+- **pr0gramm**: [Support variant formats and subtitles](https://github.com/yt-dlp/yt-dlp/commit/f98a3305eb124a0c375d03209d5c5a64fe1766c8) ([#8674](https://github.com/yt-dlp/yt-dlp/issues/8674)) by [Grub4K](https://github.com/Grub4K)
+- **rinsefm**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/c91af948e43570025e4aa887e248fd025abae394) ([#8778](https://github.com/yt-dlp/yt-dlp/issues/8778)) by [hashFactory](https://github.com/hashFactory)
+- **rudovideo**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/0d531c35eca4c2eb36e160530a7a333edbc727cc) ([#8664](https://github.com/yt-dlp/yt-dlp/issues/8664)) by [nicodato](https://github.com/nicodato)
+- **theguardian**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/1fa3f24d4b5d22176b11d78420f1f4b64a5af0a8) ([#8535](https://github.com/yt-dlp/yt-dlp/issues/8535)) by [SirElderling](https://github.com/SirElderling)
+- **theplatform**: [Extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/7e09c147fdccb44806bbf601573adc4b77210a89) ([#8635](https://github.com/yt-dlp/yt-dlp/issues/8635)) by [trainman261](https://github.com/trainman261)
+- **twitcasting**: [Detect livestreams via API and `show` page](https://github.com/yt-dlp/yt-dlp/commit/585d0ed9abcfcb957f2b2684b8ad43c3af160383) ([#8601](https://github.com/yt-dlp/yt-dlp/issues/8601)) by [bashonly](https://github.com/bashonly), [JC-Chung](https://github.com/JC-Chung)
+- **twitcastinguser**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/ff2fde1b8f922fd34bae6172602008cd67c07c93) ([#8650](https://github.com/yt-dlp/yt-dlp/issues/8650)) by [bashonly](https://github.com/bashonly)
+- **twitter**
+    - [Extract stale tweets](https://github.com/yt-dlp/yt-dlp/commit/1c54a98e19d047e7c15184237b6ef8ad50af489c) ([#8724](https://github.com/yt-dlp/yt-dlp/issues/8724)) by [bashonly](https://github.com/bashonly)
+    - [Prioritize m3u8 formats](https://github.com/yt-dlp/yt-dlp/commit/e7d22348e77367740da78a3db27167ecf894b7c9) ([#8826](https://github.com/yt-dlp/yt-dlp/issues/8826)) by [bashonly](https://github.com/bashonly)
+    - [Work around API rate-limit](https://github.com/yt-dlp/yt-dlp/commit/116c268438ea4d3738f6fa502c169081ca8f0ee7) ([#8825](https://github.com/yt-dlp/yt-dlp/issues/8825)) by [bashonly](https://github.com/bashonly)
+    - broadcast: [Extract `concurrent_view_count`](https://github.com/yt-dlp/yt-dlp/commit/6fe82491ed622b948c512cf4aab46ac3a234ae0a) ([#8600](https://github.com/yt-dlp/yt-dlp/issues/8600)) by [sonmezberkay](https://github.com/sonmezberkay)
+- **vidly**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/34df1c1f60fa652c0a6a5c712b06c10e45daf6b7) ([#8612](https://github.com/yt-dlp/yt-dlp/issues/8612)) by [seproDev](https://github.com/seproDev)
+- **vocaroo**: [Do not use deprecated `getheader`](https://github.com/yt-dlp/yt-dlp/commit/f223b1b0789f65e06619dcc9fc9e74f50d259379) ([#8606](https://github.com/yt-dlp/yt-dlp/issues/8606)) by [qbnu](https://github.com/qbnu)
+- **vvvvid**: [Set user-agent to fix extraction](https://github.com/yt-dlp/yt-dlp/commit/1725e943b0e8a8b585305660d4611e684374409c) ([#8615](https://github.com/yt-dlp/yt-dlp/issues/8615)) by [Kyraminol](https://github.com/Kyraminol)
+- **youtube**
+    - [Fix `like_count` extraction](https://github.com/yt-dlp/yt-dlp/commit/6b5d93b0b0240e287389d1d43b2d5293e18aa4cc) ([#8763](https://github.com/yt-dlp/yt-dlp/issues/8763)) by [Ganesh910](https://github.com/Ganesh910)
+    - [Improve detection of faulty HLS formats](https://github.com/yt-dlp/yt-dlp/commit/bb5a54e6db2422bbd155d93a0e105b6616c09467) ([#8646](https://github.com/yt-dlp/yt-dlp/issues/8646)) by [bashonly](https://github.com/bashonly)
+    - [Return empty playlist when channel/tab has no videos](https://github.com/yt-dlp/yt-dlp/commit/044886c220620a7679109e92352890e18b6079e3) by [pukkandan](https://github.com/pukkandan)
+    - [Support cf.piped.video](https://github.com/yt-dlp/yt-dlp/commit/6a9c7a2b52655bacfa7ab2da24fd0d14a6fff495) ([#8514](https://github.com/yt-dlp/yt-dlp/issues/8514)) by [OIRNOIR](https://github.com/OIRNOIR)
+- **zingmp3**: [Add support for radio and podcasts](https://github.com/yt-dlp/yt-dlp/commit/64de1a4c25bada90374b88d7353754fe8fbfcc51) ([#7189](https://github.com/yt-dlp/yt-dlp/issues/7189)) by [hatienl0i261299](https://github.com/hatienl0i261299)
+
+#### Postprocessor changes
+- **ffmpegmetadata**: [Embed stream metadata in single format downloads](https://github.com/yt-dlp/yt-dlp/commit/deeb13eae82e60f82a2c0c5861f460399a997528) ([#8647](https://github.com/yt-dlp/yt-dlp/issues/8647)) by [bashonly](https://github.com/bashonly)
+
+#### Networking changes
+- [Strip whitespace around header values](https://github.com/yt-dlp/yt-dlp/commit/196eb0fe77b78e2e5ca02c506c3837c2b1a7964c) ([#8802](https://github.com/yt-dlp/yt-dlp/issues/8802)) by [coletdjnz](https://github.com/coletdjnz)
+- **Request Handler**: websockets: [Migrate websockets to networking framework](https://github.com/yt-dlp/yt-dlp/commit/ccfd70f4c24b579c72123ca76ab50164f8f122b7) ([#7720](https://github.com/yt-dlp/yt-dlp/issues/7720)) by [coletdjnz](https://github.com/coletdjnz)
+
+#### Misc. changes
+- **ci**
+    - [Concurrency optimizations](https://github.com/yt-dlp/yt-dlp/commit/f124fa458826308afc86cf364c509f857686ecfd) ([#8614](https://github.com/yt-dlp/yt-dlp/issues/8614)) by [Grub4K](https://github.com/Grub4K)
+    - [Run core tests only for core changes](https://github.com/yt-dlp/yt-dlp/commit/13b3cb3c2b7169a1e17d6fc62593bf744170521c) ([#8841](https://github.com/yt-dlp/yt-dlp/issues/8841)) by [Grub4K](https://github.com/Grub4K)
+- **cleanup**
+    - [Fix spelling of `IE_NAME`](https://github.com/yt-dlp/yt-dlp/commit/bc4ab17b38f01000d99c5c2bedec89721fee65ec) ([#8810](https://github.com/yt-dlp/yt-dlp/issues/8810)) by [barsnick](https://github.com/barsnick)
+    - [Remove dead extractors](https://github.com/yt-dlp/yt-dlp/commit/9751a457cfdb18bf99d9ee0d10e4e6a594502bbf) ([#8604](https://github.com/yt-dlp/yt-dlp/issues/8604)) by [seproDev](https://github.com/seproDev)
+    - Miscellaneous: [f9fb3ce](https://github.com/yt-dlp/yt-dlp/commit/f9fb3ce86e3c6a0c3c33b45392b8d7288bceba76) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev)
+- **devscripts**: `run_tests`: [Create Python script](https://github.com/yt-dlp/yt-dlp/commit/2d1d683a541d71f3d3bb999dfe8eeb1976fb91ce) ([#8720](https://github.com/yt-dlp/yt-dlp/issues/8720)) by [Grub4K](https://github.com/Grub4K) (With fixes in [225cf2b](https://github.com/yt-dlp/yt-dlp/commit/225cf2b830a1de2c5eacd257edd2a01aed1e1114))
+- **docs**: [Update youtube-dl merge commit in `README.md`](https://github.com/yt-dlp/yt-dlp/commit/f10589e3453009bb523f55849bba144c9b91cf2a) by [bashonly](https://github.com/bashonly)
+- **test**: networking: [Update tests for OpenSSL 3.2](https://github.com/yt-dlp/yt-dlp/commit/37755a037e612bfc608c3d4722e8ef2ce6a022ee) ([#8814](https://github.com/yt-dlp/yt-dlp/issues/8814)) by [bashonly](https://github.com/bashonly)
+
 ### 2023.11.16
 
 #### Extractor changes
diff --git a/supportedsites.md b/supportedsites.md
index 0e971c135..96681c16b 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -1,6 +1,4 @@
 # Supported sites
- - **0000studio:archive**
- - **0000studio:clip**
  - **17live**
  - **17live:clip**
  - **1News**: 1news.co.nz article videos
@@ -9,7 +7,6 @@ # Supported sites
  - **23video**
  - **247sports**
  - **24tv.ua**
- - **24video**
  - **3qsdn**: 3Q SDN
  - **3sat**
  - **4tube**
@@ -50,15 +47,18 @@ # Supported sites
  - **afreecatv**: [*afreecatv*](## "netrc machine") afreecatv.com
  - **afreecatv:live**: [*afreecatv*](## "netrc machine") afreecatv.com
  - **afreecatv:user**
- - **AirMozilla**
  - **AirTV**
  - **AitubeKZVideo**
  - **AliExpressLive**
  - **AlJazeera**
  - **Allocine**
+ - **Allstar**
+ - **AllstarProfile**
  - **AlphaPorno**
  - **Alsace20TV**
  - **Alsace20TVEmbed**
+ - **altcensored**
+ - **altcensored:channel**
  - **Alura**: [*alura*](## "netrc machine")
  - **AluraCourse**: [*aluracourse*](## "netrc machine")
  - **Amara**
@@ -79,7 +79,7 @@ # Supported sites
  - **ant1newsgr:embed**: ant1news.gr embedded videos
  - **antenna:watch**: antenna.gr and ant1news.gr videos
  - **Anvato**
- - **aol.com**: Yahoo screen and movies
+ - **aol.com**: Yahoo screen and movies (**Currently broken**)
  - **APA**
  - **Aparat**
  - **AppleConnect**
@@ -90,8 +90,8 @@ # Supported sites
  - **archive.org**: archive.org video and audio
  - **ArcPublishing**
  - **ARD**
- - **ARD:mediathek**
- - **ARDBetaMediathek**
+ - **ARDMediathek**
+ - **ARDMediathekCollection**
  - **Arkena**
  - **arte.sky.it**
  - **ArteTV**
@@ -100,7 +100,6 @@ # Supported sites
  - **ArteTVPlaylist**
  - **AtresPlayer**: [*atresplayer*](## "netrc machine")
  - **AtScaleConfEvent**
- - **ATTTechChannel**
  - **ATVAt**
  - **AudiMedia**
  - **AudioBoom**
@@ -140,12 +139,12 @@ # Supported sites
  - **BeatBumpVideo**
  - **Beatport**
  - **Beeg**
- - **BehindKink**
+ - **BehindKink**: (**Currently broken**)
  - **Bellator**
  - **BellMedia**
  - **BerufeTV**
- - **Bet**
- - **bfi:player**
+ - **Bet**: (**Currently broken**)
+ - **bfi:player**: (**Currently broken**)
  - **bfmtv**
  - **bfmtv:article**
  - **bfmtv:live**
@@ -162,6 +161,8 @@ # Supported sites
  - **BiliBiliBangumi**
  - **BiliBiliBangumiMedia**
  - **BiliBiliBangumiSeason**
+ - **BilibiliCheese**
+ - **BilibiliCheeseSeason**
  - **BilibiliCollectionList**
  - **BilibiliFavoritesList**
  - **BiliBiliPlayer**
@@ -176,11 +177,8 @@ # Supported sites
  - **BiliLive**
  - **BioBioChileTV**
  - **Biography**
- - **BIQLE**
  - **BitChute**
  - **BitChuteChannel**
- - **bitwave:replay**
- - **bitwave:stream**
  - **BlackboardCollaborate**
  - **BleacherReport**
  - **BleacherReportCMS**
@@ -193,7 +191,7 @@ # Supported sites
  - **Box**
  - **BoxCastVideo**
  - **Bpb**: Bundeszentrale für politische Bildung
- - **BR**: Bayerischer Rundfunk
+ - **BR**: Bayerischer Rundfunk (**Currently broken**)
  - **BrainPOP**: [*brainpop*](## "netrc machine")
  - **BrainPOPELL**: [*brainpop*](## "netrc machine")
  - **BrainPOPEsp**: [*brainpop*](## "netrc machine") BrainPOP Español
@@ -201,19 +199,18 @@ # Supported sites
  - **BrainPOPIl**: [*brainpop*](## "netrc machine") BrainPOP Hebrew
  - **BrainPOPJr**: [*brainpop*](## "netrc machine")
  - **BravoTV**
- - **Break**
  - **BreitBart**
  - **brightcove:legacy**
  - **brightcove:new**
  - **Brilliantpala:Classes**: [*brilliantpala*](## "netrc machine") VoD on classes.brilliantpala.org
  - **Brilliantpala:Elearn**: [*brilliantpala*](## "netrc machine") VoD on elearn.brilliantpala.org
- - **BRMediathek**: Bayerischer Rundfunk Mediathek
  - **bt:article**: Bergens Tidende Articles
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **Bundesliga**
+ - **Bundestag**
  - **BusinessInsider**
  - **BuzzFeed**
- - **BYUtv**
+ - **BYUtv**: (**Currently broken**)
  - **CableAV**
  - **Callin**
  - **Caltrans**
@@ -225,14 +222,11 @@ # Supported sites
  - **CamModels**
  - **Camsoda**
  - **CamtasiaEmbed**
- - **CamWithHer**
  - **Canal1**
  - **CanalAlpha**
  - **canalc2.tv**
  - **Canalplus**: mycanal.fr and piwiplus.fr
  - **CaracolTvPlay**: [*caracoltv-play*](## "netrc machine")
- - **CarambaTV**
- - **CarambaTVPage**
  - **CartoonNetwork**
  - **cbc.ca**
  - **cbc.ca:player**
@@ -254,16 +248,12 @@ # Supported sites
  - **Cellebrite**
  - **CeskaTelevize**
  - **CGTN**
- - **channel9**: Channel 9
  - **CharlieRose**
  - **Chaturbate**
  - **Chilloutzone**
  - **Chingari**
  - **ChingariUser**
- - **chirbit**
- - **chirbit:profile**
  - **cielotv.it**
- - **Cinchcast**
  - **Cinemax**
  - **CinetecaMilano**
  - **Cineverse**
@@ -276,14 +266,12 @@ # Supported sites
  - **cliphunter**
  - **Clippit**
  - **ClipRs**
- - **Clipsyndicate**
  - **ClipYouEmbed**
  - **CloserToTruth**
  - **CloudflareStream**
- - **Cloudy**
- - **Clubic**
+ - **Clubic**: (**Currently broken**)
  - **Clyp**
- - **cmt.com**
+ - **cmt.com**: (**Currently broken**)
  - **CNBC**
  - **CNBCVideo**
  - **CNN**
@@ -328,7 +316,6 @@ # Supported sites
  - **CybraryCourse**: [*cybrary*](## "netrc machine")
  - **DacastPlaylist**
  - **DacastVOD**
- - **Daftsex**
  - **DagelijkseKost**: dagelijksekost.een.be
  - **DailyMail**
  - **dailymotion**: [*dailymotion*](## "netrc machine")
@@ -347,13 +334,12 @@ # Supported sites
  - **DctpTv**
  - **DeezerAlbum**
  - **DeezerPlaylist**
- - **defense.gouv.fr**
  - **democracynow**
  - **DestinationAmerica**
  - **DetikEmbed**
  - **DeuxM**
  - **DeuxMNews**
- - **DHM**: Filmarchiv - Deutsches Historisches Museum
+ - **DHM**: Filmarchiv - Deutsches Historisches Museum (**Currently broken**)
  - **Digg**
  - **DigitalConcertHall**: [*digitalconcerthall*](## "netrc machine") DigitalConcertHall extractor
  - **DigitallySpeaking**
@@ -373,7 +359,6 @@ # Supported sites
  - **dlf:corpus**: DLF Multi-feed Archives
  - **dlive:stream**
  - **dlive:vod**
- - **Dotsub**
  - **Douyin**
  - **DouyuShow**
  - **DouyuTV**: 斗鱼直播
@@ -392,35 +377,29 @@ # Supported sites
  - **duboku**: www.duboku.io
  - **duboku:list**: www.duboku.io entire series
  - **Dumpert**
+ - **Duoplay**
  - **dvtv**: http://video.aktualne.cz/
  - **dw**
  - **dw:article**
  - **EaglePlatform**
  - **EbaumsWorld**
  - **Ebay**
- - **EchoMsk**
  - **egghead:course**: egghead.io course
  - **egghead:lesson**: egghead.io lesson
- - **ehftv**
- - **eHow**
  - **EinsUndEinsTV**: [*1und1tv*](## "netrc machine")
  - **EinsUndEinsTVLive**: [*1und1tv*](## "netrc machine")
  - **EinsUndEinsTVRecordings**: [*1und1tv*](## "netrc machine")
  - **Einthusan**
  - **eitb.tv**
- - **ElevenSports**
- - **EllenTube**
- - **EllenTubePlaylist**
- - **EllenTubeVideo**
  - **Elonet**
  - **ElPais**: El País
  - **ElTreceTV**: El Trece TV (Argentina)
  - **Embedly**
  - **EMPFlix**
- - **Engadget**
  - **Epicon**
  - **EpiconSeries**
- - **eplus:inbound**: e+ (イープラス) overseas
+ - **EpidemicSound**
+ - **eplus**: [*eplus*](## "netrc machine") e+ (イープラス)
  - **Epoch**
  - **Eporner**
  - **Erocast**
@@ -429,11 +408,9 @@ # Supported sites
  - **ertflix**: ERTFLIX videos
  - **ertflix:codename**: ERTFLIX videos by codename
  - **ertwebtv:embed**: ert.gr webtv embedded videos
- - **Escapist**
  - **ESPN**
  - **ESPNArticle**
  - **ESPNCricInfo**
- - **EsriVideo**
  - **EttuTv**
  - **Europa**
  - **EuroParlWebstream**
@@ -443,9 +420,7 @@ # Supported sites
  - **EWETV**: [*ewetv*](## "netrc machine")
  - **EWETVLive**: [*ewetv*](## "netrc machine")
  - **EWETVRecordings**: [*ewetv*](## "netrc machine")
- - **ExpoTV**
  - **Expressen**
- - **ExtremeTube**
  - **EyedoTV**
  - **facebook**: [*facebook*](## "netrc machine")
  - **facebook:reel**
@@ -465,6 +440,8 @@ # Supported sites
  - **FiveThirtyEight**
  - **FiveTV**
  - **Flickr**
+ - **Floatplane**
+ - **FloatplaneChannel**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FoodNetwork**
  - **FootyRoom**
@@ -472,7 +449,6 @@ # Supported sites
  - **FOX**
  - **FOX9**
  - **FOX9News**
- - **Foxgay**
  - **foxnews**: Fox News and Fox Business Video
  - **foxnews:article**
  - **FoxNewsVideo**
@@ -496,7 +472,6 @@ # Supported sites
  - **funimation:show**: [*funimation*](## "netrc machine")
  - **Funk**
  - **Funker530**
- - **Fusion**
  - **Fux**
  - **FuyinTV**
  - **Gab**
@@ -522,7 +497,6 @@ # Supported sites
  - **GeniusLyrics**
  - **Gettr**
  - **GettrStreaming**
- - **Gfycat**
  - **GiantBomb**
  - **Giga**
  - **GlattvisionTV**: [*glattvisiontv*](## "netrc machine")
@@ -564,7 +538,6 @@ # Supported sites
  - **HearThisAt**
  - **Heise**
  - **HellPorno**
- - **Helsinki**: helsinki.fi
  - **hetklokhuis**
  - **hgtv.com:show**
  - **HGTVDe**
@@ -573,8 +546,6 @@ # Supported sites
  - **HistoricFilms**
  - **history:player**
  - **history:topic**: History.com Topic
- - **hitbox**
- - **hitbox:live**
  - **HitRecord**
  - **hketv**: 香港教育局教育電視 (HKETV) Educational Television, Hong Kong Educational Bureau
  - **HollywoodReporter**
@@ -585,8 +556,6 @@ # Supported sites
  - **hotstar:playlist**
  - **hotstar:season**
  - **hotstar:series**
- - **Howcast**
- - **HowStuffWorks**
  - **hrfernsehen**
  - **HRTi**: [*hrti*](## "netrc machine")
  - **HRTiPlaylist**: [*hrti*](## "netrc machine")
@@ -608,7 +577,7 @@ # Supported sites
  - **ign.com**
  - **IGNArticle**
  - **IGNVideo**
- - **IHeartRadio**
+ - **iheartradio**
  - **iheartradio:podcast**
  - **Iltalehti**
  - **imdb**: Internet Movie Database trailers
@@ -638,7 +607,6 @@ # Supported sites
  - **IsraelNationalNews**
  - **ITProTV**
  - **ITProTVCourse**
- - **ITTF**
  - **ITV**
  - **ITVBTCC**
  - **ivi**: ivi.ru
@@ -658,6 +626,7 @@ # Supported sites
  - **JioSaavnAlbum**
  - **JioSaavnSong**
  - **Joj**
+ - **JoqrAg**: 超!A&G+ 文化放送 (f.k.a. AGQR) Nippon Cultural Broadcasting, Inc. (JOQR)
  - **Jove**
  - **JStream**
  - **JTBC**: jtbc.co.kr
@@ -670,7 +639,6 @@ # Supported sites
  - **Karaoketv**
  - **KarriereVideos**
  - **Katsomo**
- - **KeezMovies**
  - **KelbyOne**
  - **Ketnet**
  - **khanacademy**
@@ -679,7 +647,7 @@ # Supported sites
  - **Kicker**
  - **KickStarter**
  - **KickVOD**
- - **KinjaEmbed**
+ - **kinja:embed**
  - **KinoPoisk**
  - **Kommunetv**
  - **KompasVideo**
@@ -698,8 +666,6 @@ # Supported sites
  - **la7.it**
  - **la7.it:​pod:episode**
  - **la7.it:podcast**
- - **laola1tv**
- - **laola1tv:embed**
  - **LastFM**
  - **LastFMPlaylist**
  - **LastFMUser**
@@ -733,7 +699,6 @@ # Supported sites
  - **LinkedIn**: [*linkedin*](## "netrc machine")
  - **linkedin:learning**: [*linkedin*](## "netrc machine")
  - **linkedin:​learning:course**: [*linkedin*](## "netrc machine")
- - **LinuxAcademy**: [*linuxacademy*](## "netrc machine")
  - **Liputan6**
  - **ListenNotes**
  - **LiTV**
@@ -751,7 +716,7 @@ # Supported sites
  - **Lumni**
  - **lynda**: [*lynda*](## "netrc machine") lynda.com videos
  - **lynda:course**: [*lynda*](## "netrc machine") lynda.com online courses
- - **m6**
+ - **maariv.co.il**
  - **MagellanTV**
  - **MagentaMusik360**
  - **mailru**: Видео@Mail.Ru
@@ -793,11 +758,8 @@ # Supported sites
  - **megatvcom:embed**: megatv.com embedded videos
  - **Meipai**: 美拍
  - **MelonVOD**
- - **META**
- - **metacafe**
  - **Metacritic**
  - **mewatch**
- - **Mgoon**
  - **MiaoPai**
  - **MicrosoftEmbed**
  - **microsoftstream**: Microsoft Stream
@@ -810,7 +772,6 @@ # Supported sites
  - **minds:group**
  - **MinistryGrid**
  - **Minoto**
- - **miomio.tv**
  - **mirrativ**
  - **mirrativ:user**
  - **MirrorCoUK**
@@ -825,14 +786,10 @@ # Supported sites
  - **MLBTV**: [*mlb*](## "netrc machine")
  - **MLBVideo**
  - **MLSSoccer**
- - **Mnet**
  - **MNetTV**: [*mnettv*](## "netrc machine")
  - **MNetTVLive**: [*mnettv*](## "netrc machine")
  - **MNetTVRecordings**: [*mnettv*](## "netrc machine")
  - **MochaVideo**
- - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
- - **Mofosex**
- - **MofosexEmbed**
  - **Mojvideo**
  - **Monstercat**
  - **MonsterSirenHypergryphMusic**
@@ -843,13 +800,12 @@ # Supported sites
  - **Motorsport**: motorsport.com
  - **MotorTrend**
  - **MotorTrendOnDemand**
- - **MovieClips**
  - **MovieFap**
  - **Moviepilot**
  - **MoviewPlay**
  - **Moviezine**
  - **MovingImage**
- - **MSN**
+ - **MSN**: (**Currently broken**)
  - **mtg**: MTG services
  - **mtv**
  - **mtv.de**
@@ -871,18 +827,13 @@ # Supported sites
  - **MusicdexSong**
  - **mva**: Microsoft Virtual Academy videos
  - **mva:course**: Microsoft Virtual Academy courses
- - **Mwave**
- - **MwaveMeetGreet**
  - **Mxplayer**
  - **MxplayerShow**
- - **MyChannels**
  - **MySpace**
  - **MySpace:album**
  - **MySpass**
- - **Myvi**
  - **MyVideoGe**
  - **MyVidster**
- - **MyviEmbed**
  - **Mzaalo**
  - **n-tv.de**
  - **N1Info:article**
@@ -894,12 +845,12 @@ # Supported sites
  - **Naver**
  - **Naver:live**
  - **navernow**
- - **NBA**
+ - **nba**
+ - **nba:channel**
+ - **nba:embed**
  - **nba:watch**
  - **nba:​watch:collection**
- - **NBAChannel**
- - **NBAEmbed**
- - **NBAWatchEmbed**
+ - **nba:​watch:embed**
  - **NBC**
  - **NBCNews**
  - **nbcolympics**
@@ -914,6 +865,7 @@ # Supported sites
  - **NDTV**
  - **Nebula**: [*watchnebula*](## "netrc machine")
  - **nebula:channel**: [*watchnebula*](## "netrc machine")
+ - **nebula:class**: [*watchnebula*](## "netrc machine")
  - **nebula:subscriptions**: [*watchnebula*](## "netrc machine")
  - **NekoHacker**
  - **NerdCubedFeed**
@@ -935,7 +887,6 @@ # Supported sites
  - **Newgrounds:playlist**
  - **Newgrounds:user**
  - **NewsPicks**
- - **Newstube**
  - **Newsy**
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
@@ -961,7 +912,6 @@ # Supported sites
  - **nick.de**
  - **nickelodeon:br**
  - **nickelodeonru**
- - **nicknight**
  - **niconico**: [*niconico*](## "netrc machine") ニコニコ動画
  - **niconico:history**: NicoNico user history or likes. Requires cookies.
  - **niconico:live**: ニコニコ生放送
@@ -984,9 +934,7 @@ # Supported sites
  - **NonkTube**
  - **NoodleMagazine**
  - **Noovo**
- - **Normalboots**
  - **NOSNLArticle**
- - **NosVideo**
  - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
  - **NovaEmbed**
  - **NovaPlay**
@@ -1009,7 +957,7 @@ # Supported sites
  - **NRKTVEpisodes**
  - **NRKTVSeason**
  - **NRKTVSeries**
- - **NRLTV**
+ - **NRLTV**: (**Currently broken**)
  - **ntv.ru**
  - **NubilesPorn**: [*nubiles-porn*](## "netrc machine")
  - **Nuvid**
@@ -1037,8 +985,6 @@ # Supported sites
  - **onet.tv:channel**
  - **OnetMVP**
  - **OnionStudios**
- - **Ooyala**
- - **OoyalaExternal**
  - **Opencast**
  - **OpencastPlaylist**
  - **openrec**
@@ -1060,7 +1006,6 @@ # Supported sites
  - **PalcoMP3:artist**
  - **PalcoMP3:song**
  - **PalcoMP3:video**
- - **pandora.tv**: 판도라TV
  - **Panopto**
  - **PanoptoList**
  - **PanoptoPlaylist**
@@ -1082,7 +1027,6 @@ # Supported sites
  - **PeerTube:Playlist**
  - **peloton**: [*peloton*](## "netrc machine")
  - **peloton:live**: Peloton Live
- - **People**
  - **PerformGroup**
  - **periscope**: Periscope
  - **periscope:user**: Periscope user videos
@@ -1104,14 +1048,11 @@ # Supported sites
  - **PlanetMarathi**
  - **Platzi**: [*platzi*](## "netrc machine")
  - **PlatziCourse**: [*platzi*](## "netrc machine")
- - **play.fm**
  - **player.sky.it**
  - **PlayPlusTV**: [*playplustv*](## "netrc machine")
  - **PlayStuff**
- - **PlaysTV**
  - **PlaySuisse**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
- - **Playvid**
  - **PlayVids**
  - **Playwire**
  - **pluralsight**: [*pluralsight*](## "netrc machine")
@@ -1136,11 +1077,8 @@ # Supported sites
  - **Popcorntimes**
  - **PopcornTV**
  - **Pornbox**
- - **PornCom**
  - **PornerBros**
- - **Pornez**
  - **PornFlip**
- - **PornHd**
  - **PornHub**: [*pornhub*](## "netrc machine") PornHub and Thumbzilla
  - **PornHubPagedVideoList**: [*pornhub*](## "netrc machine")
  - **PornHubPlaylist**: [*pornhub*](## "netrc machine")
@@ -1182,7 +1120,6 @@ # Supported sites
  - **Radiko**
  - **RadikoRadio**
  - **radio.de**
- - **radiobremen**
  - **radiocanada**
  - **radiocanada:audiovideo**
  - **RadioComercial**
@@ -1222,7 +1159,6 @@ # Supported sites
  - **RCTIPlusSeries**
  - **RCTIPlusTV**
  - **RDS**: RDS.ca
- - **Recurbate**
  - **RedBull**
  - **RedBullEmbed**
  - **RedBullTV**
@@ -1239,7 +1175,7 @@ # Supported sites
  - **Reuters**
  - **ReverbNation**
  - **RheinMainTV**
- - **RICE**
+ - **RinseFM**
  - **RMCDecouverte**
  - **RockstarGames**
  - **Rokfin**: [*rokfin*](## "netrc machine")
@@ -1260,8 +1196,6 @@ # Supported sites
  - **rtl.lu:tele-vod**
  - **rtl.nl**: rtl.nl and rtlxl.nl
  - **rtl2**
- - **rtl2:you**
- - **rtl2:​you:series**
  - **RTLLuLive**
  - **RTLLuRadio**
  - **RTNews**
@@ -1276,10 +1210,9 @@ # Supported sites
  - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
  - **rtve.es:television**
- - **RTVNH**
  - **RTVS**
  - **rtvslo.si**
- - **RUHD**
+ - **RudoVideo**
  - **Rule34Video**
  - **Rumble**
  - **RumbleChannel**
@@ -1326,8 +1259,8 @@ # Supported sites
  - **ScrippsNetworks**
  - **scrippsnetworks:watch**
  - **Scrolller**
- - **SCTE**: [*scte*](## "netrc machine")
- - **SCTECourse**: [*scte*](## "netrc machine")
+ - **SCTE**: [*scte*](## "netrc machine") (**Currently broken**)
+ - **SCTECourse**: [*scte*](## "netrc machine") (**Currently broken**)
  - **Seeker**
  - **SenalColombiaLive**
  - **SenateGov**
@@ -1339,7 +1272,6 @@ # Supported sites
  - **SeznamZpravyArticle**
  - **Shahid**: [*shahid*](## "netrc machine")
  - **ShahidShow**
- - **Shared**: shared.sx
  - **ShareVideosEmbed**
  - **ShemarooMe**
  - **ShowRoomLive**
@@ -1391,7 +1323,6 @@ # Supported sites
  - **SovietsClosetPlaylist**
  - **SpankBang**
  - **SpankBangPlaylist**
- - **Spankwire**
  - **Spiegel**
  - **Sport5**
  - **SportBox**
@@ -1404,7 +1335,7 @@ # Supported sites
  - **SpreakerShowPage**
  - **SpringboardPlatform**
  - **Sprout**
- - **sr:mediathek**: Saarländischer Rundfunk
+ - **sr:mediathek**: Saarländischer Rundfunk (**Currently broken**)
  - **SRGSSR**
  - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
  - **StacommuLive**: [*stacommu*](## "netrc machine")
@@ -1421,7 +1352,6 @@ # Supported sites
  - **StoryFireSeries**
  - **StoryFireUser**
  - **Streamable**
- - **streamcloud.eu**
  - **StreamCZ**
  - **StreamFF**
  - **StreetVoice**
@@ -1437,7 +1367,6 @@ # Supported sites
  - **SVTPlay**: SVT Play and Öppet arkiv
  - **SVTSeries**
  - **SwearnetEpisode**
- - **SWRMediathek**
  - **Syfy**
  - **SYVDK**
  - **SztvHu**
@@ -1456,7 +1385,6 @@ # Supported sites
  - **TeachingChannel**
  - **Teamcoco**
  - **TeamTreeHouse**: [*teamtreehouse*](## "netrc machine")
- - **TechTalks**
  - **techtv.mit.edu**
  - **TedEmbed**
  - **TedPlaylist**
@@ -1486,6 +1414,8 @@ # Supported sites
  - **TFO**
  - **theatercomplextown:ppv**: [*theatercomplextown*](## "netrc machine")
  - **theatercomplextown:vod**: [*theatercomplextown*](## "netrc machine")
+ - **TheGuardianPodcast**
+ - **TheGuardianPodcastPlaylist**
  - **TheHoleTv**
  - **TheIntercept**
  - **ThePlatform**
@@ -1506,27 +1436,23 @@ # Supported sites
  - **tiktok:sound**: (**Currently broken**)
  - **tiktok:tag**: (**Currently broken**)
  - **tiktok:user**: (**Currently broken**)
- - **tinypic**: tinypic.com videos
  - **TLC**
  - **TMZ**
  - **TNAFlix**
  - **TNAFlixNetworkEmbed**
  - **toggle**
  - **toggo**
- - **Tokentube**
- - **Tokentube:channel**
  - **tokfm:audition**
  - **tokfm:podcast**
  - **ToonGoggles**
  - **tou.tv**: [*toutv*](## "netrc machine")
- - **Toypics**: Toypics video
- - **ToypicsUser**: Toypics user profile
+ - **Toypics**: Toypics video (**Currently broken**)
+ - **ToypicsUser**: Toypics user profile (**Currently broken**)
  - **TrailerAddict**: (**Currently broken**)
  - **TravelChannel**
  - **Triller**: [*triller*](## "netrc machine")
  - **TrillerShort**
  - **TrillerUser**: [*triller*](## "netrc machine")
- - **Trilulilu**
  - **Trovo**
  - **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix
  - **TrovoChannelVod**: All VODs of a trovo.live channel; "trovovod:" prefix
@@ -1536,7 +1462,7 @@ # Supported sites
  - **TruNews**
  - **Truth**
  - **TruTV**
- - **Tube8**
+ - **Tube8**: (**Currently broken**)
  - **TubeTuGraz**: [*tubetugraz*](## "netrc machine") tube.tugraz.at
  - **TubeTuGrazSeries**: [*tubetugraz*](## "netrc machine")
  - **TubiTv**: [*tubitv*](## "netrc machine")
@@ -1545,7 +1471,6 @@ # Supported sites
  - **TuneInPodcast**
  - **TuneInPodcastEpisode**
  - **TuneInStation**
- - **TunePk**
  - **Turbo**
  - **tv.dfb.de**
  - **TV2**
@@ -1569,14 +1494,7 @@ # Supported sites
  - **TVIPlayer**
  - **tvland.com**
  - **TVN24**
- - **TVNet**
  - **TVNoe**
- - **TVNow**
- - **TVNowAnnual**
- - **TVNowFilm**
- - **TVNowNew**
- - **TVNowSeason**
- - **TVNowShow**
  - **tvopengr:embed**: tvopen.gr embedded videos
  - **tvopengr:watch**: tvopen.gr (and ethnos.gr) videos
  - **tvp**: Telewizja Polska
@@ -1614,7 +1532,6 @@ # Supported sites
  - **umg:de**: Universal Music Deutschland
  - **Unistra**
  - **Unity**
- - **UnscriptedNewsVideo**
  - **uol.com.br**
  - **uplynk**
  - **uplynk:preplay**
@@ -1629,7 +1546,6 @@ # Supported sites
  - **Utreon**
  - **Varzesh3**
  - **Vbox7**
- - **VeeHD**
  - **Veo**
  - **Veoh**
  - **veoh:user**
@@ -1642,7 +1558,6 @@ # Supported sites
  - **vice**
  - **vice:article**
  - **vice:show**
- - **Vidbit**
  - **Viddler**
  - **Videa**
  - **video.arnes.si**: Arnes Video
@@ -1664,6 +1579,7 @@ # Supported sites
  - **VidioLive**: [*vidio*](## "netrc machine")
  - **VidioPremier**: [*vidio*](## "netrc machine")
  - **VidLii**
+ - **Vidly**
  - **viewlift**
  - **viewlift:embed**
  - **Viidea**
@@ -1683,7 +1599,6 @@ # Supported sites
  - **Vimm:stream**
  - **ViMP**
  - **ViMP:Playlist**
- - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**
  - **Viqeo**
@@ -1691,7 +1606,6 @@ # Supported sites
  - **viu:ott**: [*viu*](## "netrc machine")
  - **viu:playlist**
  - **ViuOTTIndonesia**
- - **Vivo**: vivo.sx
  - **vk**: [*vk*](## "netrc machine") VK
  - **vk:uservideos**: [*vk*](## "netrc machine") VK - User's Videos
  - **vk:wallpost**: [*vk*](## "netrc machine")
@@ -1699,37 +1613,27 @@ # Supported sites
  - **VKPlayLive**
  - **vm.tiktok**
  - **Vocaroo**
- - **Vodlocker**
  - **VODPl**
  - **VODPlatform**
- - **VoiceRepublic**
  - **voicy**
  - **voicy:channel**
  - **VolejTV**
- - **Voot**: [*voot*](## "netrc machine")
- - **VootSeries**: [*voot*](## "netrc machine")
+ - **Voot**: [*voot*](## "netrc machine") (**Currently broken**)
+ - **VootSeries**: [*voot*](## "netrc machine") (**Currently broken**)
  - **VoxMedia**
  - **VoxMediaVolume**
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **vqq:series**
  - **vqq:video**
- - **Vrak**
  - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza
  - **VrtNU**: [*vrtnu*](## "netrc machine") VRT MAX
- - **vrv**: [*vrv*](## "netrc machine")
- - **vrv:series**
- - **VShare**
  - **VTM**
  - **VTXTV**: [*vtxtv*](## "netrc machine")
  - **VTXTVLive**: [*vtxtv*](## "netrc machine")
  - **VTXTVRecordings**: [*vtxtv*](## "netrc machine")
  - **VuClip**
- - **Vupload**
  - **VVVVID**
  - **VVVVIDShow**
- - **VyboryMos**
- - **Vzaar**
- - **Wakanim**
  - **Walla**
  - **WalyTV**: [*walytv*](## "netrc machine")
  - **WalyTVLive**: [*walytv*](## "netrc machine")
@@ -1740,9 +1644,7 @@ # Supported sites
  - **washingtonpost**
  - **washingtonpost:article**
  - **wat.tv**
- - **WatchBox**
  - **WatchESPN**
- - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
  - **wdr:mobile**: (**Currently broken**)
  - **WDRElefant**
@@ -1770,7 +1672,6 @@ # Supported sites
  - **whowatch**
  - **Whyp**
  - **wikimedia.org**
- - **Willow**
  - **Wimbledon**
  - **WimTV**
  - **WinSportsVideo**
@@ -1795,7 +1696,6 @@ # Supported sites
  - **wykop:post**
  - **wykop:​post:comment**
  - **Xanimu**
- - **XBef**
  - **XboxClips**
  - **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, WolfStream, XVideoSharing
  - **XHamster**
@@ -1807,9 +1707,6 @@ # Supported sites
  - **XMinus**
  - **XNXX**
  - **Xstream**
- - **XTube**
- - **XTubeUser**: XTube user profile
- - **Xuite**: 隨意窩Xuite影音
  - **XVideos**
  - **xvideos:quickies**
  - **XXXYMovies**
@@ -1826,10 +1723,7 @@ # Supported sites
  - **YapFiles**
  - **Yappy**
  - **YappyProfile**
- - **YesJapan**
- - **yinyuetai:video**: 音悦Tai
  - **YleAreena**
- - **Ynet**
  - **YouJizz**
  - **youku**: 优酷
  - **youku:show**
@@ -1877,6 +1771,9 @@ # Supported sites
  - **zingmp3:chart-home**
  - **zingmp3:chart-music-video**
  - **zingmp3:hub**
+ - **zingmp3:liveradio**
+ - **zingmp3:podcast**
+ - **zingmp3:podcast-episode**
  - **zingmp3:user**
  - **zingmp3:week-chart**
  - **zoom**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index fd923fe45..687ef8788 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.11.16'
+__version__ = '2023.12.30'
 
-RELEASE_GIT_HEAD = '24f827875c6ba513f12ed09a3aef2bbed223760d'
+RELEASE_GIT_HEAD = 'f10589e3453009bb523f55849bba144c9b91cf2a'
 
 VARIANT = None
 
@@ -12,4 +12,4 @@
 
 ORIGIN = 'yt-dlp/yt-dlp'
 
-_pkg_version = '2023.11.16'
+_pkg_version = '2023.12.30'

From 85a2d07c1f82c2082b568963d1c32ad3fc848f61 Mon Sep 17 00:00:00 2001
From: DmitryScaletta <DmitryScaletta@users.noreply.github.com>
Date: Sun, 31 Dec 2023 16:04:11 +0300
Subject: [PATCH 763/871] [ie/Bigo] Fix JSON extraction (#8893)

Closes #8852
Authored by: DmitryScaletta
---
 yt_dlp/extractor/bigo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/bigo.py b/yt_dlp/extractor/bigo.py
index 1cb6e58be..acf78e49a 100644
--- a/yt_dlp/extractor/bigo.py
+++ b/yt_dlp/extractor/bigo.py
@@ -29,7 +29,8 @@ def _real_extract(self, url):
 
         info_raw = self._download_json(
             'https://ta.bigo.tv/official_website/studio/getInternalStudioInfo',
-            user_id, data=urlencode_postdata({'siteId': user_id}))
+            user_id, data=urlencode_postdata({'siteId': user_id}),
+            headers={'Accept': 'application/json'})
 
         if not isinstance(info_raw, dict):
             raise ExtractorError('Received invalid JSON data')

From 85b33f5c163f60dbd089a6b9bc2ba1366d3ddf93 Mon Sep 17 00:00:00 2001
From: Ralph Drake <rama.o.ralph@gmail.com>
Date: Tue, 2 Jan 2024 00:58:36 +0000
Subject: [PATCH 764/871] [cookies] Fix `--cookies-from-browser` with macOS
 Firefox profiles (#8909)

Ref: https://support.mozilla.org/en-US/kb/profile-manager-create-remove-switch-firefox-profiles#firefox:mac

Closes #8898
Authored by: RalphORama
---
 yt_dlp/cookies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index a71fbc28b..eac033e39 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -186,7 +186,7 @@ def _firefox_browser_dir():
     if sys.platform in ('cygwin', 'win32'):
         return os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles')
     elif sys.platform == 'darwin':
-        return os.path.expanduser('~/Library/Application Support/Firefox')
+        return os.path.expanduser('~/Library/Application Support/Firefox/Profiles')
     return os.path.expanduser('~/.mozilla/firefox')
 
 

From 292d60b1ed3b9fe5bcb2775a894cca99b0f9473e Mon Sep 17 00:00:00 2001
From: mara004 <geisserml@gmail.com>
Date: Fri, 5 Jan 2024 18:13:46 +0100
Subject: [PATCH 765/871] [cleanup] Fix typo in README.md (#8894)

Authored by: antonkesy
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cfd861c56..16947ce30 100644
--- a/README.md
+++ b/README.md
@@ -280,7 +280,7 @@ ### Strongly recommended
 
 * [**ffmpeg** and **ffprobe**](https://www.ffmpeg.org) - Required for [merging separate video and audio files](#format-selection) as well as for various [post-processing](#post-processing-options) tasks. License [depends on the build](https://www.ffmpeg.org/legal.html)
 
-    There are bugs in ffmpeg that causes various issues when used alongside yt-dlp. Since ffmpeg is such an important dependency, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds#ffmpeg-static-auto-builds) with patches for some of these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specific issues solved by these builds
+    There are bugs in ffmpeg that cause various issues when used alongside yt-dlp. Since ffmpeg is such an important dependency, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds#ffmpeg-static-auto-builds) with patches for some of these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specific issues solved by these builds
     
     **Important**: What you need is ffmpeg *binary*, **NOT** [the python package of the same name](https://pypi.org/project/ffmpeg)
 

From ffbd4f2a02fee387ea5e0a267ce32df5259111ac Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Fri, 5 Jan 2024 21:26:17 +0100
Subject: [PATCH 766/871] [utils] `traverse_obj`: Support
 `xml.etree.ElementTree.Element` (#8911)

Authored by: Grub4K
---
 test/test_utils.py        | 52 +++++++++++++++++++++++++++++++++++++++
 yt_dlp/utils/traversal.py | 35 +++++++++++++++++++++++---
 2 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index c3e387cd0..09c648cf8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2340,6 +2340,58 @@ def test_traverse_obj(self):
         self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 'group')), ['0123', '3'],
                          msg='function on a `re.Match` should give group name as well')
 
+        # Test xml.etree.ElementTree.Element as input obj
+        etree = xml.etree.ElementTree.fromstring('''<?xml version="1.0"?>
+        <data>
+            <country name="Liechtenstein">
+                <rank>1</rank>
+                <year>2008</year>
+                <gdppc>141100</gdppc>
+                <neighbor name="Austria" direction="E"/>
+                <neighbor name="Switzerland" direction="W"/>
+            </country>
+            <country name="Singapore">
+                <rank>4</rank>
+                <year>2011</year>
+                <gdppc>59900</gdppc>
+                <neighbor name="Malaysia" direction="N"/>
+            </country>
+            <country name="Panama">
+                <rank>68</rank>
+                <year>2011</year>
+                <gdppc>13600</gdppc>
+                <neighbor name="Costa Rica" direction="W"/>
+                <neighbor name="Colombia" direction="E"/>
+            </country>
+        </data>''')
+        self.assertEqual(traverse_obj(etree, ''), etree,
+                         msg='empty str key should return the element itself')
+        self.assertEqual(traverse_obj(etree, 'country'), list(etree),
+                         msg='str key should lead all children with that tag name')
+        self.assertEqual(traverse_obj(etree, ...), list(etree),
+                         msg='`...` as key should return all children')
+        self.assertEqual(traverse_obj(etree, lambda _, x: x[0].text == '4'), [etree[1]],
+                         msg='function as key should get element as value')
+        self.assertEqual(traverse_obj(etree, lambda i, _: i == 1), [etree[1]],
+                         msg='function as key should get index as key')
+        self.assertEqual(traverse_obj(etree, 0), etree[0],
+                         msg='int key should return the nth child')
+        self.assertEqual(traverse_obj(etree, './/neighbor/@name'),
+                         ['Austria', 'Switzerland', 'Malaysia', 'Costa Rica', 'Colombia'],
+                         msg='`@<attribute>` at end of path should give that attribute')
+        self.assertEqual(traverse_obj(etree, '//neighbor/@fail'), [None, None, None, None, None],
+                         msg='`@<nonexistant>` at end of path should give `None`')
+        self.assertEqual(traverse_obj(etree, ('//neighbor/@', 2)), {'name': 'Malaysia', 'direction': 'N'},
+                         msg='`@` should give the full attribute dict')
+        self.assertEqual(traverse_obj(etree, '//year/text()'), ['2008', '2011', '2011'],
+                         msg='`text()` at end of path should give the inner text')
+        self.assertEqual(traverse_obj(etree, '//*[@direction]/@direction'), ['E', 'W', 'N', 'W', 'E'],
+                         msg='full python xpath features should be supported')
+        self.assertEqual(traverse_obj(etree, (0, '@name')), 'Liechtenstein',
+                         msg='special transformations should act on current element')
+        self.assertEqual(traverse_obj(etree, ('country', 0, ..., 'text()', {int_or_none})), [1, 2008, 141100],
+                         msg='special transformations should act on current element')
+
     def test_http_header_dict(self):
         headers = HTTPHeaderDict()
         headers['ytdl-test'] = b'0'
diff --git a/yt_dlp/utils/traversal.py b/yt_dlp/utils/traversal.py
index 5a2f69fcc..8938f4c78 100644
--- a/yt_dlp/utils/traversal.py
+++ b/yt_dlp/utils/traversal.py
@@ -3,6 +3,7 @@
 import inspect
 import itertools
 import re
+import xml.etree.ElementTree
 
 from ._utils import (
     IDENTITY,
@@ -118,7 +119,7 @@ def apply_key(key, obj, is_last):
             branching = True
             if isinstance(obj, collections.abc.Mapping):
                 result = obj.values()
-            elif is_iterable_like(obj):
+            elif is_iterable_like(obj) or isinstance(obj, xml.etree.ElementTree.Element):
                 result = obj
             elif isinstance(obj, re.Match):
                 result = obj.groups()
@@ -132,7 +133,7 @@ def apply_key(key, obj, is_last):
             branching = True
             if isinstance(obj, collections.abc.Mapping):
                 iter_obj = obj.items()
-            elif is_iterable_like(obj):
+            elif is_iterable_like(obj) or isinstance(obj, xml.etree.ElementTree.Element):
                 iter_obj = enumerate(obj)
             elif isinstance(obj, re.Match):
                 iter_obj = itertools.chain(
@@ -168,7 +169,7 @@ def apply_key(key, obj, is_last):
                 result = next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
 
         elif isinstance(key, (int, slice)):
-            if is_iterable_like(obj, collections.abc.Sequence):
+            if is_iterable_like(obj, (collections.abc.Sequence, xml.etree.ElementTree.Element)):
                 branching = isinstance(key, slice)
                 with contextlib.suppress(IndexError):
                     result = obj[key]
@@ -176,6 +177,34 @@ def apply_key(key, obj, is_last):
                 with contextlib.suppress(IndexError):
                     result = str(obj)[key]
 
+        elif isinstance(obj, xml.etree.ElementTree.Element) and isinstance(key, str):
+            xpath, _, special = key.rpartition('/')
+            if not special.startswith('@') and special != 'text()':
+                xpath = key
+                special = None
+
+            # Allow abbreviations of relative paths, absolute paths error
+            if xpath.startswith('/'):
+                xpath = f'.{xpath}'
+            elif xpath and not xpath.startswith('./'):
+                xpath = f'./{xpath}'
+
+            def apply_specials(element):
+                if special is None:
+                    return element
+                if special == '@':
+                    return element.attrib
+                if special.startswith('@'):
+                    return try_call(element.attrib.get, args=(special[1:],))
+                if special == 'text()':
+                    return element.text
+                assert False, f'apply_specials is missing case for {special!r}'
+
+            if xpath:
+                result = list(map(apply_specials, obj.iterfind(xpath)))
+            else:
+                result = apply_specials(obj)
+
         return branching, result if branching else (result,)
 
     def lazy_last(iterable):

From b6951271ac014761c9c317b9cecd5e8e139cfa7c Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Fri, 5 Jan 2024 21:34:38 +0100
Subject: [PATCH 767/871] [ie/ard:mediathek] Revert to using old id (#8916)

Authored by: Grub4K
---
 yt_dlp/extractor/ard.py | 40 +++++++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 15 deletions(-)

diff --git a/yt_dlp/extractor/ard.py b/yt_dlp/extractor/ard.py
index 91d297e8b..f4b1cd075 100644
--- a/yt_dlp/extractor/ard.py
+++ b/yt_dlp/extractor/ard.py
@@ -4,6 +4,7 @@
 from .common import InfoExtractor
 from ..utils import (
     OnDemandPagedList,
+    bug_reports_message,
     determine_ext,
     int_or_none,
     join_nonempty,
@@ -233,7 +234,7 @@ class ARDBetaMediathekIE(InfoExtractor):
         (?:(?:beta|www)\.)?ardmediathek\.de/
         (?:[^/]+/)?
         (?:player|live|video)/
-        (?:(?P<display_id>[^?#]+)/)?
+        (?:[^?#]+/)?
         (?P<id>[a-zA-Z0-9]+)
         /?(?:[?#]|$)'''
     _GEO_COUNTRIES = ['DE']
@@ -242,8 +243,8 @@ class ARDBetaMediathekIE(InfoExtractor):
         'url': 'https://www.ardmediathek.de/video/filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen/Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
         'md5': 'b6e8ab03f2bcc6e1f9e6cef25fcc03c4',
         'info_dict': {
-            'display_id': 'filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen',
-            'id': 'Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
+            'display_id': 'Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
+            'id': '12939099',
             'title': 'Liebe auf vier Pfoten',
             'description': r're:^Claudia Schmitt, Anwältin in Salzburg',
             'duration': 5222,
@@ -255,7 +256,7 @@ class ARDBetaMediathekIE(InfoExtractor):
             'series': 'Filme im MDR',
             'age_limit': 0,
             'channel': 'MDR',
-            '_old_archive_ids': ['ardbetamediathek 12939099'],
+            '_old_archive_ids': ['ardbetamediathek Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0'],
         },
     }, {
         'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
@@ -276,37 +277,37 @@ class ARDBetaMediathekIE(InfoExtractor):
         'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
         'md5': '1e73ded21cb79bac065117e80c81dc88',
         'info_dict': {
-            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
+            'id': '10049223',
             'ext': 'mp4',
             'title': 'tagesschau, 20:00 Uhr',
             'timestamp': 1636398000,
             'description': 'md5:39578c7b96c9fe50afdf5674ad985e6b',
             'upload_date': '20211108',
-            'display_id': 'tagesschau-oder-tagesschau-20-00-uhr/das-erste',
+            'display_id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
             'duration': 915,
             'episode': 'tagesschau, 20:00 Uhr',
             'series': 'tagesschau',
             'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49?w=960&ch=ee69108ae344f678',
             'channel': 'ARD-Aktuell',
-            '_old_archive_ids': ['ardbetamediathek 10049223'],
+            '_old_archive_ids': ['ardbetamediathek Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll'],
         },
     }, {
         'url': 'https://www.ardmediathek.de/video/7-tage/7-tage-unter-harten-jungs/hr-fernsehen/N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3',
         'md5': 'c428b9effff18ff624d4f903bda26315',
         'info_dict': {
-            'id': 'N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3',
+            'id': '94834686',
             'ext': 'mp4',
             'duration': 2700,
             'episode': '7 Tage ... unter harten Jungs',
             'description': 'md5:0f215470dcd2b02f59f4bd10c963f072',
             'upload_date': '20231005',
             'timestamp': 1696491171,
-            'display_id': '7-tage/7-tage-unter-harten-jungs/hr-fernsehen',
+            'display_id': 'N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3',
             'series': '7 Tage ...',
             'channel': 'HR',
             'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:f6e6d5ffac41925c?w=960&ch=fa32ba69bc87989a',
             'title': '7 Tage ... unter harten Jungs',
-            '_old_archive_ids': ['ardbetamediathek 94834686'],
+            '_old_archive_ids': ['ardbetamediathek N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3'],
         },
     }, {
         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
@@ -357,14 +358,25 @@ def _extract_episode_info(self, title):
         }), get_all=False)
 
     def _real_extract(self, url):
-        video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
+        display_id = self._match_id(url)
 
         page_data = self._download_json(
-            f'https://api.ardmediathek.de/page-gateway/pages/ard/item/{video_id}', video_id, query={
+            f'https://api.ardmediathek.de/page-gateway/pages/ard/item/{display_id}', display_id, query={
                 'embedded': 'false',
                 'mcV6': 'true',
             })
 
+        # For user convenience we use the old contentId instead of the longer crid
+        # Ref: https://github.com/yt-dlp/yt-dlp/issues/8731#issuecomment-1874398283
+        old_id = traverse_obj(page_data, ('tracking', 'atiCustomVars', 'contentId', {int}))
+        if old_id is not None:
+            video_id = str(old_id)
+            archive_ids = [make_archive_id(ARDBetaMediathekIE, display_id)]
+        else:
+            self.report_warning(f'Could not extract contentId{bug_reports_message()}')
+            video_id = display_id
+            archive_ids = None
+
         player_data = traverse_obj(
             page_data, ('widgets', lambda _, v: v['type'] in ('player_ondemand', 'player_live'), {dict}), get_all=False)
         is_live = player_data.get('type') == 'player_live'
@@ -419,8 +431,6 @@ def _real_extract(self, url):
                 })
 
         age_limit = traverse_obj(page_data, ('fskRating', {lambda x: remove_start(x, 'FSK')}, {int_or_none}))
-        old_id = traverse_obj(page_data, ('tracking', 'atiCustomVars', 'contentId'))
-
         return {
             'id': video_id,
             'display_id': display_id,
@@ -438,7 +448,7 @@ def _real_extract(self, url):
                 'channel': 'clipSourceName',
             })),
             **self._extract_episode_info(page_data.get('title')),
-            '_old_archive_ids': [make_archive_id(ARDBetaMediathekIE, old_id)],
+            '_old_archive_ids': archive_ids,
         }
 
 

From 5af1f19787f7d652fce72dd3ab9536cdd980fe85 Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Mon, 8 Jan 2024 17:59:44 +0000
Subject: [PATCH 768/871] [ie/NhkRadiruLive] Make metadata extraction non-fatal
 (#8956)

Authored by: garret1317
---
 yt_dlp/extractor/nhk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py
index cc3c79174..4b3d185a3 100644
--- a/yt_dlp/extractor/nhk.py
+++ b/yt_dlp/extractor/nhk.py
@@ -665,7 +665,7 @@ def _real_extract(self, url):
 
         noa_info = self._download_json(
             f'https:{config.find(".//url_program_noa").text}'.format(area=data.find('areakey').text),
-            station, note=f'Downloading {area} station metadata')
+            station, note=f'Downloading {area} station metadata', fatal=False)
         present_info = traverse_obj(noa_info, ('nowonair_list', self._NOA_STATION_IDS.get(station), 'present'))
 
         return {

From 5b8c69ae04444a4c80a5a99917e40f75a116c3b8 Mon Sep 17 00:00:00 2001
From: DmitryScaletta <DmitryScaletta@users.noreply.github.com>
Date: Tue, 9 Jan 2024 05:47:13 +0300
Subject: [PATCH 769/871] [ie/twitch] Fix m3u8 extraction (#8960)

Closes #8958
Authored by: DmitryScaletta
---
 yt_dlp/extractor/twitch.py | 50 +++++++++++++++-----------------------
 1 file changed, 20 insertions(+), 30 deletions(-)

diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index 3297ef091..6dc0993af 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -8,7 +8,6 @@
 from ..compat import (
     compat_parse_qs,
     compat_str,
-    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
@@ -191,6 +190,20 @@ def _get_thumbnails(self, thumbnail):
             'url': thumbnail,
         }] if thumbnail else None
 
+    def _extract_twitch_m3u8_formats(self, video_id, token, signature):
+        """Subclasses must define _M3U8_PATH"""
+        return self._extract_m3u8_formats(
+            f'{self._USHER_BASE}/{self._M3U8_PATH}/{video_id}.m3u8', video_id, 'mp4', query={
+                'allow_source': 'true',
+                'allow_audio_only': 'true',
+                'allow_spectre': 'true',
+                'p': random.randint(1000000, 10000000),
+                'player': 'twitchweb',
+                'playlist_include_framerate': 'true',
+                'sig': signature,
+                'token': token,
+            })
+
 
 class TwitchVodIE(TwitchBaseIE):
     IE_NAME = 'twitch:vod'
@@ -203,6 +216,7 @@ class TwitchVodIE(TwitchBaseIE):
                         )
                         (?P<id>\d+)
                     '''
+    _M3U8_PATH = 'vod'
 
     _TESTS = [{
         'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
@@ -532,20 +546,8 @@ def _real_extract(self, url):
         info = self._extract_info_gql(video, vod_id)
         access_token = self._download_access_token(vod_id, 'video', 'id')
 
-        formats = self._extract_m3u8_formats(
-            '%s/vod/%s.m3u8?%s' % (
-                self._USHER_BASE, vod_id,
-                compat_urllib_parse_urlencode({
-                    'allow_source': 'true',
-                    'allow_audio_only': 'true',
-                    'allow_spectre': 'true',
-                    'player': 'twitchweb',
-                    'playlist_include_framerate': 'true',
-                    'nauth': access_token['value'],
-                    'nauthsig': access_token['signature'],
-                })),
-            vod_id, 'mp4', entry_protocol='m3u8_native')
-
+        formats = self._extract_twitch_m3u8_formats(
+            vod_id, access_token['value'], access_token['signature'])
         formats.extend(self._extract_storyboard(vod_id, video.get('storyboard'), info.get('duration')))
 
         self._prefer_source(formats)
@@ -924,6 +926,7 @@ class TwitchStreamIE(TwitchBaseIE):
                         )
                         (?P<id>[^/#?]+)
                     '''
+    _M3U8_PATH = 'api/channel/hls'
 
     _TESTS = [{
         'url': 'http://www.twitch.tv/shroomztv',
@@ -1026,23 +1029,10 @@ def _real_extract(self, url):
 
         access_token = self._download_access_token(
             channel_name, 'stream', 'channelName')
-        token = access_token['value']
 
         stream_id = stream.get('id') or channel_name
-        query = {
-            'allow_source': 'true',
-            'allow_audio_only': 'true',
-            'allow_spectre': 'true',
-            'p': random.randint(1000000, 10000000),
-            'player': 'twitchweb',
-            'playlist_include_framerate': 'true',
-            'segment_preference': '4',
-            'sig': access_token['signature'].encode('utf-8'),
-            'token': token.encode('utf-8'),
-        }
-        formats = self._extract_m3u8_formats(
-            '%s/api/channel/hls/%s.m3u8' % (self._USHER_BASE, channel_name),
-            stream_id, 'mp4', query=query)
+        formats = self._extract_twitch_m3u8_formats(
+            channel_name, access_token['value'], access_token['signature'])
         self._prefer_source(formats)
 
         view_count = stream.get('viewers')

From 95e82347b398d8bb160767cdd975edecd62cbabd Mon Sep 17 00:00:00 2001
From: Max <github@nbr23.com>
Date: Tue, 9 Jan 2024 03:11:52 +0000
Subject: [PATCH 770/871] [ie/Viously] Add extractor (#8927)

Replaces Turbo extractor

Authored by: nbr23, seproDev

Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 yt_dlp/extractor/_extractors.py |  2 +-
 yt_dlp/extractor/turbo.py       | 64 ---------------------------------
 yt_dlp/extractor/viously.py     | 60 +++++++++++++++++++++++++++++++
 3 files changed, 61 insertions(+), 65 deletions(-)
 delete mode 100644 yt_dlp/extractor/turbo.py
 create mode 100644 yt_dlp/extractor/viously.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 6f7a1e4f1..557ff9447 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2019,7 +2019,6 @@
     TuneInPodcastEpisodeIE,
     TuneInShortenerIE,
 )
-from .turbo import TurboIE
 from .tv2 import (
     TV2IE,
     TV2ArticleIE,
@@ -2223,6 +2222,7 @@
     VikiIE,
     VikiChannelIE,
 )
+from .viously import ViouslyIE
 from .viqeo import ViqeoIE
 from .viu import (
     ViuIE,
diff --git a/yt_dlp/extractor/turbo.py b/yt_dlp/extractor/turbo.py
deleted file mode 100644
index cdb7dcff8..000000000
--- a/yt_dlp/extractor/turbo.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    qualities,
-    xpath_text,
-)
-
-
-class TurboIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?turbo\.fr/videos-voiture/(?P<id>[0-9]+)-'
-    _API_URL = 'http://www.turbo.fr/api/tv/xml.php?player_generique=player_generique&id={0:}'
-    _TEST = {
-        'url': 'http://www.turbo.fr/videos-voiture/454443-turbo-du-07-09-2014-renault-twingo-3-bentley-continental-gt-speed-ces-guide-achat-dacia.html',
-        'md5': '33f4b91099b36b5d5a91f84b5bcba600',
-        'info_dict': {
-            'id': '454443',
-            'ext': 'mp4',
-            'duration': 3715,
-            'title': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia... ',
-            'description': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        video_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, video_id)
-
-        playlist = self._download_xml(self._API_URL.format(video_id), video_id)
-        item = playlist.find('./channel/item')
-        if item is None:
-            raise ExtractorError('Playlist item was not found', expected=True)
-
-        title = xpath_text(item, './title', 'title')
-        duration = int_or_none(xpath_text(item, './durate', 'duration'))
-        thumbnail = xpath_text(item, './visuel_clip', 'thumbnail')
-        description = self._html_search_meta('description', webpage)
-
-        formats = []
-        get_quality = qualities(['3g', 'sd', 'hq'])
-        for child in item:
-            m = re.search(r'url_video_(?P<quality>.+)', child.tag)
-            if m:
-                quality = compat_str(m.group('quality'))
-                formats.append({
-                    'format_id': quality,
-                    'url': child.text,
-                    'quality': get_quality(quality),
-                })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'duration': duration,
-            'thumbnail': thumbnail,
-            'description': description,
-            'formats': formats,
-        }
diff --git a/yt_dlp/extractor/viously.py b/yt_dlp/extractor/viously.py
new file mode 100644
index 000000000..9ec7ed35f
--- /dev/null
+++ b/yt_dlp/extractor/viously.py
@@ -0,0 +1,60 @@
+import base64
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+    parse_iso8601,
+)
+from ..utils.traversal import traverse_obj
+
+
+class ViouslyIE(InfoExtractor):
+    _VALID_URL = False
+    _WEBPAGE_TESTS = [{
+        'url': 'http://www.turbo.fr/videos-voiture/454443-turbo-du-07-09-2014-renault-twingo-3-bentley-continental-gt-speed-ces-guide-achat-dacia.html',
+        'md5': '37a6c3381599381ff53a7e1e0575c0bc',
+        'info_dict': {
+            'id': 'F_xQzS2jwb3',
+            'ext': 'mp4',
+            'title': 'Turbo du 07/09/2014\xa0: Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...',
+            'description': 'Turbo du 07/09/2014\xa0: Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...',
+            'age_limit': 0,
+            'upload_date': '20230328',
+            'timestamp': 1680037507,
+            'duration': 3716,
+            'categories': ['motors'],
+        }
+    }]
+
+    def _extract_from_webpage(self, url, webpage):
+        viously_players = re.findall(r'<div[^>]*class="(?:[^"]*\s)?v(?:iou)?sly-player(?:\s[^"]*)?"[^>]*>', webpage)
+        if not viously_players:
+            return
+
+        def custom_decode(text):
+            STANDARD_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='
+            CUSTOM_ALPHABET = 'VIOUSLYABCDEFGHJKMNPQRTWXZviouslyabcdefghjkmnpqrtwxz9876543210+/='
+            data = base64.b64decode(text.translate(str.maketrans(CUSTOM_ALPHABET, STANDARD_ALPHABET)))
+            return data.decode('utf-8').strip('\x00')
+
+        for video_id in traverse_obj(viously_players, (..., {extract_attributes}, 'id')):
+            formats = self._extract_m3u8_formats(
+                f'https://www.viously.com/video/hls/{video_id}/index.m3u8', video_id, fatal=False)
+            if not formats:
+                continue
+            data = self._download_json(
+                f'https://www.viously.com/export/json/{video_id}', video_id,
+                transform_source=custom_decode, fatal=False)
+            yield {
+                'id': video_id,
+                'formats': formats,
+                **traverse_obj(data, ('video', {
+                    'title': ('title', {str}),
+                    'description': ('description', {str}),
+                    'duration': ('duration', {int_or_none}),
+                    'timestamp': ('iso_date', {parse_iso8601}),
+                    'categories': ('category', 'name', {str}, {lambda x: [x] if x else None}),
+                })),
+            }

From 8e6e3651727b0b85764857fc6329fe5e0a3f00de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Finn=20R=2E=20G=C3=A4rtner?=
 <65015656+FinnRG@users.noreply.github.com>
Date: Sun, 14 Jan 2024 19:28:03 +0100
Subject: [PATCH 771/871] [ie/Piapro] Improve `_VALID_URL` (#8999)

Authored by: FinnRG
---
 yt_dlp/extractor/piapro.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/piapro.py b/yt_dlp/extractor/piapro.py
index 5f39e0639..3ae985da2 100644
--- a/yt_dlp/extractor/piapro.py
+++ b/yt_dlp/extractor/piapro.py
@@ -12,7 +12,7 @@
 
 class PiaproIE(InfoExtractor):
     _NETRC_MACHINE = 'piapro'
-    _VALID_URL = r'https?://piapro\.jp/(?:t|content)/(?P<id>\w+)/?'
+    _VALID_URL = r'https?://piapro\.jp/(?:t|content)/(?P<id>[\w-]+)/?'
     _TESTS = [{
         'url': 'https://piapro.jp/t/NXYR',
         'md5': 'f7c0f760913fb1d44a1c45a4af793909',
@@ -49,6 +49,9 @@ class PiaproIE(InfoExtractor):
     }, {
         'url': 'https://piapro.jp/content/hcw0z3a169wtemz6',
         'only_matching': True
+    }, {
+        'url': 'https://piapro.jp/t/-SO-',
+        'only_matching': True
     }]
 
     _login_status = False

From 014cb5774d7afe624b6eb4e07f7be924b9e5e186 Mon Sep 17 00:00:00 2001
From: Andrew Gibson <agibby5@gmail.com>
Date: Thu, 18 Jan 2024 16:18:04 -0500
Subject: [PATCH 772/871] [ie/aenetworks] Rating should be optional for AP
 extraction (#9005)

Authored by: agibson-fl
---
 yt_dlp/extractor/aenetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/aenetworks.py b/yt_dlp/extractor/aenetworks.py
index 63a0532ef..ab4b6c0eb 100644
--- a/yt_dlp/extractor/aenetworks.py
+++ b/yt_dlp/extractor/aenetworks.py
@@ -93,7 +93,7 @@ def _extract_aetn_info(self, domain, filter_key, filter_value, url):
             resource = self._get_mvpd_resource(
                 requestor_id, theplatform_metadata['title'],
                 theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'),
-                theplatform_metadata['ratings'][0]['rating'])
+                traverse_obj(theplatform_metadata, ('ratings', 0, 'rating')))
             auth = self._extract_mvpd_auth(
                 url, video_id, requestor_id, resource)
         info.update(self._extract_aen_smil(media_url, video_id, auth))

From 4d9dc0abe24ad5d9d22a16f40fc61137dcd103f7 Mon Sep 17 00:00:00 2001
From: Bibhav48 <76898850+Bibhav48@users.noreply.github.com>
Date: Fri, 19 Jan 2024 03:05:04 +0545
Subject: [PATCH 773/871] [ie/cloudflarestream] Extract subtitles (#9007)

Closes #8830
Authored by: Bibhav48
---
 yt_dlp/extractor/cloudflarestream.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/cloudflarestream.py b/yt_dlp/extractor/cloudflarestream.py
index 748e8e908..c4c7d66a5 100644
--- a/yt_dlp/extractor/cloudflarestream.py
+++ b/yt_dlp/extractor/cloudflarestream.py
@@ -46,15 +46,18 @@ def _real_extract(self, url):
                 video_id.split('.')[1] + '==='), video_id)['sub']
         manifest_base_url = base_url + 'manifest/video.'
 
-        formats = self._extract_m3u8_formats(
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
             manifest_base_url + 'm3u8', video_id, 'mp4',
             'm3u8_native', m3u8_id='hls', fatal=False)
-        formats.extend(self._extract_mpd_formats(
-            manifest_base_url + 'mpd', video_id, mpd_id='dash', fatal=False))
+        fmts, subs = self._extract_mpd_formats_and_subtitles(
+            manifest_base_url + 'mpd', video_id, mpd_id='dash', fatal=False)
+        formats.extend(fmts)
+        self._merge_subtitles(subs, target=subtitles)
 
         return {
             'id': video_id,
             'title': video_id,
             'thumbnail': base_url + 'thumbnails/thumbnail.jpg',
             'formats': formats,
+            'subtitles': subtitles,
         }

From 393b487a4ea391c44e811505ec98531031d7e81e Mon Sep 17 00:00:00 2001
From: Nicolas Appriou <nicolas.appriou@gmail.com>
Date: Fri, 19 Jan 2024 00:23:29 +0100
Subject: [PATCH 774/871] [ie/ArteTV] Separate closed captions (#8231)

Authored by: Nicals, seproDev

Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 yt_dlp/extractor/arte.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/arte.py b/yt_dlp/extractor/arte.py
index 139a3a729..92b4900f9 100644
--- a/yt_dlp/extractor/arte.py
+++ b/yt_dlp/extractor/arte.py
@@ -70,7 +70,24 @@ class ArteTVIE(ArteTVBaseIE):
             'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/q82dTTfyuCXupPsGxXsd7B/940x530',
             'upload_date': '20230930',
             'ext': 'mp4',
-        }
+        },
+    }, {
+        'url': 'https://www.arte.tv/de/videos/085374-003-A/im-hohen-norden-geboren/',
+        'info_dict': {
+            'id': '085374-003-A',
+            'ext': 'mp4',
+            'description': 'md5:ab79ec7cc472a93164415b4e4916abf9',
+            'timestamp': 1702872000,
+            'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/TnyHBfPxv3v2GEY3suXGZP/940x530',
+            'duration': 2594,
+            'title': 'Die kurze Zeit der Jugend',
+            'alt_title': 'Im hohen Norden geboren',
+            'upload_date': '20231218',
+            'subtitles': {
+                'fr': 'mincount:1',
+                'fr-acc': 'mincount:1',
+            },
+        },
     }]
 
     _GEO_BYPASS = True
@@ -121,6 +138,16 @@ class ArteTVIE(ArteTVBaseIE):
         ),
     }
 
+    @staticmethod
+    def _fix_accessible_subs_locale(subs):
+        updated_subs = {}
+        for lang, sub_formats in subs.items():
+            for format in sub_formats:
+                if format.get('url', '').endswith('-MAL.m3u8'):
+                    lang += '-acc'
+                updated_subs.setdefault(lang, []).append(format)
+        return updated_subs
+
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         video_id = mobj.group('id')
@@ -174,6 +201,7 @@ def _real_extract(self, url):
                     secondary_formats.extend(fmts)
                 else:
                     formats.extend(fmts)
+                subs = self._fix_accessible_subs_locale(subs)
                 self._merge_subtitles(subs, target=subtitles)
 
             elif stream['protocol'] in ('HTTPS', 'RTMP'):

From 5498729c59b03a9511c64552da3ba2f802166f8d Mon Sep 17 00:00:00 2001
From: jazz1611 <jazz1611@users.noreply.github.com>
Date: Fri, 19 Jan 2024 06:24:34 +0700
Subject: [PATCH 775/871] [ie/GoogleDrive] Fix source file extraction (#8990)

Closes #8976
Authored by: jazz1611
---
 yt_dlp/extractor/googledrive.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/googledrive.py b/yt_dlp/extractor/googledrive.py
index 2fdec20f6..06658dd47 100644
--- a/yt_dlp/extractor/googledrive.py
+++ b/yt_dlp/extractor/googledrive.py
@@ -19,9 +19,9 @@ class GoogleDriveIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                         https?://
                             (?:
-                                (?:docs|drive)\.google\.com/
+                                (?:docs|drive|drive\.usercontent)\.google\.com/
                                 (?:
-                                    (?:uc|open)\?.*?id=|
+                                    (?:uc|open|download)\?.*?id=|
                                     file/d/
                                 )|
                                 video\.google\.com/get_player\?.*?docid=
@@ -53,6 +53,9 @@ class GoogleDriveIE(InfoExtractor):
     }, {
         'url': 'https://drive.google.com/uc?id=0B2fjwgkl1A_CX083Tkowdmt6d28',
         'only_matching': True,
+    }, {
+        'url': 'https://drive.usercontent.google.com/download?id=0ByeS4oOUV-49Zzh4R1J6R09zazQ',
+        'only_matching': True,
     }]
     _FORMATS_EXT = {
         '5': 'flv',
@@ -205,9 +208,10 @@ def get_value(key):
                 formats.append(f)
 
         source_url = update_url_query(
-            'https://drive.google.com/uc', {
+            'https://drive.usercontent.google.com/download', {
                 'id': video_id,
                 'export': 'download',
+                'confirm': 't',
             })
 
         def request_source_file(source_url, kind, data=None):

From cf6413e840476c15e5b166dc2f7cc2a90a4a9aad Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Fri, 19 Jan 2024 08:27:25 +0900
Subject: [PATCH 776/871] [ie/BiliIntl] Fix and improve subtitles extraction
 (#7077)

Closes #7075, Closes #6664
Authored by: HobbyistDev, itachi-19, dirkf, seproDev

Co-authored-by: itachi-19 <16500619+itachi-19@users.noreply.github.com>
Co-authored-by: dirkf <fieldhouse@gmx.net>
Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 yt_dlp/extractor/bilibili.py | 42 +++++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index bc25dc75e..5475b3650 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -18,6 +18,7 @@
     OnDemandPagedList,
     bool_or_none,
     clean_html,
+    determine_ext,
     filter_dict,
     float_or_none,
     format_field,
@@ -1658,19 +1659,34 @@ def _get_subtitles(self, *, ep_id=None, aid=None):
                 'aid': aid,
             })) or {}
         subtitles = {}
-        for sub in sub_json.get('subtitles') or []:
-            sub_url = sub.get('url')
-            if not sub_url:
-                continue
-            sub_data = self._download_json(
-                sub_url, ep_id or aid, errnote='Unable to download subtitles', fatal=False,
-                note='Downloading subtitles%s' % f' for {sub["lang"]}' if sub.get('lang') else '')
-            if not sub_data:
-                continue
-            subtitles.setdefault(sub.get('lang_key', 'en'), []).append({
-                'ext': 'srt',
-                'data': self.json2srt(sub_data)
-            })
+        fetched_urls = set()
+        for sub in traverse_obj(sub_json, (('subtitles', 'video_subtitle'), ..., {dict})):
+            for url in traverse_obj(sub, ((None, 'ass', 'srt'), 'url', {url_or_none})):
+                if url in fetched_urls:
+                    continue
+                fetched_urls.add(url)
+                sub_ext = determine_ext(url)
+                sub_lang = sub.get('lang_key') or 'en'
+
+                if sub_ext == 'ass':
+                    subtitles.setdefault(sub_lang, []).append({
+                        'ext': 'ass',
+                        'url': url,
+                    })
+                elif sub_ext == 'json':
+                    sub_data = self._download_json(
+                        url, ep_id or aid, fatal=False,
+                        note=f'Downloading subtitles{format_field(sub, "lang", " for %s")} ({sub_lang})',
+                        errnote='Unable to download subtitles')
+
+                    if sub_data:
+                        subtitles.setdefault(sub_lang, []).append({
+                            'ext': 'srt',
+                            'data': self.json2srt(sub_data),
+                        })
+                else:
+                    self.report_warning('Unexpected subtitle extension', ep_id or aid)
+
         return subtitles
 
     def _get_formats(self, *, ep_id=None, aid=None):

From cf9af2c7f1fedd881a157b3fbe725e5494b00924 Mon Sep 17 00:00:00 2001
From: Akmal <72781956+Wikidepia@users.noreply.github.com>
Date: Fri, 19 Jan 2024 06:40:08 +0700
Subject: [PATCH 777/871] [ie/Facebook] Add new ID format (#3824)

Closes #3496
Authored by: Wikidepia, kclauhk

Co-authored-by: kclauhk <78251477+kclauhk@users.noreply.github.com>
---
 yt_dlp/extractor/facebook.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index a07a0d344..a16a067ab 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -57,7 +57,7 @@ class FacebookIE(InfoExtractor):
                         )|
                     facebook:
                 )
-                (?P<id>[0-9]+)
+                (?P<id>pfbid[A-Za-z0-9]+|\d+)
                 '''
     _EMBED_REGEX = [
         r'<iframe[^>]+?src=(["\'])(?P<url>https?://www\.facebook\.com/(?:video/embed|plugins/video\.php).+?)\1',
@@ -247,6 +247,24 @@ class FacebookIE(InfoExtractor):
             'thumbnail': r're:^https?://.*',
             'duration': 148.435,
         },
+    }, {
+        'url': 'https://www.facebook.com/attn/posts/pfbid0j1Czf2gGDVqeQ8KiMLFm3pWN8GxsQmeRrVhimWDzMuKQoR8r4b1knNsejELmUgyhl',
+        'info_dict': {
+            'id': '6968553779868435',
+            'ext': 'mp4',
+            'description': 'md5:2f2fcf93e97ac00244fe64521bbdb0cb',
+            'uploader': 'ATTN:',
+            'upload_date': '20231207',
+            'title': 'ATTN:',
+            'duration': 132.675,
+            'uploader_id': '100064451419378',
+            'view_count': int,
+            'thumbnail': r're:^https?://.*',
+            'timestamp': 1701975646,
+        },
+    }, {
+        'url': 'https://www.facebook.com/story.php?story_fbid=pfbid0Fnzhm8UuzjBYpPMNFzaSpFE9UmLdU4fJN8qTANi1Dmtj5q7DNrL5NERXfsAzDEV7l&id=100073071055552',
+        'only_matching': True,
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,

From fee2d8d9c38f9b5f0a8df347c1e698983339c34d Mon Sep 17 00:00:00 2001
From: gmes78 <gmes.078@gmail.com>
Date: Thu, 18 Jan 2024 23:41:28 +0000
Subject: [PATCH 778/871] [ie/Rule34Video] Extract more metadata (#7416)

Closes #7233
Authored by: gmes78
---
 yt_dlp/extractor/rule34video.py | 77 +++++++++++++++++++++++++++++----
 1 file changed, 68 insertions(+), 9 deletions(-)

diff --git a/yt_dlp/extractor/rule34video.py b/yt_dlp/extractor/rule34video.py
index f3250b557..e6bb4258e 100644
--- a/yt_dlp/extractor/rule34video.py
+++ b/yt_dlp/extractor/rule34video.py
@@ -1,7 +1,20 @@
 import re
 
-from ..utils import parse_duration, unescapeHTML
 from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    extract_attributes,
+    get_element_by_attribute,
+    get_element_by_class,
+    get_element_html_by_class,
+    get_elements_by_class,
+    int_or_none,
+    join_nonempty,
+    parse_count,
+    parse_duration,
+    unescapeHTML,
+)
+from ..utils.traversal import traverse_obj
 
 
 class Rule34VideoIE(InfoExtractor):
@@ -17,7 +30,16 @@ class Rule34VideoIE(InfoExtractor):
                 'thumbnail': 'https://rule34video.com/contents/videos_screenshots/3065000/3065157/preview.jpg',
                 'duration': 347.0,
                 'age_limit': 18,
-                'tags': 'count:14'
+                'view_count': int,
+                'like_count': int,
+                'comment_count': int,
+                'timestamp': 1639872000,
+                'description': 'https://discord.gg/aBqPrHSHvv',
+                'upload_date': '20211219',
+                'uploader': 'Sweet HMV',
+                'uploader_url': 'https://rule34video.com/members/22119/',
+                'categories': ['3D', 'MMD', 'iwara'],
+                'tags': 'mincount:10'
             }
         },
         {
@@ -30,7 +52,17 @@ class Rule34VideoIE(InfoExtractor):
                 'thumbnail': 'https://rule34video.com/contents/videos_screenshots/3065000/3065296/preview.jpg',
                 'duration': 938.0,
                 'age_limit': 18,
-                'tags': 'count:50'
+                'view_count': int,
+                'like_count': int,
+                'comment_count': int,
+                'timestamp': 1640131200,
+                'description': '',
+                'creator': 'WildeerStudio',
+                'upload_date': '20211222',
+                'uploader': 'CerZule',
+                'uploader_url': 'https://rule34video.com/members/36281/',
+                'categories': ['3D', 'Tomb Raider'],
+                'tags': 'mincount:40'
             }
         },
     ]
@@ -49,17 +81,44 @@ def _real_extract(self, url):
                 'quality': quality,
             })
 
-        title = self._html_extract_title(webpage)
-        thumbnail = self._html_search_regex(r'preview_url:\s+\'([^\']+)\'', webpage, 'thumbnail', default=None)
-        duration = self._html_search_regex(r'"icon-clock"></i>\s+<span>((?:\d+:?)+)', webpage, 'duration', default=None)
+        categories, creator, uploader, uploader_url = [None] * 4
+        for col in get_elements_by_class('col', webpage):
+            label = clean_html(get_element_by_class('label', col))
+            if label == 'Categories:':
+                categories = list(map(clean_html, get_elements_by_class('item', col)))
+            elif label == 'Artist:':
+                creator = join_nonempty(*map(clean_html, get_elements_by_class('item', col)), delim=', ')
+            elif label == 'Uploaded By:':
+                uploader = clean_html(get_element_by_class('name', col))
+                uploader_url = extract_attributes(get_element_html_by_class('name', col) or '').get('href')
 
         return {
+            **traverse_obj(self._search_json_ld(webpage, video_id, default={}), ({
+                'title': 'title',
+                'view_count': 'view_count',
+                'like_count': 'like_count',
+                'duration': 'duration',
+                'timestamp': 'timestamp',
+                'description': 'description',
+                'thumbnail': ('thumbnails', 0, 'url'),
+            })),
             'id': video_id,
             'formats': formats,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': parse_duration(duration),
+            'title': self._html_extract_title(webpage),
+            'thumbnail': self._html_search_regex(
+                r'preview_url:\s+\'([^\']+)\'', webpage, 'thumbnail', default=None),
+            'duration': parse_duration(self._html_search_regex(
+                r'"icon-clock"></i>\s+<span>((?:\d+:?)+)', webpage, 'duration', default=None)),
+            'view_count': int_or_none(self._html_search_regex(
+                r'"icon-eye"></i>\s+<span>([ \d]+)', webpage, 'views', default='').replace(' ', '')),
+            'like_count': parse_count(get_element_by_class('voters count', webpage)),
+            'comment_count': int_or_none(self._search_regex(
+                r'[^(]+\((\d+)\)', get_element_by_attribute('href', '#tab_comments', webpage), 'comment count', fatal=False)),
             'age_limit': 18,
+            'creator': creator,
+            'uploader': uploader,
+            'uploader_url': uploader_url,
+            'categories': categories,
             'tags': list(map(unescapeHTML, re.findall(
                 r'<a class="tag_item"[^>]+\bhref="https://rule34video\.com/tags/\d+/"[^>]*>(?P<tag>[^>]*)</a>', webpage))),
         }

From 5e2e24b2c5795756d81785b06b10723ddb6db7b2 Mon Sep 17 00:00:00 2001
From: Philipp Waldhauer <ich@philippwaldhauer.de>
Date: Fri, 19 Jan 2024 00:52:13 +0100
Subject: [PATCH 779/871] [ie/MagentaMusik] Add extractor (#7790)

Authored by: pwaldhauer, seproDev

Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 yt_dlp/extractor/_extractors.py     |  2 +-
 yt_dlp/extractor/magentamusik.py    | 62 +++++++++++++++++++++++++++++
 yt_dlp/extractor/magentamusik360.py | 58 ---------------------------
 3 files changed, 63 insertions(+), 59 deletions(-)
 create mode 100644 yt_dlp/extractor/magentamusik.py
 delete mode 100644 yt_dlp/extractor/magentamusik360.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 557ff9447..b49e0366c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -996,7 +996,7 @@
 )
 from .maariv import MaarivIE
 from .magellantv import MagellanTVIE
-from .magentamusik360 import MagentaMusik360IE
+from .magentamusik import MagentaMusikIE
 from .mailru import (
     MailRuIE,
     MailRuMusicIE,
diff --git a/yt_dlp/extractor/magentamusik.py b/yt_dlp/extractor/magentamusik.py
new file mode 100644
index 000000000..9d86a1b21
--- /dev/null
+++ b/yt_dlp/extractor/magentamusik.py
@@ -0,0 +1,62 @@
+from .common import InfoExtractor
+from ..utils import ExtractorError, int_or_none, join_nonempty, url_or_none
+from ..utils.traversal import traverse_obj
+
+
+class MagentaMusikIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?magentamusik\.de/(?P<id>[^/?#]+)'
+
+    _TESTS = [{
+        'url': 'https://www.magentamusik.de/marty-friedman-woa-2023-9208205928595409235',
+        'md5': 'd82dd4748f55fc91957094546aaf8584',
+        'info_dict': {
+            'id': '9208205928595409235',
+            'display_id': 'marty-friedman-woa-2023-9208205928595409235',
+            'ext': 'mp4',
+            'title': 'Marty Friedman: W:O:A 2023',
+            'alt_title': 'Konzert vom: 05.08.2023 13:00',
+            'duration': 2760,
+            'categories': ['Musikkonzert'],
+            'release_year': 2023,
+            'location': 'Deutschland',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        player_config = self._search_json(
+            r'data-js-element="o-video-player__config">', webpage, 'player config', display_id, fatal=False)
+        if not player_config:
+            raise ExtractorError('No video found', expected=True)
+
+        asset_id = player_config['assetId']
+        asset_details = self._download_json(
+            f'https://wcps.t-online.de/cvss/magentamusic/vodclient/v2/assetdetails/58938/{asset_id}',
+            display_id, note='Downloading asset details')
+
+        video_id = traverse_obj(
+            asset_details, ('content', 'partnerInformation', ..., 'reference', {str}), get_all=False)
+        if not video_id:
+            raise ExtractorError('Unable to extract video id')
+
+        vod_data = self._download_json(
+            f'https://wcps.t-online.de/cvss/magentamusic/vodclient/v2/player/58935/{video_id}/Main%20Movie', video_id)
+        smil_url = traverse_obj(
+            vod_data, ('content', 'feature', 'representations', ...,
+                       'contentPackages', ..., 'media', 'href', {url_or_none}), get_all=False)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'formats': self._extract_smil_formats(smil_url, video_id),
+            **traverse_obj(vod_data, ('content', 'feature', 'metadata', {
+                'title': 'title',
+                'alt_title': 'originalTitle',
+                'description': 'longDescription',
+                'duration': ('runtimeInSeconds', {int_or_none}),
+                'location': ('countriesOfProduction', {list}, {lambda x: join_nonempty(*x, delim=', ')}),
+                'release_year': ('yearOfProduction', {int_or_none}),
+                'categories': ('mainGenre', {str}, {lambda x: x and [x]}),
+            })),
+        }
diff --git a/yt_dlp/extractor/magentamusik360.py b/yt_dlp/extractor/magentamusik360.py
deleted file mode 100644
index 5d0cb3bfb..000000000
--- a/yt_dlp/extractor/magentamusik360.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from .common import InfoExtractor
-
-
-class MagentaMusik360IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?magenta-musik-360\.de/([a-z0-9-]+-(?P<id>[0-9]+)|festivals/.+)'
-    _TESTS = [{
-        'url': 'https://www.magenta-musik-360.de/within-temptation-wacken-2019-1-9208205928595185932',
-        'md5': '65b6f060b40d90276ec6fb9b992c1216',
-        'info_dict': {
-            'id': '9208205928595185932',
-            'ext': 'm3u8',
-            'title': 'WITHIN TEMPTATION',
-            'description': 'Robert Westerholt und Sharon Janny den Adel gründeten die Symphonic Metal-Band. Privat sind die Niederländer ein Paar und haben zwei Kinder. Die Single Ice Queen brachte ihnen Platin und Gold und verhalf 2002 zum internationalen Durchbruch. Charakteristisch für die Band war Anfangs der hohe Gesang von Frontfrau Sharon. Stilistisch fing die Band im Gothic Metal an. Mit neuem Sound, schnellen Gitarrenriffs und Gitarrensoli, avancierte Within Temptation zur erfolgreichen Rockband. Auch dieses Jahr wird die Band ihre Fangemeinde wieder mitreißen.',
-        }
-    }, {
-        'url': 'https://www.magenta-musik-360.de/festivals/wacken-world-wide-2020-body-count-feat-ice-t',
-        'md5': '81010d27d7cab3f7da0b0f681b983b7e',
-        'info_dict': {
-            'id': '9208205928595231363',
-            'ext': 'm3u8',
-            'title': 'Body Count feat. Ice-T',
-            'description': 'Body Count feat. Ice-T konnten bereits im vergangenen Jahr auf dem „Holy Ground“ in Wacken überzeugen. 2020 gehen die Crossover-Metaller aus einem Club in Los Angeles auf Sendung und bringen mit ihrer Mischung aus Metal und Hip-Hop Abwechslung und ordentlich Alarm zum WWW. Bereits seit 1990 stehen die beiden Gründer Ice-T (Gesang) und Ernie C (Gitarre) auf der Bühne. Sieben Studioalben hat die Gruppe bis jetzt veröffentlicht, darunter das Debüt „Body Count“ (1992) mit dem kontroversen Track „Cop Killer“.',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        # _match_id casts to string, but since "None" is not a valid video_id for magenta
-        # there is no risk for confusion
-        if video_id == "None":
-            webpage = self._download_webpage(url, video_id)
-            video_id = self._html_search_regex(r'data-asset-id="([^"]+)"', webpage, 'video_id')
-        json = self._download_json("https://wcps.t-online.de/cvss/magentamusic/vodplayer/v3/player/58935/%s/Main%%20Movie" % video_id, video_id)
-        xml_url = json['content']['feature']['representations'][0]['contentPackages'][0]['media']['href']
-        metadata = json['content']['feature'].get('metadata')
-        title = None
-        description = None
-        duration = None
-        thumbnails = []
-        if metadata:
-            title = metadata.get('title')
-            description = metadata.get('fullDescription')
-            duration = metadata.get('runtimeInSeconds')
-            for img_key in ('teaserImageWide', 'smallCoverImage'):
-                if img_key in metadata:
-                    thumbnails.append({'url': metadata[img_key].get('href')})
-
-        xml = self._download_xml(xml_url, video_id)
-        final_url = xml[0][0][0].attrib['src']
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'url': final_url,
-            'duration': duration,
-            'thumbnails': thumbnails
-        }

From aa5dcc4ee65916a36cbe1b1b5b29b9110c3163ed Mon Sep 17 00:00:00 2001
From: Giulio Muscarello <capacitorset@gmail.com>
Date: Fri, 19 Jan 2024 02:51:53 +0000
Subject: [PATCH 780/871] [ie/IlPost] Add extractor (#9001)

Authored by: CapacitorSet
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/ilpost.py      | 69 +++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 yt_dlp/extractor/ilpost.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b49e0366c..5fc39d111 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -787,6 +787,7 @@
     IHeartRadioIE,
     IHeartRadioPodcastIE,
 )
+from .ilpost import IlPostIE
 from .iltalehti import IltalehtiIE
 from .imdb import (
     ImdbIE,
diff --git a/yt_dlp/extractor/ilpost.py b/yt_dlp/extractor/ilpost.py
new file mode 100644
index 000000000..ae98399ee
--- /dev/null
+++ b/yt_dlp/extractor/ilpost.py
@@ -0,0 +1,69 @@
+import functools
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    url_or_none,
+    urlencode_postdata,
+)
+from ..utils.traversal import traverse_obj
+
+
+class IlPostIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ilpost\.it/episodes/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.ilpost.it/episodes/1-avis-akvasas-ka/',
+        'md5': '43649f002d85e1c2f319bb478d479c40',
+        'info_dict': {
+            'id': '2972047',
+            'ext': 'mp3',
+            'display_id': '1-avis-akvasas-ka',
+            'title': '1. Avis akvasas ka',
+            'url': 'https://www.ilpost.it/wp-content/uploads/2023/12/28/1703781217-l-invasione-pt1-v6.mp3',
+            'timestamp': 1703835014,
+            'upload_date': '20231229',
+            'duration': 2495.0,
+            'availability': 'public',
+            'series_id': '235598',
+            'description': '',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        endpoint_metadata = self._search_json(
+            r'var\s+ilpostpodcast\s*=', webpage, 'metadata', display_id)
+        episode_id = endpoint_metadata['post_id']
+        podcast_id = endpoint_metadata['podcast_id']
+        podcast_metadata = self._download_json(
+            endpoint_metadata['ajax_url'], display_id, data=urlencode_postdata({
+                'action': 'checkpodcast',
+                'cookie': endpoint_metadata['cookie'],
+                'post_id': episode_id,
+                'podcast_id': podcast_id,
+            }))
+
+        episode = traverse_obj(podcast_metadata, (
+            'data', 'postcastList', lambda _, v: str(v['id']) == episode_id, {dict}), get_all=False)
+        if not episode:
+            raise ExtractorError('Episode could not be extracted')
+
+        return {
+            'id': episode_id,
+            'display_id': display_id,
+            'series_id': podcast_id,
+            'vcodec': 'none',
+            **traverse_obj(episode, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'url': ('podcast_raw_url', {url_or_none}),
+                'thumbnail': ('image', {url_or_none}),
+                'timestamp': ('timestamp', {int_or_none}),
+                'duration': ('milliseconds', {functools.partial(float_or_none, scale=1000)}),
+                'availability': ('free', {lambda v: 'public' if v else 'subscriber_only'}),
+            }),
+        }

From 6171b050d70435008e64fa06aa6f19c4e5bec75f Mon Sep 17 00:00:00 2001
From: Karavellas <149634176+pompos02@users.noreply.github.com>
Date: Fri, 19 Jan 2024 05:00:49 +0200
Subject: [PATCH 781/871] [ie/ElementorEmbed] Add extractor (#8948)

Authored by: pompos02, seproDev

Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 yt_dlp/extractor/_extractors.py    |  1 +
 yt_dlp/extractor/elementorembed.py | 72 ++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 yt_dlp/extractor/elementorembed.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 5fc39d111..7250ad5e0 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -540,6 +540,7 @@
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE
 from .eitb import EitbIE
+from .elementorembed import ElementorEmbedIE
 from .elonet import ElonetIE
 from .elpais import ElPaisIE
 from .eltrecetv import ElTreceTVIE
diff --git a/yt_dlp/extractor/elementorembed.py b/yt_dlp/extractor/elementorembed.py
new file mode 100644
index 000000000..638893f6f
--- /dev/null
+++ b/yt_dlp/extractor/elementorembed.py
@@ -0,0 +1,72 @@
+import re
+
+from .common import InfoExtractor
+from .vimeo import VimeoIE
+from .youtube import YoutubeIE
+from ..utils import unescapeHTML, url_or_none
+from ..utils.traversal import traverse_obj
+
+
+class ElementorEmbedIE(InfoExtractor):
+    _VALID_URL = False
+    _WEBPAGE_TESTS = [{
+        'url': 'https://capitaltv.cy/2023/12/14/υγεια-και-ζωη-14-12-2023-δρ-ξενια-κωσταντινιδο/',
+        'info_dict': {
+            'id': 'KgzuxwuQwM4',
+            'ext': 'mp4',
+            'title': 'ΥΓΕΙΑ ΚΑΙ ΖΩΗ 14 12 2023 ΔΡ  ΞΕΝΙΑ ΚΩΣΤΑΝΤΙΝΙΔΟΥ',
+            'thumbnail': 'https://i.ytimg.com/vi/KgzuxwuQwM4/maxresdefault.jpg',
+            'playable_in_embed': True,
+            'tags': 'count:16',
+            'like_count': int,
+            'channel': 'Capital TV Cyprus',
+            'channel_id': 'UCR8LwVKTLGEXt4ZAErpCMrg',
+            'availability': 'public',
+            'description': 'md5:7a3308a22881aea4612358c4ba121f77',
+            'duration': 2891,
+            'upload_date': '20231214',
+            'uploader_id': '@capitaltvcyprus6389',
+            'live_status': 'not_live',
+            'channel_url': 'https://www.youtube.com/channel/UCR8LwVKTLGEXt4ZAErpCMrg',
+            'uploader_url': 'https://www.youtube.com/@capitaltvcyprus6389',
+            'uploader': 'Capital TV Cyprus',
+            'age_limit': 0,
+            'categories': ['News & Politics'],
+            'view_count': int,
+            'channel_follower_count': int,
+        },
+    }, {
+        'url': 'https://elementor.com/academy/theme-builder-collection/?playlist=76011151&video=9e59909',
+        'info_dict': {
+            'id': '?playlist=76011151&video=9e59909',
+            'title': 'Theme Builder Collection - Academy',
+            'age_limit': 0,
+            'timestamp': 1702196984.0,
+            'upload_date': '20231210',
+            'description': 'md5:7f52c52715ee9e54fd7f82210511673d',
+            'thumbnail': 'https://elementor.com/academy/wp-content/uploads/2021/07/Theme-Builder-1.png',
+        },
+        'playlist_count': 11,
+        'params': {
+            'skip_download': True,
+        },
+    }]
+    _WIDGET_REGEX = r'<div[^>]+class="[^"]*elementor-widget-video(?:-playlist)?[^"]*"[^>]*data-settings="([^"]*)"'
+
+    def _extract_from_webpage(self, url, webpage):
+        for data_settings in re.findall(self._WIDGET_REGEX, webpage):
+            data = self._parse_json(data_settings, None, fatal=False, transform_source=unescapeHTML)
+            if youtube_url := traverse_obj(data, ('youtube_url', {url_or_none})):
+                yield self.url_result(youtube_url, ie=YoutubeIE)
+
+            for video in traverse_obj(data, ('tabs', lambda _, v: v['_id'], {dict})):
+                if youtube_url := traverse_obj(video, ('youtube_url', {url_or_none})):
+                    yield self.url_result(youtube_url, ie=YoutubeIE)
+                if vimeo_url := traverse_obj(video, ('vimeo_url', {url_or_none})):
+                    yield self.url_result(vimeo_url, ie=VimeoIE)
+                for direct_url in traverse_obj(video, (('hosted_url', 'external_url'), 'url', {url_or_none})):
+                    yield {
+                        'id': video['_id'],
+                        'url': direct_url,
+                        'title': video.get('title'),
+                    }

From ba6b0c8261e9f0a6373885736ff90a89dd1fb614 Mon Sep 17 00:00:00 2001
From: DmitryScaletta <DmitryScaletta@users.noreply.github.com>
Date: Fri, 19 Jan 2024 06:16:21 +0300
Subject: [PATCH 782/871] [ie/chzzk] Add extractors (#8887)

Closes #8804
Authored by: DmitryScaletta
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/chzzk.py       | 139 ++++++++++++++++++++++++++++++++
 2 files changed, 143 insertions(+)
 create mode 100644 yt_dlp/extractor/chzzk.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 7250ad5e0..3d360a52f 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -345,6 +345,10 @@
     ChingariIE,
     ChingariUserIE,
 )
+from .chzzk import (
+    CHZZKLiveIE,
+    CHZZKVideoIE,
+)
 from .cinemax import CinemaxIE
 from .cinetecamilano import CinetecaMilanoIE
 from .cineverse import (
diff --git a/yt_dlp/extractor/chzzk.py b/yt_dlp/extractor/chzzk.py
new file mode 100644
index 000000000..6894baea5
--- /dev/null
+++ b/yt_dlp/extractor/chzzk.py
@@ -0,0 +1,139 @@
+import functools
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class CHZZKLiveIE(InfoExtractor):
+    IE_NAME = 'chzzk:live'
+    _VALID_URL = r'https?://chzzk\.naver\.com/live/(?P<id>[\da-f]+)'
+    _TESTS = [{
+        'url': 'https://chzzk.naver.com/live/c68b8ef525fb3d2fa146344d84991753',
+        'info_dict': {
+            'id': 'c68b8ef525fb3d2fa146344d84991753',
+            'ext': 'mp4',
+            'title': str,
+            'channel': '진짜도현',
+            'channel_id': 'c68b8ef525fb3d2fa146344d84991753',
+            'channel_is_verified': False,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1705510344,
+            'upload_date': '20240117',
+            'live_status': 'is_live',
+            'view_count': int,
+            'concurrent_view_count': int,
+        },
+        'skip': 'The channel is not currently live',
+    }]
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+        live_detail = self._download_json(
+            f'https://api.chzzk.naver.com/service/v2/channels/{channel_id}/live-detail', channel_id,
+            note='Downloading channel info', errnote='Unable to download channel info')['content']
+
+        if live_detail.get('status') == 'CLOSE':
+            raise ExtractorError('The channel is not currently live', expected=True)
+
+        live_playback = self._parse_json(live_detail['livePlaybackJson'], channel_id)
+
+        thumbnails = []
+        thumbnail_template = traverse_obj(
+            live_playback, ('thumbnail', 'snapshotThumbnailTemplate', {url_or_none}))
+        if thumbnail_template and '{type}' in thumbnail_template:
+            for width in traverse_obj(live_playback, ('thumbnail', 'types', ..., {str})):
+                thumbnails.append({
+                    'id': width,
+                    'url': thumbnail_template.replace('{type}', width),
+                    'width': int_or_none(width),
+                })
+
+        formats, subtitles = [], {}
+        for media in traverse_obj(live_playback, ('media', lambda _, v: url_or_none(v['path']))):
+            is_low_latency = media.get('mediaId') == 'LLHLS'
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                media['path'], channel_id, 'mp4', fatal=False, live=True,
+                m3u8_id='hls-ll' if is_low_latency else 'hls')
+            for f in fmts:
+                if is_low_latency:
+                    f['source_preference'] = -2
+                if '-afragalow.stream-audio.stream' in f['format_id']:
+                    f['quality'] = -2
+            formats.extend(fmts)
+            self._merge_subtitles(subs, target=subtitles)
+
+        return {
+            'id': channel_id,
+            'is_live': True,
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnails': thumbnails,
+            **traverse_obj(live_detail, {
+                'title': ('liveTitle', {str}),
+                'timestamp': ('openDate', {functools.partial(parse_iso8601, delimiter=' ')}),
+                'concurrent_view_count': ('concurrentUserCount', {int_or_none}),
+                'view_count': ('accumulateCount', {int_or_none}),
+                'channel': ('channel', 'channelName', {str}),
+                'channel_id': ('channel', 'channelId', {str}),
+                'channel_is_verified': ('channel', 'verifiedMark', {bool}),
+            }),
+        }
+
+
+class CHZZKVideoIE(InfoExtractor):
+    IE_NAME = 'chzzk:video'
+    _VALID_URL = r'https?://chzzk\.naver\.com/video/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://chzzk.naver.com/video/1754',
+        'md5': 'b0c0c1bb888d913b93d702b1512c7f06',
+        'info_dict': {
+            'id': '1754',
+            'ext': 'mp4',
+            'title': '치지직 테스트 방송',
+            'channel': '침착맨',
+            'channel_id': 'bb382c2c0cc9fa7c86ab3b037fb5799c',
+            'channel_is_verified': False,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 15577,
+            'timestamp': 1702970505.417,
+            'upload_date': '20231219',
+            'view_count': int,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_meta = self._download_json(
+            f'https://api.chzzk.naver.com/service/v2/videos/{video_id}', video_id,
+            note='Downloading video info', errnote='Unable to download video info')['content']
+        formats, subtitles = self._extract_mpd_formats_and_subtitles(
+            f'https://apis.naver.com/neonplayer/vodplay/v1/playback/{video_meta["videoId"]}', video_id,
+            query={
+                'key': video_meta['inKey'],
+                'env': 'real',
+                'lc': 'en_US',
+                'cpl': 'en_US',
+            }, note='Downloading video playback', errnote='Unable to download video playback')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(video_meta, {
+                'title': ('videoTitle', {str}),
+                'thumbnail': ('thumbnailImageUrl', {url_or_none}),
+                'timestamp': ('publishDateAt', {functools.partial(float_or_none, scale=1000)}),
+                'view_count': ('readCount', {int_or_none}),
+                'duration': ('duration', {int_or_none}),
+                'channel': ('channel', 'channelName', {str}),
+                'channel_id': ('channel', 'channelId', {str}),
+                'channel_is_verified': ('channel', 'verifiedMark', {bool}),
+            }),
+        }

From a281beba8d8f007cf220f96dd1d9412bb070c7d8 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Fri, 19 Jan 2024 05:41:10 +0100
Subject: [PATCH 783/871] [ie/naver] Fix extractors (#8883)

Closes #8850, Closes #8692
Authored by: seproDev
---
 yt_dlp/extractor/naver.py | 173 ++++++++++++++++++++------------------
 1 file changed, 90 insertions(+), 83 deletions(-)

diff --git a/yt_dlp/extractor/naver.py b/yt_dlp/extractor/naver.py
index 2d8459b02..806b79082 100644
--- a/yt_dlp/extractor/naver.py
+++ b/yt_dlp/extractor/naver.py
@@ -1,20 +1,25 @@
+import base64
+import hashlib
+import hmac
 import itertools
+import json
 import re
-from urllib.parse import urlparse, parse_qs
+import time
+from urllib.parse import parse_qs, urlparse
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    clean_html,
     dict_get,
     int_or_none,
     join_nonempty,
     merge_dicts,
-    parse_duration,
+    parse_iso8601,
     traverse_obj,
     try_get,
     unified_timestamp,
     update_url_query,
+    url_or_none,
 )
 
 
@@ -110,6 +115,18 @@ def get_subs(caption_url):
             **self.process_subtitles(video_data, get_subs),
         }
 
+    def _call_api(self, path, video_id):
+        api_endpoint = f'https://apis.naver.com/now_web2/now_web_api/v1{path}'
+        key = b'nbxvs5nwNG9QKEWK0ADjYA4JZoujF4gHcIwvoCxFTPAeamq5eemvt5IWAYXxrbYM'
+        msgpad = int(time.time() * 1000)
+        md = base64.b64encode(hmac.HMAC(
+            key, f'{api_endpoint[:255]}{msgpad}'.encode(), digestmod=hashlib.sha1).digest()).decode()
+
+        return self._download_json(api_endpoint, video_id=video_id, headers=self.geo_verification_headers(), query={
+            'msgpad': msgpad,
+            'md': md,
+        })['result']
+
 
 class NaverIE(NaverBaseIE):
     _VALID_URL = r'https?://(?:m\.)?tv(?:cast)?\.naver\.com/(?:v|embed)/(?P<id>\d+)'
@@ -125,21 +142,32 @@ class NaverIE(NaverBaseIE):
             'upload_date': '20130903',
             'uploader': '메가스터디, 합격불변의 법칙',
             'uploader_id': 'megastudy',
+            'uploader_url': 'https://tv.naver.com/megastudy',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'duration': 2118,
+            'thumbnail': r're:^https?://.*\.jpg',
         },
     }, {
         'url': 'http://tv.naver.com/v/395837',
-        'md5': '8a38e35354d26a17f73f4e90094febd3',
+        'md5': '7791205fa89dbed2f5e3eb16d287ff05',
         'info_dict': {
             'id': '395837',
             'ext': 'mp4',
             'title': '9년이 지나도 아픈 기억, 전효성의 아버지',
-            'description': 'md5:eb6aca9d457b922e43860a2a2b1984d3',
+            'description': 'md5:c76be23e21403a6473d8119678cdb5cb',
             'timestamp': 1432030253,
             'upload_date': '20150519',
-            'uploader': '4가지쇼 시즌2',
-            'uploader_id': 'wrappinguser29',
+            'uploader': '4가지쇼',
+            'uploader_id': '4show',
+            'uploader_url': 'https://tv.naver.com/4show',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'duration': 277,
+            'thumbnail': r're:^https?://.*\.jpg',
         },
-        'skip': 'Georestricted',
     }, {
         'url': 'http://tvcast.naver.com/v/81652',
         'only_matching': True,
@@ -147,56 +175,63 @@ class NaverIE(NaverBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        content = self._download_json(
-            'https://tv.naver.com/api/json/v/' + video_id,
-            video_id, headers=self.geo_verification_headers())
-        player_info_json = content.get('playerInfoJson') or {}
-        current_clip = player_info_json.get('currentClip') or {}
+        data = self._call_api(f'/clips/{video_id}/play-info', video_id)
 
-        vid = current_clip.get('videoId')
-        in_key = current_clip.get('inKey')
+        vid = traverse_obj(data, ('clip', 'videoId', {str}))
+        in_key = traverse_obj(data, ('play', 'inKey', {str}))
 
         if not vid or not in_key:
-            player_auth = try_get(player_info_json, lambda x: x['playerOption']['auth'])
-            if player_auth == 'notCountry':
-                self.raise_geo_restricted(countries=['KR'])
-            elif player_auth == 'notLogin':
-                self.raise_login_required()
-            raise ExtractorError('couldn\'t extract vid and key')
+            raise ExtractorError('Unable to extract video info')
+
         info = self._extract_video_info(video_id, vid, in_key)
-        info.update({
-            'description': clean_html(current_clip.get('description')),
-            'timestamp': int_or_none(current_clip.get('firstExposureTime'), 1000),
-            'duration': parse_duration(current_clip.get('displayPlayTime')),
-            'like_count': int_or_none(current_clip.get('recommendPoint')),
-            'age_limit': 19 if current_clip.get('adult') else None,
-        })
+        info.update(traverse_obj(data, ('clip', {
+            'title': 'title',
+            'description': 'description',
+            'timestamp': ('firstExposureDatetime', {parse_iso8601}),
+            'duration': ('playTime', {int_or_none}),
+            'like_count': ('likeItCount', {int_or_none}),
+            'view_count': ('playCount', {int_or_none}),
+            'comment_count': ('commentCount', {int_or_none}),
+            'thumbnail': ('thumbnailImageUrl', {url_or_none}),
+            'uploader': 'channelName',
+            'uploader_id': 'channelId',
+            'uploader_url': ('channelUrl', {url_or_none}),
+            'age_limit': ('adultVideo', {lambda x: 19 if x else None}),
+        })))
         return info
 
 
-class NaverLiveIE(InfoExtractor):
+class NaverLiveIE(NaverBaseIE):
     IE_NAME = 'Naver:live'
     _VALID_URL = r'https?://(?:m\.)?tv(?:cast)?\.naver\.com/l/(?P<id>\d+)'
     _GEO_BYPASS = False
     _TESTS = [{
-        'url': 'https://tv.naver.com/l/52010',
+        'url': 'https://tv.naver.com/l/127062',
         'info_dict': {
-            'id': '52010',
+            'id': '127062',
             'ext': 'mp4',
-            'title': '[LIVE] 뉴스특보 : "수도권 거리두기, 2주간 2단계로 조정"',
-            'description': 'md5:df7f0c237a5ed5e786ce5c91efbeaab3',
-            'channel_id': 'NTV-ytnnews24-0',
-            'start_time': 1597026780000,
+            'live_status': 'is_live',
+            'channel': '뉴스는 YTN',
+            'channel_id': 'ytnnews24',
+            'title': 're:^대한민국 24시간 뉴스 채널 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:f938b5956711beab6f882314ffadf4d5',
+            'start_time': 1677752280,
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)',
+            'like_count': int,
         },
     }, {
-        'url': 'https://tv.naver.com/l/51549',
+        'url': 'https://tv.naver.com/l/140535',
         'info_dict': {
-            'id': '51549',
+            'id': '140535',
             'ext': 'mp4',
-            'title': '연합뉴스TV - 코로나19 뉴스특보',
-            'description': 'md5:c655e82091bc21e413f549c0eaccc481',
-            'channel_id': 'NTV-yonhapnewstv-0',
-            'start_time': 1596406380000,
+            'live_status': 'is_live',
+            'channel': 'KBS뉴스',
+            'channel_id': 'kbsnews',
+            'start_time': 1696867320,
+            'title': 're:^언제 어디서나! KBS 뉴스 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:6ad419c0bf2f332829bda3f79c295284',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)',
+            'like_count': int,
         },
     }, {
         'url': 'https://tv.naver.com/l/54887',
@@ -205,55 +240,27 @@ class NaverLiveIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id, 'Downloading Page', 'Unable to download Page')
-        secure_url = self._search_regex(r'sApiF:\s+(?:"|\')([^"\']+)', page, 'secureurl')
-
-        info = self._extract_video_info(video_id, secure_url)
-        info.update({
-            'description': self._og_search_description(page)
-        })
-
-        return info
-
-    def _extract_video_info(self, video_id, url):
-        video_data = self._download_json(url, video_id, headers=self.geo_verification_headers())
-        meta = video_data.get('meta')
-        status = meta.get('status')
+        data = self._call_api(f'/live-end/normal/{video_id}/play-info?renewLastPlayDate=true', video_id)
 
+        status = traverse_obj(data, ('live', 'liveStatus'))
         if status == 'CLOSED':
             raise ExtractorError('Stream is offline.', expected=True)
         elif status != 'OPENED':
-            raise ExtractorError('Unknown status %s' % status)
-
-        title = meta.get('title')
-        stream_list = video_data.get('streams')
-
-        if stream_list is None:
-            raise ExtractorError('Could not get stream data.', expected=True)
-
-        formats = []
-        for quality in stream_list:
-            if not quality.get('url'):
-                continue
-
-            prop = quality.get('property')
-            if prop.get('abr'):  # This abr doesn't mean Average audio bitrate.
-                continue
-
-            formats.extend(self._extract_m3u8_formats(
-                quality.get('url'), video_id, 'mp4',
-                m3u8_id=quality.get('qualityId'), live=True
-            ))
+            raise ExtractorError(f'Unknown status {status!r}')
 
         return {
             'id': video_id,
-            'title': title,
-            'formats': formats,
-            'channel_id': meta.get('channelId'),
-            'channel_url': meta.get('channelUrl'),
-            'thumbnail': meta.get('imgUrl'),
-            'start_time': meta.get('startTime'),
-            'categories': [meta.get('categoryId')],
+            'formats': self._extract_m3u8_formats(
+                traverse_obj(data, ('playbackBody', {json.loads}, 'media', 0, 'path')), video_id, live=True),
+            **traverse_obj(data, ('live', {
+                'title': 'title',
+                'channel': 'channelName',
+                'channel_id': 'channelId',
+                'description': 'description',
+                'like_count': (('likeCount', 'likeItCount'), {int_or_none}),
+                'thumbnail': ('thumbnailImageUrl', {url_or_none}),
+                'start_time': (('startTime', 'startDateTime', 'startYmdt'), {parse_iso8601}),
+            }), get_all=False),
             'is_live': True
         }
 

From c51316f8a69fbd0080f2720777d42ab438e254a3 Mon Sep 17 00:00:00 2001
From: sefidel <contact@sefidel.net>
Date: Fri, 19 Jan 2024 18:43:13 +0900
Subject: [PATCH 784/871] [ie/abematv] Fix extraction with cache (#8895)

Closes #6532
Authored by: sefidel
---
 yt_dlp/extractor/abematv.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index 57ccb928b..0a610e315 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -136,11 +136,15 @@ def _get_device_token(self):
         if self._USERTOKEN:
             return self._USERTOKEN
 
+        add_opener(self._downloader, AbemaLicenseHandler(self))
+
         username, _ = self._get_login_info()
-        AbemaTVBaseIE._USERTOKEN = username and self.cache.load(self._NETRC_MACHINE, username)
+        auth_cache = username and self.cache.load(self._NETRC_MACHINE, username, min_ver='2024.01.19')
+        AbemaTVBaseIE._USERTOKEN = auth_cache and auth_cache.get('usertoken')
         if AbemaTVBaseIE._USERTOKEN:
             # try authentication with locally stored token
             try:
+                AbemaTVBaseIE._DEVICE_ID = auth_cache.get('device_id')
                 self._get_media_token(True)
                 return
             except ExtractorError as e:
@@ -159,7 +163,6 @@ def _get_device_token(self):
             })
         AbemaTVBaseIE._USERTOKEN = user_data['token']
 
-        add_opener(self._downloader, AbemaLicenseHandler(self))
         return self._USERTOKEN
 
     def _get_media_token(self, invalidate=False, to_show=True):
@@ -255,7 +258,7 @@ class AbemaTVIE(AbemaTVBaseIE):
 
     def _perform_login(self, username, password):
         self._get_device_token()
-        if self.cache.load(self._NETRC_MACHINE, username) and self._get_media_token():
+        if self.cache.load(self._NETRC_MACHINE, username, min_ver='2024.01.19') and self._get_media_token():
             self.write_debug('Skipping logging in')
             return
 
@@ -278,7 +281,11 @@ def _perform_login(self, username, password):
 
         AbemaTVBaseIE._USERTOKEN = login_response['token']
         self._get_media_token(True)
-        self.cache.store(self._NETRC_MACHINE, username, AbemaTVBaseIE._USERTOKEN)
+        auth_cache = {
+            'device_id': AbemaTVBaseIE._DEVICE_ID,
+            'usertoken': AbemaTVBaseIE._USERTOKEN,
+        }
+        self.cache.store(self._NETRC_MACHINE, username, auth_cache)
 
     def _real_extract(self, url):
         # starting download using infojson from this extractor is undefined behavior,

From 8226a3818f804478c756cf460baa9bf3a3b062a5 Mon Sep 17 00:00:00 2001
From: sefidel <contact@sefidel.net>
Date: Fri, 19 Jan 2024 18:50:16 +0900
Subject: [PATCH 785/871] [ie/abematv] Support login for playlists (#8901)

Authored by: sefidel
---
 yt_dlp/extractor/abematv.py | 65 +++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 32 deletions(-)

diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index 0a610e315..6453dde97 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -92,6 +92,8 @@ def abematv_license_open(self, url):
 
 
 class AbemaTVBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'abematv'
+
     _USERTOKEN = None
     _DEVICE_ID = None
     _MEDIATOKEN = None
@@ -184,6 +186,37 @@ def _get_media_token(self, invalidate=False, to_show=True):
 
         return self._MEDIATOKEN
 
+    def _perform_login(self, username, password):
+        self._get_device_token()
+        if self.cache.load(self._NETRC_MACHINE, username, min_ver='2024.01.19') and self._get_media_token():
+            self.write_debug('Skipping logging in')
+            return
+
+        if '@' in username:  # don't strictly check if it's email address or not
+            ep, method = 'user/email', 'email'
+        else:
+            ep, method = 'oneTimePassword', 'userId'
+
+        login_response = self._download_json(
+            f'https://api.abema.io/v1/auth/{ep}', None, note='Logging in',
+            data=json.dumps({
+                method: username,
+                'password': password
+            }).encode('utf-8'), headers={
+                'Authorization': f'bearer {self._get_device_token()}',
+                'Origin': 'https://abema.tv',
+                'Referer': 'https://abema.tv/',
+                'Content-Type': 'application/json',
+            })
+
+        AbemaTVBaseIE._USERTOKEN = login_response['token']
+        self._get_media_token(True)
+        auth_cache = {
+            'device_id': AbemaTVBaseIE._DEVICE_ID,
+            'usertoken': AbemaTVBaseIE._USERTOKEN,
+        }
+        self.cache.store(self._NETRC_MACHINE, username, auth_cache)
+
     def _call_api(self, endpoint, video_id, query=None, note='Downloading JSON metadata'):
         return self._download_json(
             f'https://api.abema.io/{endpoint}', video_id, query=query or {},
@@ -207,7 +240,6 @@ def _extract_breadcrumb_list(self, webpage, video_id):
 
 class AbemaTVIE(AbemaTVBaseIE):
     _VALID_URL = r'https?://abema\.tv/(?P<type>now-on-air|video/episode|channels/.+?/slots)/(?P<id>[^?/]+)'
-    _NETRC_MACHINE = 'abematv'
     _TESTS = [{
         'url': 'https://abema.tv/video/episode/194-25_s2_p1',
         'info_dict': {
@@ -256,37 +288,6 @@ class AbemaTVIE(AbemaTVBaseIE):
     }]
     _TIMETABLE = None
 
-    def _perform_login(self, username, password):
-        self._get_device_token()
-        if self.cache.load(self._NETRC_MACHINE, username, min_ver='2024.01.19') and self._get_media_token():
-            self.write_debug('Skipping logging in')
-            return
-
-        if '@' in username:  # don't strictly check if it's email address or not
-            ep, method = 'user/email', 'email'
-        else:
-            ep, method = 'oneTimePassword', 'userId'
-
-        login_response = self._download_json(
-            f'https://api.abema.io/v1/auth/{ep}', None, note='Logging in',
-            data=json.dumps({
-                method: username,
-                'password': password
-            }).encode('utf-8'), headers={
-                'Authorization': f'bearer {self._get_device_token()}',
-                'Origin': 'https://abema.tv',
-                'Referer': 'https://abema.tv/',
-                'Content-Type': 'application/json',
-            })
-
-        AbemaTVBaseIE._USERTOKEN = login_response['token']
-        self._get_media_token(True)
-        auth_cache = {
-            'device_id': AbemaTVBaseIE._DEVICE_ID,
-            'usertoken': AbemaTVBaseIE._USERTOKEN,
-        }
-        self.cache.store(self._NETRC_MACHINE, username, auth_cache)
-
     def _real_extract(self, url):
         # starting download using infojson from this extractor is undefined behavior,
         # and never be fixed in the future; you must trigger downloads by directly specifying URL.

From 43694ce13c5a9f1afca8b02b8b2b9b1576d6503d Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Fri, 19 Jan 2024 15:19:09 +0000
Subject: [PATCH 786/871] [ie/NineNews] Add extractor (#8840)

Closes #8831
Authored by: SirElderling
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/ninenews.py    | 72 +++++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 yt_dlp/extractor/ninenews.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 3d360a52f..abba5bfa2 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1269,6 +1269,7 @@
     NiconicoChannelPlusChannelLivesIE,
 )
 from .ninegag import NineGagIE
+from .ninenews import NineNewsIE
 from .ninenow import NineNowIE
 from .nintendo import NintendoIE
 from .nitter import NitterIE
diff --git a/yt_dlp/extractor/ninenews.py b/yt_dlp/extractor/ninenews.py
new file mode 100644
index 000000000..900d9ba60
--- /dev/null
+++ b/yt_dlp/extractor/ninenews.py
@@ -0,0 +1,72 @@
+from .common import InfoExtractor
+from .brightcove import BrightcoveNewIE
+from ..utils import ExtractorError
+from ..utils.traversal import traverse_obj
+
+
+class NineNewsIE(InfoExtractor):
+    IE_NAME = '9News'
+    _VALID_URL = r'https?://(?:www\.)?9news\.com\.au/(?:[\w-]+/){2,3}(?P<id>[\w-]+)/?(?:$|[?#])'
+    _TESTS = [{
+        'url': 'https://www.9news.com.au/videos/national/fair-trading-pulls-dozens-of-toys-from-shelves/clqgc7dvj000y0jnvfism0w5m',
+        'md5': 'd1a65b2e9d126e5feb9bc5cb96e62c80',
+        'info_dict': {
+            'id': '6343717246112',
+            'ext': 'mp4',
+            'title': 'Fair Trading pulls dozens of toys from shelves',
+            'description': 'Fair Trading Australia have been forced to pull dozens of toys from shelves over hazard fears.',
+            'thumbnail': 'md5:bdbe44294e2323b762d97acf8843f66c',
+            'duration': 93.44,
+            'timestamp': 1703231748,
+            'upload_date': '20231222',
+            'uploader_id': '664969388001',
+            'tags': ['networkclip', 'aunews_aunationalninenews', 'christmas presents', 'toys', 'fair trading', 'au_news'],
+        }
+    }, {
+        'url': 'https://www.9news.com.au/world/tape-reveals-donald-trump-pressured-michigan-officials-not-to-certify-2020-vote-a-new-report-says/0b8b880e-7d3c-41b9-b2bd-55bc7e492259',
+        'md5': 'a885c44d20898c3e70e9a53e8188cea1',
+        'info_dict': {
+            'id': '6343587450112',
+            'ext': 'mp4',
+            'title': 'Trump found ineligible to run for president by state court',
+            'description': 'md5:40e6e7db7a4ac6be0e960569a5af6066',
+            'thumbnail': 'md5:3e132c48c186039fd06c10787de9bff2',
+            'duration': 104.64,
+            'timestamp': 1703058034,
+            'upload_date': '20231220',
+            'uploader_id': '664969388001',
+            'tags': ['networkclip', 'aunews_aunationalninenews', 'ineligible', 'presidential candidate', 'donald trump', 'au_news'],
+        }
+    }, {
+        'url': 'https://www.9news.com.au/national/outrage-as-parents-banned-from-giving-gifts-to-kindergarten-teachers/e19b49d4-a1a4-4533-9089-6e10e2d9386a',
+        'info_dict': {
+            'id': '6343716797112',
+            'ext': 'mp4',
+            'title': 'Outrage as parents banned from giving gifts to kindergarten teachers',
+            'description': 'md5:7a8b0ed2f9e08875fd9a3e86e462bc46',
+            'thumbnail': 'md5:5ee4d66717bdd0dee9fc9a705ef041b8',
+            'duration': 91.307,
+            'timestamp': 1703229584,
+            'upload_date': '20231222',
+            'uploader_id': '664969388001',
+            'tags': ['networkclip', 'aunews_aunationalninenews', 'presents', 'teachers', 'kindergarten', 'au_news'],
+        },
+    }]
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        webpage = self._download_webpage(url, article_id)
+        initial_state = self._search_json(
+            r'var\s+__INITIAL_STATE__\s*=', webpage, 'initial state', article_id)
+        video_id = traverse_obj(
+            initial_state, ('videoIndex', 'currentVideo', 'brightcoveId', {str}),
+            ('article', ..., 'media', lambda _, v: v['type'] == 'video', 'urn', {str}), get_all=False)
+        account = traverse_obj(initial_state, (
+            'videoIndex', 'config', (None, 'video'), 'account', {str}), get_all=False)
+
+        if not video_id or not account:
+            raise ExtractorError('Unable to get the required video data')
+
+        return self.url_result(
+            f'https://players.brightcove.net/{account}/default_default/index.html?videoId={video_id}',
+            BrightcoveNewIE, video_id)

From 20cdad5a2c0499d5a6746f5466a2ab0c97b75884 Mon Sep 17 00:00:00 2001
From: DmitryScaletta <DmitryScaletta@users.noreply.github.com>
Date: Fri, 19 Jan 2024 18:21:25 +0300
Subject: [PATCH 787/871] [ie/KukuluLive] Add extractor (#8877)

Closes #8865
Authored by: DmitryScaletta
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/kukululive.py  | 140 ++++++++++++++++++++++++++++++++
 2 files changed, 141 insertions(+)
 create mode 100644 yt_dlp/extractor/kukululive.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index abba5bfa2..aacb08fb6 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -905,6 +905,7 @@
 from .kth import KTHIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
+from .kukululive import KukuluLiveIE
 from .kusi import KUSIIE
 from .kuwo import (
     KuwoIE,
diff --git a/yt_dlp/extractor/kukululive.py b/yt_dlp/extractor/kukululive.py
new file mode 100644
index 000000000..86ab5d40e
--- /dev/null
+++ b/yt_dlp/extractor/kukululive.py
@@ -0,0 +1,140 @@
+import urllib.parse
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    filter_dict,
+    get_element_by_id,
+    int_or_none,
+    join_nonempty,
+    js_to_json,
+    qualities,
+    url_or_none,
+    urljoin,
+)
+from ..utils.traversal import traverse_obj
+
+
+class KukuluLiveIE(InfoExtractor):
+    _VALID_URL = r'https?://live\.erinn\.biz/live\.php\?h(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://live.erinn.biz/live.php?h675134569',
+        'md5': 'e380fa6a47fc703d91cea913ab44ec2e',
+        'info_dict': {
+            'id': '675134569',
+            'ext': 'mp4',
+            'title': 'プロセカ',
+            'description': 'テストも兼ねたプロセカ配信。',
+            'timestamp': 1702689148,
+            'upload_date': '20231216',
+            'thumbnail': r're:^https?://.*',
+        },
+    }, {
+        'url': 'https://live.erinn.biz/live.php?h102338092',
+        'md5': 'dcf5167a934b1c60333461e13a81a6e2',
+        'info_dict': {
+            'id': '102338092',
+            'ext': 'mp4',
+            'title': 'Among Usで遊びます！！',
+            'description': 'VTuberになりましたねんねこ㌨ですよろしくお願いします',
+            'timestamp': 1704603118,
+            'upload_date': '20240107',
+            'thumbnail': r're:^https?://.*',
+        },
+    }, {
+        'url': 'https://live.erinn.biz/live.php?h878049531',
+        'only_matching': True,
+    }]
+
+    def _get_quality_meta(self, video_id, desc, code, force_h264=None):
+        desc += ' (force_h264)' if force_h264 else ''
+        qs = self._download_webpage(
+            'https://live.erinn.biz/live.player.fplayer.php', video_id,
+            f'Downloading {desc} quality metadata', f'Unable to download {desc} quality metadata',
+            query=filter_dict({
+                'hash': video_id,
+                'action': f'get{code}liveByAjax',
+                'force_h264': force_h264,
+            }))
+        return urllib.parse.parse_qs(qs)
+
+    def _add_quality_formats(self, formats, quality_meta):
+        vcodec = traverse_obj(quality_meta, ('vcodec', 0, {str}))
+        quality = traverse_obj(quality_meta, ('now_quality', 0, {str}))
+        quality_priority = qualities(('low', 'h264', 'high'))(quality)
+        if traverse_obj(quality_meta, ('hlsaddr', 0, {url_or_none})):
+            formats.append({
+                'format_id': quality,
+                'url': quality_meta['hlsaddr'][0],
+                'ext': 'mp4',
+                'vcodec': vcodec,
+                'quality': quality_priority,
+            })
+        if traverse_obj(quality_meta, ('hlsaddr_audioonly', 0, {url_or_none})):
+            formats.append({
+                'format_id': join_nonempty(quality, 'audioonly'),
+                'url': quality_meta['hlsaddr_audioonly'][0],
+                'ext': 'm4a',
+                'vcodec': 'none',
+                'quality': quality_priority,
+            })
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        html = self._download_webpage(url, video_id)
+
+        if '>タイムシフトが見つかりませんでした。<' in html:
+            raise ExtractorError('This stream has expired', expected=True)
+
+        title = clean_html(
+            get_element_by_id('livetitle', html.replace('<SPAN', '<span').replace('SPAN>', 'span>')))
+        description = self._html_search_meta('Description', html)
+        thumbnail = self._html_search_meta(['og:image', 'twitter:image'], html)
+
+        if self._search_regex(r'(var\s+timeshift\s*=\s*false)', html, 'is livestream', default=False):
+            formats = []
+            for (desc, code) in [('high', 'Z'), ('low', 'ForceLow')]:
+                quality_meta = self._get_quality_meta(video_id, desc, code)
+                self._add_quality_formats(formats, quality_meta)
+                if desc == 'high' and traverse_obj(quality_meta, ('vcodec', 0)) == 'HEVC':
+                    self._add_quality_formats(
+                        formats, self._get_quality_meta(video_id, desc, code, force_h264='1'))
+
+            return {
+                'id': video_id,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'is_live': True,
+                'formats': formats,
+            }
+
+        # VOD extraction
+        player_html = self._download_webpage(
+            'https://live.erinn.biz/live.timeshift.fplayer.php', video_id,
+            'Downloading player html', 'Unable to download player html', query={'hash': video_id})
+
+        sources = traverse_obj(self._search_json(
+            r'var\s+fplayer_source\s*=', player_html, 'stream data', video_id,
+            contains_pattern=r'\[(?s:.+)\]', transform_source=js_to_json), lambda _, v: v['file'])
+
+        def entries(segments, playlist=True):
+            for i, segment in enumerate(segments, 1):
+                yield {
+                    'id': f'{video_id}_{i}' if playlist else video_id,
+                    'title': f'{title} (Part {i})' if playlist else title,
+                    'description': description,
+                    'timestamp': traverse_obj(segment, ('time_start', {int_or_none})),
+                    'thumbnail': thumbnail,
+                    'formats': [{
+                        'url': urljoin('https://live.erinn.biz', segment['file']),
+                        'ext': 'mp4',
+                        'protocol': 'm3u8_native',
+                    }],
+                }
+
+        if len(sources) == 1:
+            return next(entries(sources, playlist=False))
+
+        return self.playlist_result(entries(sources), video_id, title, description, multi_video=True)

From e641aab7a61df7406df60ebfe0c77bd5186b2b41 Mon Sep 17 00:00:00 2001
From: ArnauvGilotra <arnauvgilotra@gmail.com>
Date: Fri, 19 Jan 2024 20:57:34 +0530
Subject: [PATCH 788/871] [ie/AmadeusTV] Add extractor (#8744)

Closes #8155
Authored by: ArnauvGilotra
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/amadeustv.py   | 77 +++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 yt_dlp/extractor/amadeustv.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index aacb08fb6..8a7f62ccd 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -93,6 +93,7 @@
     AluraIE,
     AluraCourseIE
 )
+from .amadeustv import AmadeusTVIE
 from .amara import AmaraIE
 from .amcnetworks import AMCNetworksIE
 from .amazon import (
diff --git a/yt_dlp/extractor/amadeustv.py b/yt_dlp/extractor/amadeustv.py
new file mode 100644
index 000000000..2f5ca9137
--- /dev/null
+++ b/yt_dlp/extractor/amadeustv.py
@@ -0,0 +1,77 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class AmadeusTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?amadeus\.tv/library/(?P<id>[\da-f]+)'
+    _TESTS = [{
+        'url': 'http://www.amadeus.tv/library/65091a87ff85af59d9fc54c3',
+        'info_dict': {
+            'id': '5576678021301411311',
+            'ext': 'mp4',
+            'title': 'Jieon Park - 第五届珠海莫扎特国际青少年音乐周小提琴C组第三轮',
+            'thumbnail': 'http://1253584441.vod2.myqcloud.com/a0046a27vodtransbj1253584441/7db4af535576678021301411311/coverBySnapshot_10_0.jpg',
+            'duration': 1264.8,
+            'upload_date': '20230918',
+            'timestamp': 1695034800,
+            'display_id': '65091a87ff85af59d9fc54c3',
+            'view_count': int,
+            'description': 'md5:a0357b9c215489e2067cbae0b777bb95',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        nuxt_data = self._search_nuxt_data(webpage, display_id, traverse=('fetch', '0'))
+        video_id = traverse_obj(nuxt_data, ('item', 'video', {str}))
+
+        if not video_id:
+            raise ExtractorError('Unable to extract actual video ID')
+
+        video_data = self._download_json(
+            f'http://playvideo.qcloud.com/getplayinfo/v2/1253584441/{video_id}',
+            video_id, headers={'Referer': 'http://www.amadeus.tv/'})
+
+        formats = []
+        for video in traverse_obj(video_data, ('videoInfo', ('sourceVideo', ('transcodeList', ...)), {dict})):
+            if not url_or_none(video.get('url')):
+                continue
+            formats.append({
+                **traverse_obj(video, {
+                    'url': 'url',
+                    'format_id': ('definition', {lambda x: f'http-{x or "0"}'}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                    'filesize': (('totalSize', 'size'), {int_or_none}),
+                    'vcodec': ('videoStreamList', 0, 'codec'),
+                    'acodec': ('audioStreamList', 0, 'codec'),
+                    'fps': ('videoStreamList', 0, 'fps', {float_or_none}),
+                }, get_all=False),
+                'http_headers': {'Referer': 'http://www.amadeus.tv/'},
+            })
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'formats': formats,
+            **traverse_obj(video_data, {
+                'title': ('videoInfo', 'basicInfo', 'name', {str}),
+                'thumbnail': ('coverInfo', 'coverUrl', {url_or_none}),
+                'duration': ('videoInfo', 'sourceVideo', ('floatDuration', 'duration'), {float_or_none}),
+            }, get_all=False),
+            **traverse_obj(nuxt_data, ('item', {
+                'title': (('title', 'title_en', 'title_cn'), {str}),
+                'description': (('description', 'description_en', 'description_cn'), {str}),
+                'timestamp': ('date', {parse_iso8601}),
+                'view_count': ('view', {int_or_none}),
+            }), get_all=False),
+        }

From 8ab84650837e58046430c9f4b615c56a8886e071 Mon Sep 17 00:00:00 2001
From: ufukk <5383665+ufukk@users.noreply.github.com>
Date: Fri, 19 Jan 2024 18:38:39 +0300
Subject: [PATCH 789/871] [ie/TrtWorld] Add extractor (#8701)

Closes #8455
Authored by: ufukk
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/trtworld.py    | 101 ++++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 yt_dlp/extractor/trtworld.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 8a7f62ccd..489f638f4 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2011,6 +2011,7 @@
     TrovoChannelClipIE,
 )
 from .trtcocuk import TrtCocukVideoIE
+from .trtworld import TrtWorldIE
 from .trueid import TrueIDIE
 from .trunews import TruNewsIE
 from .truth import TruthIE
diff --git a/yt_dlp/extractor/trtworld.py b/yt_dlp/extractor/trtworld.py
new file mode 100644
index 000000000..dbb72a4fe
--- /dev/null
+++ b/yt_dlp/extractor/trtworld.py
@@ -0,0 +1,101 @@
+from .common import InfoExtractor
+from ..utils import ExtractorError, determine_ext, parse_iso8601, url_or_none
+from ..utils.traversal import traverse_obj
+
+
+class TrtWorldIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.trtworld\.com/video/[\w-]+/[\w-]+-(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://www.trtworld.com/video/news/turkiye-switches-to-sustainable-tourism-16067690',
+        'info_dict': {
+            'id': '16067690',
+            'ext': 'mp4',
+            'title': 'Türkiye switches to sustainable tourism',
+            'release_timestamp': 1701529569,
+            'release_date': '20231202',
+            'thumbnail': 'https://cdn-i.pr.trt.com.tr/trtworld/17647563_0-0-1920-1080.jpeg',
+            'description': 'md5:0a975c04257fb529c8f99c7b76a2cf12',
+        }
+    }, {
+        'url': 'https://www.trtworld.com/video/one-offs/frames-from-anatolia-recreating-a-james-bond-scene-in-istanbuls-grand-bazaar-14541780',
+        'info_dict': {
+            'id': '14541780',
+            'ext': 'mp4',
+            'title': 'Frames From Anatolia: Recreating a ‘James Bond’ Scene in Istanbul’s Grand Bazaar',
+            'release_timestamp': 1692440844,
+            'release_date': '20230819',
+            'thumbnail': 'https://cdn-i.pr.trt.com.tr/trtworld/16939810_0-0-1920-1080.jpeg',
+            'description': 'md5:4050e21570cc3c40b6c9badae800a94f',
+        }
+    }, {
+        'url': 'https://www.trtworld.com/video/the-newsmakers/can-sudan-find-peace-amidst-failed-transition-to-democracy-12904760',
+        'info_dict': {
+            'id': '12904760',
+            'ext': 'mp4',
+            'title': 'Can Sudan find peace amidst failed transition to democracy?',
+            'release_timestamp': 1681972747,
+            'release_date': '20230420',
+            'thumbnail': 'http://cdni0.trtworld.com/w768/q70/154214_NMYOUTUBETEMPLATE1_1681833018736.jpg'
+        }
+    }, {
+        'url': 'https://www.trtworld.com/video/africa-matters/locals-learning-to-cope-with-rising-tides-of-kenyas-great-lakes-16059545',
+        'info_dict': {
+            'id': 'zEns2dWl00w',
+            'ext': 'mp4',
+            'title': "Locals learning to cope with rising tides of Kenya's Great Lakes",
+            'thumbnail': 'https://i.ytimg.com/vi/zEns2dWl00w/maxresdefault.jpg',
+            'description': 'md5:3ad9d7c5234d752a4ead4340c79c6b8d',
+            'channel_id': 'UC7fWeaHhqgM4Ry-RMpM2YYw',
+            'channel_url': 'https://www.youtube.com/channel/UC7fWeaHhqgM4Ry-RMpM2YYw',
+            'duration': 210,
+            'view_count': int,
+            'age_limit': 0,
+            'webpage_url': 'https://www.youtube.com/watch?v=zEns2dWl00w',
+            'categories': ['News & Politics'],
+            'channel': 'TRT World',
+            'channel_follower_count': int,
+            'channel_is_verified': True,
+            'uploader': 'TRT World',
+            'uploader_id': '@trtworld',
+            'uploader_url': 'https://www.youtube.com/@trtworld',
+            'upload_date': '20231202',
+            'availability': 'public',
+            'comment_count': int,
+            'playable_in_embed': True,
+            'tags': [],
+            'live_status': 'not_live',
+            'like_count': int,
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        nuxtjs_data = self._search_nuxt_data(webpage, display_id)['videoData']['content']['platforms']
+        formats = []
+        for media_url in traverse_obj(nuxtjs_data, (
+                ('website', 'ott'), 'metadata', ('hls_url', 'url'), {url_or_none})):
+            # NB: Website sometimes serves mp4 files under `hls_url` key
+            if determine_ext(media_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(media_url, display_id, fatal=False))
+            else:
+                formats.append({
+                    'format_id': 'http',
+                    'url': media_url,
+                })
+        if not formats:
+            if youtube_id := traverse_obj(nuxtjs_data, ('youtube', 'metadata', 'youtubeId')):
+                return self.url_result(youtube_id, 'Youtube')
+            raise ExtractorError('No video found', expected=True)
+
+        return {
+            'id': display_id,
+            'formats': formats,
+            **traverse_obj(nuxtjs_data, (('website', 'ott'), {
+                'title': ('fields', 'title', 'text', {str}),
+                'description': ('fields', 'description', 'text', {str}),
+                'thumbnail': ('fields', 'thumbnail', 'url', {url_or_none}),
+                'release_timestamp': ('published', 'date', {parse_iso8601}),
+            }), get_all=False),
+        }

From 5154dc0a687528f995cde22b5ff63f82c740e98a Mon Sep 17 00:00:00 2001
From: alien-developers <154035958+alien-developers@users.noreply.github.com>
Date: Fri, 19 Jan 2024 21:18:45 +0530
Subject: [PATCH 790/871] [ie/JioSaavnSong] Support more bitrates (#8834)

Authored by: alien-developers, bashonly

Co-authored-by: bashonly <bashonly@protonmail.com>
---
 README.md                    |  3 +++
 yt_dlp/extractor/jiosaavn.py | 50 +++++++++++++++++++++++++++---------
 2 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 16947ce30..b6a79667c 100644
--- a/README.md
+++ b/README.md
@@ -1888,6 +1888,9 @@ #### nhkradirulive (NHK らじる★らじる LIVE)
 #### nflplusreplay
 * `type`: Type(s) of game replays to extract. Valid types are: `full_game`, `full_game_spanish`, `condensed_game` and `all_22`. You can use `all` to extract all available replay types, which is the default
 
+#### jiosaavn
+* `bitrate`: Audio bitrates to request. One or more of `16`, `32`, `64`, `128`, `320`. Default is `128,320`
+
 **Note**: These options may be changed/removed in the future without concern for backward compatibility
 
 <!-- MANPAGE: MOVE "INSTALLATION" SECTION HERE -->
diff --git a/yt_dlp/extractor/jiosaavn.py b/yt_dlp/extractor/jiosaavn.py
index 552b73f71..a59209835 100644
--- a/yt_dlp/extractor/jiosaavn.py
+++ b/yt_dlp/extractor/jiosaavn.py
@@ -1,5 +1,6 @@
 from .common import InfoExtractor
 from ..utils import (
+    int_or_none,
     js_to_json,
     url_or_none,
     urlencode_postdata,
@@ -20,39 +21,64 @@ class JioSaavnSongIE(JioSaavnBaseIE):
     _VALID_URL = r'https?://(?:www\.)?(?:jiosaavn\.com/song/[^/?#]+/|saavn\.com/s/song/(?:[^/?#]+/){3})(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.jiosaavn.com/song/leja-re/OQsEfQFVUXk',
-        'md5': '7b1f70de088ede3a152ea34aece4df42',
+        'md5': '3b84396d15ed9e083c3106f1fa589c04',
         'info_dict': {
             'id': 'OQsEfQFVUXk',
-            'ext': 'mp3',
+            'ext': 'mp4',
             'title': 'Leja Re',
             'album': 'Leja Re',
             'thumbnail': 'https://c.saavncdn.com/258/Leja-Re-Hindi-2018-20181124024539-500x500.jpg',
+            'duration': 205,
+            'view_count': int,
+            'release_year': 2018,
         },
     }, {
         'url': 'https://www.saavn.com/s/song/hindi/Saathiya/O-Humdum-Suniyo-Re/KAMiazoCblU',
         'only_matching': True,
     }]
 
+    _VALID_BITRATES = ('16', '32', '64', '128', '320')
+
     def _real_extract(self, url):
         audio_id = self._match_id(url)
+        extract_bitrates = self._configuration_arg('bitrate', ['128', '320'], ie_key='JioSaavn')
+        if invalid_bitrates := [br for br in extract_bitrates if br not in self._VALID_BITRATES]:
+            raise ValueError(
+                f'Invalid bitrate(s): {", ".join(invalid_bitrates)}. '
+                + f'Valid bitrates are: {", ".join(self._VALID_BITRATES)}')
+
         song_data = self._extract_initial_data(url, audio_id)['song']['song']
-        media_data = self._download_json(
-            'https://www.jiosaavn.com/api.php', audio_id, data=urlencode_postdata({
-                '__call': 'song.generateAuthToken',
-                '_format': 'json',
-                'bitrate': '128',
-                'url': song_data['encrypted_media_url'],
-            }))
+        formats = []
+        for bitrate in extract_bitrates:
+            media_data = self._download_json(
+                'https://www.jiosaavn.com/api.php', audio_id, f'Downloading format info for {bitrate}',
+                fatal=False, data=urlencode_postdata({
+                    '__call': 'song.generateAuthToken',
+                    '_format': 'json',
+                    'bitrate': bitrate,
+                    'url': song_data['encrypted_media_url'],
+                }))
+            if not media_data.get('auth_url'):
+                self.report_warning(f'Unable to extract format info for {bitrate}')
+                continue
+            formats.append({
+                'url': media_data['auth_url'],
+                'ext': media_data.get('type'),
+                'format_id': bitrate,
+                'abr': int(bitrate),
+                'vcodec': 'none',
+            })
 
         return {
             'id': audio_id,
-            'url': media_data['auth_url'],
-            'ext': media_data.get('type'),
-            'vcodec': 'none',
+            'formats': formats,
             **traverse_obj(song_data, {
                 'title': ('title', 'text'),
                 'album': ('album', 'text'),
                 'thumbnail': ('image', 0, {url_or_none}),
+                'duration': ('duration', {int_or_none}),
+                'view_count': ('play_count', {int_or_none}),
+                'release_year': ('year', {int_or_none}),
             }),
         }
 

From 12f042740550c06552819374e2251deb7a519bab Mon Sep 17 00:00:00 2001
From: Snack <korsnack@korsnack.kr>
Date: Sat, 20 Jan 2024 01:16:07 +0900
Subject: [PATCH 791/871] [ie/asobichannel] Add extractors (#8700)

Authored by: Snack-X
---
 yt_dlp/extractor/_extractors.py  |   1 +
 yt_dlp/extractor/asobichannel.py | 168 +++++++++++++++++++++++++++++++
 2 files changed, 169 insertions(+)
 create mode 100644 yt_dlp/extractor/asobichannel.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 489f638f4..eca45019e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -145,6 +145,7 @@
     ArteTVCategoryIE,
 )
 from .arnes import ArnesIE
+from .asobichannel import AsobiChannelIE, AsobiChannelTagURLIE
 from .atresplayer import AtresPlayerIE
 from .atscaleconf import AtScaleConfEventIE
 from .atvat import ATVAtIE
diff --git a/yt_dlp/extractor/asobichannel.py b/yt_dlp/extractor/asobichannel.py
new file mode 100644
index 000000000..e3479ede9
--- /dev/null
+++ b/yt_dlp/extractor/asobichannel.py
@@ -0,0 +1,168 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    merge_dicts,
+    parse_iso8601,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class AsobiChannelBaseIE(InfoExtractor):
+    _MICROCMS_HEADER = {'X-MICROCMS-API-KEY': 'qRaKehul9AHU8KtL0dnq1OCLKnFec6yrbcz3'}
+
+    def _extract_info(self, metadata):
+        return traverse_obj(metadata, {
+            'id': ('id', {str}),
+            'title': ('title', {str}),
+            'description': ('body', {clean_html}),
+            'thumbnail': ('contents', 'video_thumb', 'url', {url_or_none}),
+            'timestamp': ('publishedAt', {parse_iso8601}),
+            'modified_timestamp': ('updatedAt', {parse_iso8601}),
+            'channel': ('channel', 'name', {str}),
+            'channel_id': ('channel', 'id', {str}),
+        })
+
+
+class AsobiChannelIE(AsobiChannelBaseIE):
+    IE_NAME = 'asobichannel'
+    IE_DESC = 'ASOBI CHANNEL'
+
+    _VALID_URL = r'https?://asobichannel\.asobistore\.jp/watch/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://asobichannel.asobistore.jp/watch/1ypp48qd32p',
+        'md5': '39df74e872afe032c4eb27b89144fc92',
+        'info_dict': {
+            'id': '1ypp48qd32p',
+            'ext': 'mp4',
+            'title': 'アイドルマスター ミリオンライブ！ 765プロch 原っぱ通信 #1',
+            'description': 'md5:b930bd2199c9b2fd75951ce4aaa7efd2',
+            'thumbnail': 'https://images.microcms-assets.io/assets/d2420de4b9194e11beb164f99edb1f95/a8e6f84119f54eb9ab4ce16729239905/%E3%82%B5%E3%83%A0%E3%83%8D%20(1).png',
+            'timestamp': 1697098247,
+            'upload_date': '20231012',
+            'modified_timestamp': 1698381162,
+            'modified_date': '20231027',
+            'channel': 'アイドルマスター',
+            'channel_id': 'idolmaster',
+        },
+    }, {
+        'url': 'https://asobichannel.asobistore.jp/watch/redigiwnjzqj',
+        'md5': '229fa8fb5c591c75ce8c37a497f113f6',
+        'info_dict': {
+            'id': 'redigiwnjzqj',
+            'ext': 'mp4',
+            'title': '【おまけ放送】アイドルマスター ミリオンライブ！ 765プロch 原っぱ通信 #1',
+            'description': 'md5:7d9cd35fb54425a6967822bd564ea2d9',
+            'thumbnail': 'https://images.microcms-assets.io/assets/d2420de4b9194e11beb164f99edb1f95/20e5c1d6184242eebc2512a5dec59bf0/P1_%E5%8E%9F%E3%81%A3%E3%81%B1%E3%82%B5%E3%83%A0%E3%83%8D.png',
+            'modified_timestamp': 1697797125,
+            'modified_date': '20231020',
+            'timestamp': 1697261769,
+            'upload_date': '20231014',
+            'channel': 'アイドルマスター',
+            'channel_id': 'idolmaster',
+        },
+    }]
+
+    _survapi_header = None
+
+    def _real_initialize(self):
+        token = self._download_json(
+            'https://asobichannel-api.asobistore.jp/api/v1/vspf/token', None,
+            note='Retrieving API token')
+        self._survapi_header = {'Authorization': f'Bearer {token}'}
+
+    def _process_vod(self, video_id, metadata):
+        content_id = metadata['contents']['video_id']
+
+        vod_data = self._download_json(
+            f'https://survapi.channel.or.jp/proxy/v1/contents/{content_id}/get_by_cuid', video_id,
+            headers=self._survapi_header, note='Downloading vod data')
+
+        return {
+            'formats': self._extract_m3u8_formats(vod_data['ex_content']['streaming_url'], video_id),
+        }
+
+    def _process_live(self, video_id, metadata):
+        content_id = metadata['contents']['video_id']
+        event_data = self._download_json(
+            f'https://survapi.channel.or.jp/ex/events/{content_id}?embed=channel', video_id,
+            headers=self._survapi_header, note='Downloading event data')
+
+        player_type = traverse_obj(event_data, ('data', 'Player_type', {str}))
+        if player_type == 'poster':
+            self.raise_no_formats('Live event has not yet started', expected=True)
+            live_status = 'is_upcoming'
+            formats = []
+        elif player_type == 'player':
+            live_status = 'is_live'
+            formats = self._extract_m3u8_formats(
+                event_data['data']['Channel']['Custom_live_url'], video_id, live=True)
+        else:
+            raise ExtractorError('Unsupported player type {player_type!r}')
+
+        return {
+            'release_timestamp': traverse_obj(metadata, ('period', 'start', {parse_iso8601})),
+            'live_status': live_status,
+            'formats': formats,
+        }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        metadata = self._download_json(
+            f'https://channel.microcms.io/api/v1/media/{video_id}', video_id,
+            headers=self._MICROCMS_HEADER)
+
+        info = self._extract_info(metadata)
+
+        video_type = traverse_obj(metadata, ('contents', 'video_type', 0, {str}))
+        if video_type == 'VOD':
+            return merge_dicts(info, self._process_vod(video_id, metadata))
+        if video_type == 'LIVE':
+            return merge_dicts(info, self._process_live(video_id, metadata))
+
+        raise ExtractorError(f'Unexpected video type {video_type!r}')
+
+
+class AsobiChannelTagURLIE(AsobiChannelBaseIE):
+    IE_NAME = 'asobichannel:tag'
+    IE_DESC = 'ASOBI CHANNEL'
+
+    _VALID_URL = r'https?://asobichannel\.asobistore\.jp/tag/(?P<id>[a-z0-9-_]+)'
+    _TESTS = [{
+        'url': 'https://asobichannel.asobistore.jp/tag/bjhh-nbcja',
+        'info_dict': {
+            'id': 'bjhh-nbcja',
+            'title': 'アイドルマスター ミリオンライブ！ 765プロch 原っぱ通信',
+        },
+        'playlist_mincount': 16,
+    }, {
+        'url': 'https://asobichannel.asobistore.jp/tag/hvm5qw3c6od',
+        'info_dict': {
+            'id': 'hvm5qw3c6od',
+            'title': 'アイマスMOIW2023ラジオ',
+        },
+        'playlist_mincount': 13,
+    }]
+
+    def _real_extract(self, url):
+        tag_id = self._match_id(url)
+        webpage = self._download_webpage(url, tag_id)
+        title = traverse_obj(self._search_nextjs_data(
+            webpage, tag_id, fatal=False), ('props', 'pageProps', 'data', 'name', {str}))
+
+        media = self._download_json(
+            f'https://channel.microcms.io/api/v1/media?limit=999&filters=(tag[contains]{tag_id})',
+            tag_id, headers=self._MICROCMS_HEADER)
+
+        def entries():
+            for metadata in traverse_obj(media, ('contents', lambda _, v: v['id'])):
+                yield {
+                    '_type': 'url',
+                    'url': f'https://asobichannel.asobistore.jp/watch/{metadata["id"]}',
+                    'ie_key': AsobiChannelIE.ie_key(),
+                    **self._extract_info(metadata),
+                }
+
+        return self.playlist_result(entries(), tag_id, title)

From 1a36dbad712d359ec1c5b73d9bbbe562c03e9660 Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Fri, 19 Jan 2024 16:29:48 +0000
Subject: [PATCH 792/871] [ie/RinseFMArtistPlaylist] Add extractor (#8794)

Authored by: SirElderling
---
 yt_dlp/extractor/_extractors.py |  5 ++-
 yt_dlp/extractor/rinsefm.py     | 78 ++++++++++++++++++++++++++++-----
 2 files changed, 71 insertions(+), 12 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index eca45019e..3c94be8b4 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1604,7 +1604,10 @@
 from .reuters import ReutersIE
 from .reverbnation import ReverbNationIE
 from .rheinmaintv import RheinMainTVIE
-from .rinsefm import RinseFMIE
+from .rinsefm import (
+    RinseFMIE,
+    RinseFMArtistPlaylistIE,
+)
 from .rmcdecouverte import RMCDecouverteIE
 from .rockstargames import RockstarGamesIE
 from .rokfin import (
diff --git a/yt_dlp/extractor/rinsefm.py b/yt_dlp/extractor/rinsefm.py
index 760adf0eb..f87b895df 100644
--- a/yt_dlp/extractor/rinsefm.py
+++ b/yt_dlp/extractor/rinsefm.py
@@ -1,8 +1,34 @@
 from .common import InfoExtractor
-from ..utils import format_field, parse_iso8601
+from ..utils import (
+    MEDIA_EXTENSIONS,
+    determine_ext,
+    parse_iso8601,
+    traverse_obj,
+    url_or_none,
+)
 
 
-class RinseFMIE(InfoExtractor):
+class RinseFMBaseIE(InfoExtractor):
+    @staticmethod
+    def _parse_entry(entry):
+        return {
+            **traverse_obj(entry, {
+                'id': ('id', {str}),
+                'title': ('title', {str}),
+                'url': ('fileUrl', {url_or_none}),
+                'release_timestamp': ('episodeDate', {parse_iso8601}),
+                'thumbnail': ('featuredImage', 0, 'filename', {str},
+                              {lambda x: x and f'https://rinse.imgix.net/media/{x}'}),
+                'webpage_url': ('slug', {str},
+                                {lambda x: x and f'https://rinse.fm/episodes/{x}'}),
+            }),
+            'vcodec': 'none',
+            'extractor_key': RinseFMIE.ie_key(),
+            'extractor': RinseFMIE.IE_NAME,
+        }
+
+
+class RinseFMIE(RinseFMBaseIE):
     _VALID_URL = r'https?://(?:www\.)?rinse\.fm/episodes/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://rinse.fm/episodes/club-glow-15-12-2023-2000/',
@@ -22,12 +48,42 @@ def _real_extract(self, url):
         webpage = self._download_webpage(url, display_id)
         entry = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['entry']
 
-        return {
-            'id': entry['id'],
-            'title': entry.get('title'),
-            'url': entry['fileUrl'],
-            'vcodec': 'none',
-            'release_timestamp': parse_iso8601(entry.get('episodeDate')),
-            'thumbnail': format_field(
-                entry, [('featuredImage', 0, 'filename')], 'https://rinse.imgix.net/media/%s', default=None),
-        }
+        return self._parse_entry(entry)
+
+
+class RinseFMArtistPlaylistIE(RinseFMBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?rinse\.fm/shows/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://rinse.fm/shows/resources/',
+        'info_dict': {
+            'id': 'resources',
+            'title': '[re]sources',
+            'description': '[re]sources est un label parisien piloté par le DJ et producteur Tommy Kid.'
+        },
+        'playlist_mincount': 40
+    }, {
+        'url': 'https://rinse.fm/shows/ivy/',
+        'info_dict': {
+            'id': 'ivy',
+            'title': '[IVY]',
+            'description': 'A dedicated space for DNB/Turbo House and 4x4.'
+        },
+        'playlist_mincount': 7
+    }]
+
+    def _entries(self, data):
+        for episode in traverse_obj(data, (
+            'props', 'pageProps', 'episodes', lambda _, v: determine_ext(v['fileUrl']) in MEDIA_EXTENSIONS.audio)
+        ):
+            yield self._parse_entry(episode)
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        title = self._og_search_title(webpage) or self._html_search_meta('title', webpage)
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage)
+        data = self._search_nextjs_data(webpage, playlist_id)
+
+        return self.playlist_result(
+            self._entries(data), playlist_id, title, description=description)

From 5eb1458be4767385a9bf1d570ff08e46100cbaa2 Mon Sep 17 00:00:00 2001
From: Christopher Schreiner <git@infanf.com>
Date: Fri, 19 Jan 2024 17:38:21 +0100
Subject: [PATCH 793/871] [ie/adn] Add support for German site (#8708)

- Add extractor for seasons

Closes #6643, Closes #8945
Authored by: infanf
---
 yt_dlp/extractor/_extractors.py |   2 +-
 yt_dlp/extractor/adn.py         | 114 ++++++++++++++++++++++++++------
 2 files changed, 93 insertions(+), 23 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 3c94be8b4..b72b53fdd 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -47,7 +47,7 @@
     ACastChannelIE,
 )
 from .acfun import AcFunVideoIE, AcFunBangumiIE
-from .adn import ADNIE
+from .adn import ADNIE, ADNSeasonIE
 from .adobeconnect import AdobeConnectIE
 from .adobetv import (
     AdobeTVEmbedIE,
diff --git a/yt_dlp/extractor/adn.py b/yt_dlp/extractor/adn.py
index b59dbc850..ed23226a3 100644
--- a/yt_dlp/extractor/adn.py
+++ b/yt_dlp/extractor/adn.py
@@ -19,15 +19,35 @@
     long_to_bytes,
     pkcs1pad,
     strip_or_none,
+    str_or_none,
     try_get,
     unified_strdate,
     urlencode_postdata,
 )
+from ..utils.traversal import traverse_obj
 
 
-class ADNIE(InfoExtractor):
+class ADNBaseIE(InfoExtractor):
     IE_DESC = 'Animation Digital Network'
-    _VALID_URL = r'https?://(?:www\.)?(?:animation|anime)digitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
+    _NETRC_MACHINE = 'animationdigitalnetwork'
+    _BASE = 'animationdigitalnetwork.fr'
+    _API_BASE_URL = f'https://gw.api.{_BASE}/'
+    _PLAYER_BASE_URL = f'{_API_BASE_URL}player/'
+    _HEADERS = {}
+    _LOGIN_ERR_MESSAGE = 'Unable to log in'
+    _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
+    _POS_ALIGN_MAP = {
+        'start': 1,
+        'end': 3,
+    }
+    _LINE_ALIGN_MAP = {
+        'middle': 8,
+        'end': 4,
+    }
+
+
+class ADNIE(ADNBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:animation|anime)digitalnetwork\.(?P<lang>fr|de)/video/[^/?#]+/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://animationdigitalnetwork.fr/video/fruits-basket/9841-episode-1-a-ce-soir',
         'md5': '1c9ef066ceb302c86f80c2b371615261',
@@ -44,29 +64,35 @@ class ADNIE(InfoExtractor):
             'season_number': 1,
             'episode': 'À ce soir !',
             'episode_number': 1,
+            'thumbnail': str,
+            'season': 'Season 1',
         },
-        'skip': 'Only available in region (FR, ...)',
+        'skip': 'Only available in French and German speaking Europe',
     }, {
         'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
         'only_matching': True,
+    }, {
+        'url': 'https://animationdigitalnetwork.de/video/the-eminence-in-shadow/23550-folge-1',
+        'md5': '5c5651bf5791fa6fcd7906012b9d94e8',
+        'info_dict': {
+            'id': '23550',
+            'ext': 'mp4',
+            'episode_number': 1,
+            'duration': 1417,
+            'release_date': '20231004',
+            'series': 'The Eminence in Shadow',
+            'season_number': 2,
+            'episode': str,
+            'title': str,
+            'thumbnail': str,
+            'season': 'Season 2',
+            'comment_count': int,
+            'average_rating': float,
+            'description': str,
+        },
+        # 'skip': 'Only available in French and German speaking Europe',
     }]
 
-    _NETRC_MACHINE = 'animationdigitalnetwork'
-    _BASE = 'animationdigitalnetwork.fr'
-    _API_BASE_URL = 'https://gw.api.' + _BASE + '/'
-    _PLAYER_BASE_URL = _API_BASE_URL + 'player/'
-    _HEADERS = {}
-    _LOGIN_ERR_MESSAGE = 'Unable to log in'
-    _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
-    _POS_ALIGN_MAP = {
-        'start': 1,
-        'end': 3,
-    }
-    _LINE_ALIGN_MAP = {
-        'middle': 8,
-        'end': 4,
-    }
-
     def _get_subtitles(self, sub_url, video_id):
         if not sub_url:
             return None
@@ -116,6 +142,8 @@ def _get_subtitles(self, sub_url, video_id):
 
             if sub_lang == 'vostf':
                 sub_lang = 'fr'
+            elif sub_lang == 'vostde':
+                sub_lang = 'de'
             subtitles.setdefault(sub_lang, []).extend([{
                 'ext': 'json',
                 'data': json.dumps(sub),
@@ -147,7 +175,7 @@ def _perform_login(self, username, password):
             self.report_warning(message or self._LOGIN_ERR_MESSAGE)
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        lang, video_id = self._match_valid_url(url).group('lang', 'id')
         video_base_url = self._PLAYER_BASE_URL + 'video/%s/' % video_id
         player = self._download_json(
             video_base_url + 'configuration', video_id,
@@ -162,7 +190,7 @@ def _real_extract(self, url):
         token = self._download_json(
             user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),
             video_id, 'Downloading access token', headers={
-                'x-player-refresh-token': user['refreshToken']
+                'X-Player-Refresh-Token': user['refreshToken'],
             }, data=b'')['token']
 
         links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
@@ -184,7 +212,9 @@ def _real_extract(self, url):
             try:
                 links_data = self._download_json(
                     links_url, video_id, 'Downloading links JSON metadata', headers={
-                        'X-Player-Token': authorization
+                        'X-Player-Token': authorization,
+                        'X-Target-Distribution': lang,
+                        **self._HEADERS
                     }, query={
                         'freeWithAds': 'true',
                         'adaptive': 'false',
@@ -232,6 +262,9 @@ def _real_extract(self, url):
                 if format_id == 'vf':
                     for f in m3u8_formats:
                         f['language'] = 'fr'
+                elif format_id == 'vde':
+                    for f in m3u8_formats:
+                        f['language'] = 'de'
                 formats.extend(m3u8_formats)
 
         video = (self._download_json(
@@ -255,3 +288,40 @@ def _real_extract(self, url):
             'average_rating': float_or_none(video.get('rating') or metas.get('rating')),
             'comment_count': int_or_none(video.get('commentsCount')),
         }
+
+
+class ADNSeasonIE(ADNBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:animation|anime)digitalnetwork\.(?P<lang>fr|de)/video/(?P<id>[^/?#]+)/?(?:$|[#?])'
+    _TESTS = [{
+        'url': 'https://animationdigitalnetwork.fr/video/tokyo-mew-mew-new',
+        'playlist_count': 12,
+        'info_dict': {
+            'id': '911',
+            'title': 'Tokyo Mew Mew New',
+        },
+        # 'skip': 'Only available in French end German speaking Europe',
+    }]
+
+    def _real_extract(self, url):
+        lang, video_show_slug = self._match_valid_url(url).group('lang', 'id')
+        show = self._download_json(
+            f'{self._API_BASE_URL}show/{video_show_slug}/', video_show_slug,
+            'Downloading show JSON metadata', headers=self._HEADERS)['show']
+        show_id = str(show['id'])
+        episodes = self._download_json(
+            f'{self._API_BASE_URL}video/show/{show_id}', video_show_slug,
+            'Downloading episode list', headers={
+                'X-Target-Distribution': lang,
+                **self._HEADERS
+            }, query={
+                'order': 'asc',
+                'limit': '-1',
+            })
+
+        def entries():
+            for episode_id in traverse_obj(episodes, ('videos', ..., 'id', {str_or_none})):
+                yield self.url_result(
+                    f'https://animationdigitalnetwork.{lang}/video/{video_show_slug}/{episode_id}',
+                    ADNIE, episode_id)
+
+        return self.playlist_result(entries(), show_id, show.get('title'))

From 4a07a455bbf7acf87550053bbba949c828e350ba Mon Sep 17 00:00:00 2001
From: Alexey Neyman <stilor@att.net>
Date: Fri, 19 Jan 2024 08:49:15 -0800
Subject: [PATCH 794/871] [ie/GoPro] Fix extractor (#9019)

Authored by: stilor
---
 yt_dlp/extractor/gopro.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/gopro.py b/yt_dlp/extractor/gopro.py
index ae965374c..ec1595bc5 100644
--- a/yt_dlp/extractor/gopro.py
+++ b/yt_dlp/extractor/gopro.py
@@ -57,8 +57,8 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        metadata = self._parse_json(
-            self._html_search_regex(r'window\.__reflectData\s*=\s*([^;]+)', webpage, 'metadata'), video_id)
+        metadata = self._search_json(
+            r'window\.__reflectData\s*=', webpage, 'metadata', video_id)
 
         video_info = metadata['collectionMedia'][0]
         media_data = self._download_json(
@@ -99,7 +99,7 @@ def _real_extract(self, url):
             'duration': int_or_none(
                 video_info.get('source_duration')),
             'artist': str_or_none(
-                video_info.get('music_track_artist')),
+                video_info.get('music_track_artist')) or None,
             'track': str_or_none(
-                video_info.get('music_track_name')),
+                video_info.get('music_track_name')) or None,
         }

From 1713c882730a928ac344c099874d2093fc2c8b51 Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Fri, 19 Jan 2024 20:11:00 +0000
Subject: [PATCH 795/871] [ie/bilibili] Add referer header and fix metadata
 extraction (#8832)

Closes #6640
Authored by: SirElderling
---
 yt_dlp/extractor/bilibili.py | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 5475b3650..cd7df69ef 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -1622,6 +1622,7 @@ def _real_extract(self, url):
 class BiliIntlBaseIE(InfoExtractor):
     _API_URL = 'https://api.bilibili.tv/intl/gateway'
     _NETRC_MACHINE = 'biliintl'
+    _HEADERS = {'Referer': 'https://www.bilibili.com/'}
 
     def _call_api(self, endpoint, *args, **kwargs):
         json = self._download_json(self._API_URL + endpoint, *args, **kwargs)
@@ -1732,7 +1733,9 @@ def _get_formats(self, *, ep_id=None, aid=None):
     def _parse_video_metadata(self, video_data):
         return {
             'title': video_data.get('title_display') or video_data.get('title'),
+            'description': video_data.get('desc'),
             'thumbnail': video_data.get('cover'),
+            'timestamp': unified_timestamp(video_data.get('formatted_pub_date')),
             'episode_number': int_or_none(self._search_regex(
                 r'^E(\d+)(?:$| - )', video_data.get('title_display') or '', 'episode number', default=None)),
         }
@@ -1829,17 +1832,6 @@ class BiliIntlIE(BiliIntlBaseIE):
             'episode_number': 140,
         },
         'skip': 'According to the copyright owner\'s request, you may only watch the video after you log in.'
-    }, {
-        'url': 'https://www.bilibili.tv/en/video/2041863208',
-        'info_dict': {
-            'id': '2041863208',
-            'ext': 'mp4',
-            'timestamp': 1670874843,
-            'description': 'Scheduled for April 2023.\nStudio: ufotable',
-            'thumbnail': r're:https?://pic[-\.]bstarstatic.+/ugc/.+\.jpg$',
-            'upload_date': '20221212',
-            'title': 'Kimetsu no Yaiba Season 3 Official Trailer - Bstation',
-        },
     }, {
         # episode comment extraction
         'url': 'https://www.bilibili.tv/en/play/34580/340317',
@@ -1880,9 +1872,9 @@ class BiliIntlIE(BiliIntlBaseIE):
             'description': 'md5:693b6f3967fb4e7e7764ea817857c33a',
             'timestamp': 1667891924,
             'upload_date': '20221108',
-            'title': 'That Time I Got Reincarnated as a Slime: Scarlet Bond - Official Trailer 3| AnimeStan - Bstation',
+            'title': 'That Time I Got Reincarnated as a Slime: Scarlet Bond - Official Trailer 3| AnimeStan',
             'comment_count': int,
-            'thumbnail': 'https://pic.bstarstatic.com/ugc/f6c363659efd2eabe5683fbb906b1582.jpg',
+            'thumbnail': r're:https://pic\.bstarstatic\.(?:com|net)/ugc/f6c363659efd2eabe5683fbb906b1582\.jpg',
         },
         'params': {
             'getcomments': True
@@ -1945,10 +1937,12 @@ def _extract_video_metadata(self, url, video_id, season_id):
 
         # XXX: webpage metadata may not accurate, it just used to not crash when video_data not found
         return merge_dicts(
-            self._parse_video_metadata(video_data), self._search_json_ld(webpage, video_id, fatal=False), {
-                'title': self._html_search_meta('og:title', webpage),
-                'description': self._html_search_meta('og:description', webpage)
-            })
+            self._parse_video_metadata(video_data), {
+                'title': get_element_by_class(
+                    'bstar-meta__title', webpage) or self._html_search_meta('og:title', webpage),
+                'description': get_element_by_class(
+                    'bstar-meta__desc', webpage) or self._html_search_meta('og:description'),
+            }, self._search_json_ld(webpage, video_id, default={}))
 
     def _get_comments_reply(self, root_id, next_id=0, display_id=None):
         comment_api_raw_data = self._download_json(
@@ -2036,7 +2030,8 @@ def _real_extract(self, url):
             'formats': self._get_formats(ep_id=ep_id, aid=aid),
             'subtitles': self.extract_subtitles(ep_id=ep_id, aid=aid),
             'chapters': chapters,
-            '__post_extractor': self.extract_comments(video_id, ep_id)
+            '__post_extractor': self.extract_comments(video_id, ep_id),
+            'http_headers': self._HEADERS,
         }
 
 

From 4310b6650eeb5630295f4591b37720877878c57a Mon Sep 17 00:00:00 2001
From: divStar <divStar@gmail.com>
Date: Fri, 19 Jan 2024 21:27:16 +0100
Subject: [PATCH 796/871] [ie/getcourseru] Add extractors (#8873)

Authored by: divStar, seproDev

Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/getcourseru.py | 179 ++++++++++++++++++++++++++++++++
 2 files changed, 183 insertions(+)
 create mode 100644 yt_dlp/extractor/getcourseru.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index b72b53fdd..3d5c3eb60 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -687,6 +687,10 @@
     GeniusIE,
     GeniusLyricsIE,
 )
+from .getcourseru import (
+    GetCourseRuPlayerIE,
+    GetCourseRuIE
+)
 from .gettr import (
     GettrIE,
     GettrStreamingIE,
diff --git a/yt_dlp/extractor/getcourseru.py b/yt_dlp/extractor/getcourseru.py
new file mode 100644
index 000000000..6fdbcd736
--- /dev/null
+++ b/yt_dlp/extractor/getcourseru.py
@@ -0,0 +1,179 @@
+import re
+import time
+import urllib.parse
+
+from .common import InfoExtractor
+from ..utils import ExtractorError, int_or_none, url_or_none, urlencode_postdata
+from ..utils.traversal import traverse_obj
+
+
+class GetCourseRuPlayerIE(InfoExtractor):
+    _VALID_URL = r'https?://player02\.getcourse\.ru/sign-player/?\?(?:[^#]+&)?json=[^#&]+'
+    _EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL}[^\'"]*)']
+    _TESTS = [{
+        'url': 'http://player02.getcourse.ru/sign-player/?json=eyJ2aWRlb19oYXNoIjoiMTkwYmRmOTNmMWIyOTczNTMwOTg1M2E3YTE5ZTI0YjMiLCJ1c2VyX2lkIjozNTk1MjUxODMsInN1Yl9sb2dpbl91c2VyX2lkIjpudWxsLCJsZXNzb25faWQiOm51bGwsImlwIjoiNDYuMTQyLjE4Mi4yNDciLCJnY19ob3N0IjoiYWNhZGVteW1lbC5vbmxpbmUiLCJ0aW1lIjoxNzA1NDQ5NjQyLCJwYXlsb2FkIjoidV8zNTk1MjUxODMiLCJ1aV9sYW5ndWFnZSI6InJ1IiwiaXNfaGF2ZV9jdXN0b21fc3R5bGUiOnRydWV9&s=354ad2c993d95d5ac629e3133d6cefea&vh-static-feature=zigzag',
+        'info_dict': {
+            'id': '513573381',
+            'title': '190bdf93f1b29735309853a7a19e24b3',
+            'ext': 'mp4',
+            'thumbnail': 'https://preview-htz.kinescopecdn.net/preview/190bdf93f1b29735309853a7a19e24b3/preview.jpg?version=1702370546&host=vh-80',
+            'duration': 1693
+        },
+        'skip': 'JWT expired',
+    }]
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, None, 'Downloading player page')
+        window_configs = self._search_json(
+            r'window\.configs\s*=', webpage, 'config', None)
+        video_id = str(window_configs['gcFileId'])
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            window_configs['masterPlaylistUrl'], video_id)
+
+        return {
+            **traverse_obj(window_configs, {
+                'title': ('videoHash', {str}),
+                'thumbnail': ('previewUrl', {url_or_none}),
+                'duration': ('videoDuration', {int_or_none}),
+            }),
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles
+        }
+
+
+class GetCourseRuIE(InfoExtractor):
+    _NETRC_MACHINE = 'getcourseru'
+    _DOMAINS = [
+        'academymel.online',
+        'marafon.mani-beauty.com',
+        'on.psbook.ru'
+    ]
+    _BASE_URL_RE = rf'https?://(?:(?!player02\.)[^.]+\.getcourse\.(?:ru|io)|{"|".join(map(re.escape, _DOMAINS))})'
+    _VALID_URL = [
+        rf'{_BASE_URL_RE}/(?!pl/|teach/)(?P<id>[^?#]+)',
+        rf'{_BASE_URL_RE}/(:?pl/)?teach/control/lesson/view\?(?:[^#]+&)?id=(?P<id>\d+)',
+    ]
+    _TESTS = [{
+        'url': 'http://academymel.online/3video_1',
+        'info_dict': {
+            'id': '3059742',
+            'display_id': '3video_1',
+            'title': 'Промоуроки Академии МЕЛ',
+        },
+        'playlist_count': 1,
+        'playlist': [{
+            'info_dict': {
+                'id': '513573381',
+                'ext': 'mp4',
+                'title': 'Промоуроки Академии МЕЛ',
+                'thumbnail': 'https://preview-htz.kinescopecdn.net/preview/190bdf93f1b29735309853a7a19e24b3/preview.jpg?version=1702370546&host=vh-80',
+                'duration': 1693
+            },
+        }]
+    }, {
+        'url': 'https://academymel.getcourse.ru/3video_1',
+        'info_dict': {
+            'id': '3059742',
+            'display_id': '3video_1',
+            'title': 'Промоуроки Академии МЕЛ',
+        },
+        'playlist_count': 1,
+        'playlist': [{
+            'info_dict': {
+                'id': '513573381',
+                'ext': 'mp4',
+                'title': 'Промоуроки Академии МЕЛ',
+                'thumbnail': 'https://preview-htz.kinescopecdn.net/preview/190bdf93f1b29735309853a7a19e24b3/preview.jpg?version=1702370546&host=vh-80',
+                'duration': 1693
+            },
+        }]
+    }, {
+        'url': 'https://academymel.getcourse.ru/pl/teach/control/lesson/view?id=319141781&editMode=0',
+        'info_dict': {
+            'id': '319141781',
+            'title': '1. Разминка у стены',
+        },
+        'playlist_count': 1,
+        'playlist': [{
+            'info_dict': {
+                'id': '4919601',
+                'ext': 'mp4',
+                'title': '1. Разминка у стены',
+                'thumbnail': 'https://preview-htz.vhcdn.com/preview/5a521788e7dc25b4f70c3dff6512d90e/preview.jpg?version=1703223532&host=vh-81',
+                'duration': 704
+            },
+        }],
+        'skip': 'paid lesson'
+    }, {
+        'url': 'https://manibeauty.getcourse.ru/pl/teach/control/lesson/view?id=272499894',
+        'info_dict': {
+            'id': '272499894',
+            'title': 'Мотивация к тренировкам',
+        },
+        'playlist_count': 1,
+        'playlist': [{
+            'info_dict': {
+                'id': '447479687',
+                'ext': 'mp4',
+                'title': 'Мотивация к тренировкам',
+                'thumbnail': 'https://preview-htz.vhcdn.com/preview/70ed5b9f489dd03b4aff55bfdff71a26/preview.jpg?version=1685115787&host=vh-71',
+                'duration': 30
+            },
+        }],
+        'skip': 'paid lesson'
+    }, {
+        'url': 'https://gaismasmandalas.getcourse.io/ATLAUTSEVBUT',
+        'only_matching': True,
+    }]
+
+    _LOGIN_URL_PATH = '/cms/system/login'
+
+    def _login(self, hostname, username, password):
+        if self._get_cookies(f'https://{hostname}').get('PHPSESSID5'):
+            return
+        login_url = f'https://{hostname}{self._LOGIN_URL_PATH}'
+        webpage = self._download_webpage(login_url, None)
+
+        self._request_webpage(
+            login_url, None, 'Logging in', 'Failed to log in',
+            data=urlencode_postdata({
+                'action': 'processXdget',
+                'xdgetId': self._html_search_regex(
+                    r'<form[^>]+\bclass="[^"]*\bstate-login[^"]*"[^>]+\bdata-xdget-id="([^"]+)"',
+                    webpage, 'xdgetId'),
+                'params[action]': 'login',
+                'params[url]': login_url,
+                'params[object_type]': 'cms_page',
+                'params[object_id]': -1,
+                'params[email]': username,
+                'params[password]': password,
+                'requestTime': int(time.time()),
+                'requestSimpleSign': self._html_search_regex(
+                    r'window.requestSimpleSign\s*=\s*"([\da-f]+)"', webpage, 'simple sign'),
+            }))
+
+    def _real_extract(self, url):
+        hostname = urllib.parse.urlparse(url).hostname
+        username, password = self._get_login_info(netrc_machine=hostname)
+        if username:
+            self._login(hostname, username, password)
+
+        display_id = self._match_id(url)
+        # NB: 404 is returned due to yt-dlp not properly following redirects #9020
+        webpage, urlh = self._download_webpage_handle(url, display_id, expected_status=404)
+        if self._LOGIN_URL_PATH in urlh.url or urlh.status == 404:
+            raise ExtractorError(
+                f'This video is only available for registered users. {self._login_hint("any", netrc=hostname)}',
+                expected=True)
+
+        playlist_id = self._search_regex(
+            r'window\.(?:lessonId|gcsObjectId)\s*=\s*(\d+)', webpage, 'playlist id', default=display_id)
+        title = self._og_search_title(webpage) or self._html_extract_title(webpage)
+
+        return self.playlist_from_matches(
+            re.findall(GetCourseRuPlayerIE._EMBED_REGEX[0], webpage),
+            playlist_id, title, display_id=display_id, ie=GetCourseRuPlayerIE, video_kwargs={
+                'url_transparent': True,
+                'title': title,
+            })

From 50e06e21a68e336198198bda332b8e7d2314f201 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Sat, 20 Jan 2024 05:31:06 +0900
Subject: [PATCH 797/871] [ie/MLBArticle] Fix extractor (#9021)

Closes #8682
Authored by: HobbyistDev
---
 yt_dlp/extractor/mlb.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/yt_dlp/extractor/mlb.py b/yt_dlp/extractor/mlb.py
index 72057dc97..d715b9789 100644
--- a/yt_dlp/extractor/mlb.py
+++ b/yt_dlp/extractor/mlb.py
@@ -355,11 +355,11 @@ class MLBArticleIE(InfoExtractor):
         'info_dict': {
             'id': '36db7394-343c-4ea3-b8ca-ead2e61bca9a',
             'title': 'Machado\'s grab draws hilarious irate reaction',
-            'modified_timestamp': 1650130737,
+            'modified_timestamp': 1675888370,
             'description': 'md5:a19d4eb0487b2cb304e9a176f6b67676',
-            'modified_date': '20220416',
+            'modified_date': '20230208',
         },
-        'playlist_count': 2,
+        'playlist_mincount': 2,
     }]
 
     def _real_extract(self, url):
@@ -367,15 +367,13 @@ def _real_extract(self, url):
         webpage = self._download_webpage(url, display_id)
         apollo_cache_json = self._search_json(r'window\.initState\s*=', webpage, 'window.initState', display_id)['apolloCache']
 
-        content_data_id = traverse_obj(
-            apollo_cache_json, ('ROOT_QUERY', lambda k, _: k.startswith('getForgeContent'), 'id'), get_all=False)
-
-        content_real_info = apollo_cache_json[content_data_id]
+        content_real_info = traverse_obj(
+            apollo_cache_json, ('ROOT_QUERY', lambda k, _: k.startswith('getArticle')), get_all=False)
 
         return self.playlist_from_matches(
-            traverse_obj(content_real_info, ('parts', lambda _, v: v['typename'] == 'Video', 'id')),
-            getter=lambda x: f'https://www.mlb.com/video/{apollo_cache_json[x]["slug"]}',
-            ie=MLBVideoIE, playlist_id=content_real_info.get('_translationId'),
+            traverse_obj(content_real_info, ('parts', lambda _, v: v['__typename'] == 'Video' or v['type'] == 'video')),
+            getter=lambda x: f'https://www.mlb.com/video/{x["slug"]}',
+            ie=MLBVideoIE, playlist_id=content_real_info.get('translationId'),
             title=self._html_search_meta('og:title', webpage),
             description=content_real_info.get('summary'),
             modified_timestamp=parse_iso8601(content_real_info.get('lastUpdatedDate')))

From 69d31914952dd33082ac7019c6f76b43c45b9d06 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 20 Jan 2024 10:39:49 +1300
Subject: [PATCH 798/871] [test] Skip source address tests if the address
 cannot be bound to (#8900)

Fixes https://github.com/yt-dlp/yt-dlp/issues/8890

Authored by: coletdjnz
---
 test/helper.py          | 7 ++++++-
 test/test_networking.py | 5 ++++-
 test/test_socks.py      | 4 +++-
 test/test_websockets.py | 3 +++
 4 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index e5ace8fe2..4aca47025 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -10,7 +10,7 @@
 import yt_dlp.extractor
 from yt_dlp import YoutubeDL
 from yt_dlp.compat import compat_os_name
-from yt_dlp.utils import preferredencoding, try_call, write_string
+from yt_dlp.utils import preferredencoding, try_call, write_string, find_available_port
 
 if 'pytest' in sys.modules:
     import pytest
@@ -329,3 +329,8 @@ def http_server_port(httpd):
     else:
         sock = httpd.socket
     return sock.getsockname()[1]
+
+
+def verify_address_availability(address):
+    if find_available_port(address) is None:
+        pytest.skip(f'Unable to bind to source address {address} (address may not exist)')
diff --git a/test/test_networking.py b/test/test_networking.py
index dc60ca699..62325aa8e 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -26,7 +26,7 @@
 from email.message import Message
 from http.cookiejar import CookieJar
 
-from test.helper import FakeYDL, http_server_port
+from test.helper import FakeYDL, http_server_port, verify_address_availability
 from yt_dlp.cookies import YoutubeDLCookieJar
 from yt_dlp.dependencies import brotli, requests, urllib3
 from yt_dlp.networking import (
@@ -538,6 +538,9 @@ def test_timeout(self, handler):
     @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
     def test_source_address(self, handler):
         source_address = f'127.0.0.{random.randint(5, 255)}'
+        # on some systems these loopback addresses we need for testing may not be available
+        # see: https://github.com/yt-dlp/yt-dlp/issues/8890
+        verify_address_availability(source_address)
         with handler(source_address=source_address) as rh:
             data = validate_and_send(
                 rh, Request(f'http://127.0.0.1:{self.http_port}/source_address')).read().decode()
diff --git a/test/test_socks.py b/test/test_socks.py
index 71f783e13..cb22b61dc 100644
--- a/test/test_socks.py
+++ b/test/test_socks.py
@@ -25,7 +25,7 @@
     ThreadingTCPServer,
 )
 
-from test.helper import http_server_port
+from test.helper import http_server_port, verify_address_availability
 from yt_dlp.networking import Request
 from yt_dlp.networking.exceptions import ProxyError, TransportError
 from yt_dlp.socks import (
@@ -326,6 +326,7 @@ def test_socks4a_domain_target(self, handler, ctx):
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
+            verify_address_availability(source_address)
             with handler(proxies={'all': f'socks4://{server_address}'},
                          source_address=source_address) as rh:
                 response = ctx.socks_info_request(rh)
@@ -441,6 +442,7 @@ def test_ipv6_socks5_proxy(self, handler, ctx):
     def test_ipv4_client_source_address(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
+            verify_address_availability(source_address)
             with handler(proxies={'all': f'socks5://{server_address}'}, source_address=source_address) as rh:
                 response = ctx.socks_info_request(rh)
                 assert response['client_address'][0] == source_address
diff --git a/test/test_websockets.py b/test/test_websockets.py
index af6142ea3..91bac3442 100644
--- a/test/test_websockets.py
+++ b/test/test_websockets.py
@@ -6,6 +6,8 @@
 
 import pytest
 
+from test.helper import verify_address_availability
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import http.client
@@ -227,6 +229,7 @@ def test_cookies(self, handler):
     @pytest.mark.parametrize('handler', ['Websockets'], indirect=True)
     def test_source_address(self, handler):
         source_address = f'127.0.0.{random.randint(5, 255)}'
+        verify_address_availability(source_address)
         with handler(source_address=source_address) as rh:
             ws = validate_and_send(rh, Request(self.ws_base_url))
             ws.send('source_address')

From 811d298b231cfa29e75c321b23a91d1c2b17602c Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sat, 20 Jan 2024 15:26:50 +1300
Subject: [PATCH 799/871] [networking] Remove `_CompatHTTPError` (#8871)

Use `yt_dlp.networking.exceptions.HTTPError`.
`_CompatHTTPError` was to help with transition to the networking framework.

Authored by: coletdjnz
---
 test/test_networking_utils.py   |  82 ++--------------------
 yt_dlp/YoutubeDL.py             |   3 -
 yt_dlp/compat/_legacy.py        |   4 +-
 yt_dlp/networking/exceptions.py | 116 +-------------------------------
 4 files changed, 7 insertions(+), 198 deletions(-)

diff --git a/test/test_networking_utils.py b/test/test_networking_utils.py
index 419aae1e4..b7b71430e 100644
--- a/test/test_networking_utils.py
+++ b/test/test_networking_utils.py
@@ -8,13 +8,9 @@
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-import contextlib
 import io
-import platform
 import random
 import ssl
-import urllib.error
-import warnings
 
 from yt_dlp.cookies import YoutubeDLCookieJar
 from yt_dlp.dependencies import certifi
@@ -30,7 +26,6 @@
 from yt_dlp.networking.exceptions import (
     HTTPError,
     IncompleteRead,
-    _CompatHTTPError,
 )
 from yt_dlp.socks import ProxyType
 from yt_dlp.utils.networking import HTTPHeaderDict
@@ -179,11 +174,10 @@ class TestNetworkingExceptions:
     def create_response(status):
         return Response(fp=io.BytesIO(b'test'), url='http://example.com', headers={'tesT': 'test'}, status=status)
 
-    @pytest.mark.parametrize('http_error_class', [HTTPError, lambda r: _CompatHTTPError(HTTPError(r))])
-    def test_http_error(self, http_error_class):
+    def test_http_error(self):
 
         response = self.create_response(403)
-        error = http_error_class(response)
+        error = HTTPError(response)
 
         assert error.status == 403
         assert str(error) == error.msg == 'HTTP Error 403: Forbidden'
@@ -194,80 +188,12 @@ def test_http_error(self, http_error_class):
         assert data == b'test'
         assert repr(error) == '<HTTPError 403: Forbidden>'
 
-    @pytest.mark.parametrize('http_error_class', [HTTPError, lambda *args, **kwargs: _CompatHTTPError(HTTPError(*args, **kwargs))])
-    def test_redirect_http_error(self, http_error_class):
+    def test_redirect_http_error(self):
         response = self.create_response(301)
-        error = http_error_class(response, redirect_loop=True)
+        error = HTTPError(response, redirect_loop=True)
         assert str(error) == error.msg == 'HTTP Error 301: Moved Permanently (redirect loop detected)'
         assert error.reason == 'Moved Permanently'
 
-    def test_compat_http_error(self):
-        response = self.create_response(403)
-        error = _CompatHTTPError(HTTPError(response))
-        assert isinstance(error, HTTPError)
-        assert isinstance(error, urllib.error.HTTPError)
-
-        @contextlib.contextmanager
-        def raises_deprecation_warning():
-            with warnings.catch_warnings(record=True) as w:
-                warnings.simplefilter('always')
-                yield
-
-                if len(w) == 0:
-                    pytest.fail('Did not raise DeprecationWarning')
-                if len(w) > 1:
-                    pytest.fail(f'Raised multiple warnings: {w}')
-
-                if not issubclass(w[-1].category, DeprecationWarning):
-                    pytest.fail(f'Expected DeprecationWarning, got {w[-1].category}')
-                w.clear()
-
-        with raises_deprecation_warning():
-            assert error.code == 403
-
-        with raises_deprecation_warning():
-            assert error.getcode() == 403
-
-        with raises_deprecation_warning():
-            assert error.hdrs is error.response.headers
-
-        with raises_deprecation_warning():
-            assert error.info() is error.response.headers
-
-        with raises_deprecation_warning():
-            assert error.headers is error.response.headers
-
-        with raises_deprecation_warning():
-            assert error.filename == error.response.url
-
-        with raises_deprecation_warning():
-            assert error.url == error.response.url
-
-        with raises_deprecation_warning():
-            assert error.geturl() == error.response.url
-
-        # Passthrough file operations
-        with raises_deprecation_warning():
-            assert error.read() == b'test'
-
-        with raises_deprecation_warning():
-            assert not error.closed
-
-        with raises_deprecation_warning():
-            # Technically Response operations are also passed through, which should not be used.
-            assert error.get_header('test') == 'test'
-
-        # Should not raise a warning
-        error.close()
-
-    @pytest.mark.skipif(
-        platform.python_implementation() == 'PyPy', reason='garbage collector works differently in pypy')
-    def test_compat_http_error_autoclose(self):
-        # Compat HTTPError should not autoclose response
-        response = self.create_response(403)
-        _CompatHTTPError(HTTPError(response))
-        assert not response.closed
-
     def test_incomplete_read_error(self):
         error = IncompleteRead(4, 3, cause='test')
         assert isinstance(error, IncompleteRead)
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 8d96498a6..5dcefb5b8 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -40,7 +40,6 @@
     NoSupportingHandlers,
     RequestError,
     SSLError,
-    _CompatHTTPError,
     network_exceptions,
 )
 from .plugins import directories as plugin_directories
@@ -4110,8 +4109,6 @@ def urlopen(self, req):
                     'SSLV3_ALERT_HANDSHAKE_FAILURE: The server may not support the current cipher list. '
                     'Try using --legacy-server-connect', cause=e) from e
             raise
-        except HTTPError as e:  # TODO: Remove in a future release
-            raise _CompatHTTPError(e) from e
 
     def build_request_director(self, handlers, preferences=None):
         logger = _YDLLogger(self)
diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py
index 90ccf0f14..7ea5d0812 100644
--- a/yt_dlp/compat/_legacy.py
+++ b/yt_dlp/compat/_legacy.py
@@ -35,6 +35,7 @@
 from ..dependencies import brotli as compat_brotli  # noqa: F401
 from ..dependencies import websockets as compat_websockets  # noqa: F401
 from ..dependencies.Cryptodome import AES as compat_pycrypto_AES  # noqa: F401
+from ..networking.exceptions import HTTPError as compat_HTTPError  # noqa: F401
 
 passthrough_module(__name__, '...utils', ('WINDOWS_VT_MODE', 'windows_enable_vt_mode'))
 
@@ -70,7 +71,6 @@ def compat_setenv(key, value, env=os.environ):
 compat_HTMLParser = compat_html_parser_HTMLParser = html.parser.HTMLParser
 compat_http_client = http.client
 compat_http_server = http.server
-compat_HTTPError = urllib.error.HTTPError
 compat_input = input
 compat_integer_types = (int, )
 compat_itertools_count = itertools.count
@@ -88,7 +88,7 @@ def compat_setenv(key, value, env=os.environ):
 compat_subprocess_get_DEVNULL = lambda: subprocess.DEVNULL
 compat_tokenize_tokenize = tokenize.tokenize
 compat_urllib_error = urllib.error
-compat_urllib_HTTPError = urllib.error.HTTPError
+compat_urllib_HTTPError = compat_HTTPError
 compat_urllib_parse = urllib.parse
 compat_urllib_parse_parse_qs = urllib.parse.parse_qs
 compat_urllib_parse_quote = urllib.parse.quote
diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py
index 12441901c..9037f18e2 100644
--- a/yt_dlp/networking/exceptions.py
+++ b/yt_dlp/networking/exceptions.py
@@ -1,9 +1,8 @@
 from __future__ import annotations
 
 import typing
-import urllib.error
 
-from ..utils import YoutubeDLError, deprecation_warning
+from ..utils import YoutubeDLError
 
 if typing.TYPE_CHECKING:
     from .common import RequestHandler, Response
@@ -101,117 +100,4 @@ class ProxyError(TransportError):
     pass
 
 
-class _CompatHTTPError(urllib.error.HTTPError, HTTPError):
-    """
-    Provides backwards compatibility with urllib.error.HTTPError.
-    Do not use this class directly, use HTTPError instead.
-    """
-
-    def __init__(self, http_error: HTTPError):
-        super().__init__(
-            url=http_error.response.url,
-            code=http_error.status,
-            msg=http_error.msg,
-            hdrs=http_error.response.headers,
-            fp=http_error.response
-        )
-        self._closer.close_called = True  # Disable auto close
-        self._http_error = http_error
-        HTTPError.__init__(self, http_error.response, redirect_loop=http_error.redirect_loop)
-
-    @property
-    def status(self):
-        return self._http_error.status
-
-    @status.setter
-    def status(self, value):
-        return
-
-    @property
-    def reason(self):
-        return self._http_error.reason
-
-    @reason.setter
-    def reason(self, value):
-        return
-
-    @property
-    def headers(self):
-        deprecation_warning('HTTPError.headers is deprecated, use HTTPError.response.headers instead')
-        return self._http_error.response.headers
-
-    @headers.setter
-    def headers(self, value):
-        return
-
-    def info(self):
-        deprecation_warning('HTTPError.info() is deprecated, use HTTPError.response.headers instead')
-        return self.response.headers
-
-    def getcode(self):
-        deprecation_warning('HTTPError.getcode is deprecated, use HTTPError.status instead')
-        return self.status
-
-    def geturl(self):
-        deprecation_warning('HTTPError.geturl is deprecated, use HTTPError.response.url instead')
-        return self.response.url
-
-    @property
-    def code(self):
-        deprecation_warning('HTTPError.code is deprecated, use HTTPError.status instead')
-        return self.status
-
-    @code.setter
-    def code(self, value):
-        return
-
-    @property
-    def url(self):
-        deprecation_warning('HTTPError.url is deprecated, use HTTPError.response.url instead')
-        return self.response.url
-
-    @url.setter
-    def url(self, value):
-        return
-
-    @property
-    def hdrs(self):
-        deprecation_warning('HTTPError.hdrs is deprecated, use HTTPError.response.headers instead')
-        return self.response.headers
-
-    @hdrs.setter
-    def hdrs(self, value):
-        return
-
-    @property
-    def filename(self):
-        deprecation_warning('HTTPError.filename is deprecated, use HTTPError.response.url instead')
-        return self.response.url
-
-    @filename.setter
-    def filename(self, value):
-        return
-
-    def __getattr__(self, name):
-        # File operations are passed through the response.
-        # Warn for some commonly used ones
-        passthrough_warnings = {
-            'read': 'response.read()',
-            # technically possibly due to passthrough, but we should discourage this
-            'get_header': 'response.get_header()',
-            'readable': 'response.readable()',
-            'closed': 'response.closed',
-            'tell': 'response.tell()',
-        }
-        if name in passthrough_warnings:
-            deprecation_warning(f'HTTPError.{name} is deprecated, use HTTPError.{passthrough_warnings[name]} instead')
-        return super().__getattr__(name)
-
-    def __str__(self):
-        return str(self._http_error)
-
-    def __repr__(self):
-        return repr(self._http_error)
-
-
 network_exceptions = (HTTPError, TransportError)

From f24e44e8cbd88ce338d52f594a19330f64d38b50 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sat, 20 Jan 2024 06:08:55 +0100
Subject: [PATCH 800/871] [webvtt] Don't parse single fragment files (#9034)

Partially addresses #5804
Authored by: seproDev
---
 yt_dlp/downloader/hls.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index d4b3f0320..4ac5d99dc 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -369,7 +369,10 @@ def fin_fragments():
 
                 return output.getvalue().encode()
 
-            self.download_and_append_fragments(
-                ctx, fragments, info_dict, pack_func=pack_fragment, finish_func=fin_fragments)
+            if len(fragments) == 1:
+                self.download_and_append_fragments(ctx, fragments, info_dict)
+            else:
+                self.download_and_append_fragments(
+                    ctx, fragments, info_dict, pack_func=pack_fragment, finish_func=fin_fragments)
         else:
             return self.download_and_append_fragments(ctx, fragments, info_dict)

From 35f4f764a786685ea45d84abe1cf1ad3847f4c97 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Sun, 21 Jan 2024 10:03:33 +1300
Subject: [PATCH 801/871] [rh:requests] Apply `remove_dot_segments` to absolute
 redirect locations

Fixes https://github.com/yt-dlp/yt-dlp/issues/9020

Authored by: coletdjnz
---
 test/test_networking.py        | 25 ++++++++++++++++---------
 yt_dlp/networking/_requests.py |  5 +++++
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/test/test_networking.py b/test/test_networking.py
index 62325aa8e..8cadd86f5 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -180,6 +180,12 @@ def do_GET(self):
             self.send_header('Location', '/a/b/./../../headers')
             self.send_header('Content-Length', '0')
             self.end_headers()
+        elif self.path == '/redirect_dotsegments_absolute':
+            self.send_response(301)
+            # redirect to /headers but with dot segments before - absolute url
+            self.send_header('Location', f'http://127.0.0.1:{http_server_port(self.server)}/a/b/./../../headers')
+            self.send_header('Content-Length', '0')
+            self.end_headers()
         elif self.path.startswith('/redirect_'):
             self._redirect()
         elif self.path.startswith('/method'):
@@ -345,16 +351,17 @@ def test_percent_encode(self, handler):
             res.close()
 
     @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True)
-    def test_remove_dot_segments(self, handler):
-        with handler() as rh:
+    @pytest.mark.parametrize('path', [
+        '/a/b/./../../headers',
+        '/redirect_dotsegments',
+        # https://github.com/yt-dlp/yt-dlp/issues/9020
+        '/redirect_dotsegments_absolute',
+    ])
+    def test_remove_dot_segments(self, handler, path):
+        with handler(verbose=True) as rh:
             # This isn't a comprehensive test,
-            # but it should be enough to check whether the handler is removing dot segments
-            res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/a/b/./../../headers'))
-            assert res.status == 200
-            assert res.url == f'http://127.0.0.1:{self.http_port}/headers'
-            res.close()
-
-            res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/redirect_dotsegments'))
+            # but it should be enough to check whether the handler is removing dot segments in required scenarios
+            res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}{path}'))
             assert res.status == 200
             assert res.url == f'http://127.0.0.1:{self.http_port}/headers'
             res.close()
diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py
index e129110ca..00e4bdb49 100644
--- a/yt_dlp/networking/_requests.py
+++ b/yt_dlp/networking/_requests.py
@@ -8,6 +8,7 @@
 
 from ..dependencies import brotli, requests, urllib3
 from ..utils import bug_reports_message, int_or_none, variadic
+from ..utils.networking import normalize_url
 
 if requests is None:
     raise ImportError('requests module is not installed')
@@ -199,6 +200,10 @@ def rebuild_method(self, prepared_request, response):
 
         prepared_request.method = new_method
 
+        # Requests fails to resolve dot segments on absolute redirect locations
+        # See: https://github.com/yt-dlp/yt-dlp/issues/9020
+        prepared_request.url = normalize_url(prepared_request.url)
+
     def rebuild_auth(self, prepared_request, response):
         # HACK: undo status code change from rebuild_method, if applicable.
         # rebuild_auth runs after requests would remove headers/body based on status code

From fcaa2e735b00b15a2b0d9f55f4187c654b4b5b39 Mon Sep 17 00:00:00 2001
From: "lauren n. liberda" <lauren@selfisekai.rocks>
Date: Sun, 21 Jan 2024 03:22:26 +0100
Subject: [PATCH 802/871] [ie/Sejm,RedCDNLivx] Add extractors (#8676)

Authored by: selfisekai
---
 yt_dlp/extractor/_extractors.py |   2 +
 yt_dlp/extractor/redge.py       | 135 ++++++++++++++++++++
 yt_dlp/extractor/sejmpl.py      | 218 ++++++++++++++++++++++++++++++++
 3 files changed, 355 insertions(+)
 create mode 100644 yt_dlp/extractor/redge.py
 create mode 100644 yt_dlp/extractor/sejmpl.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 3d5c3eb60..31bef1eb5 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1593,6 +1593,7 @@
     RedBullIE,
 )
 from .reddit import RedditIE
+from .redge import RedCDNLivxIE
 from .redgifs import (
     RedGifsIE,
     RedGifsSearchIE,
@@ -1727,6 +1728,7 @@
 )
 from .scrolller import ScrolllerIE
 from .seeker import SeekerIE
+from .sejmpl import SejmIE
 from .senalcolombia import SenalColombiaLiveIE
 from .senategov import SenateISVPIE, SenateGovIE
 from .sendtonews import SendtoNewsIE
diff --git a/yt_dlp/extractor/redge.py b/yt_dlp/extractor/redge.py
new file mode 100644
index 000000000..875d6f8aa
--- /dev/null
+++ b/yt_dlp/extractor/redge.py
@@ -0,0 +1,135 @@
+import functools
+
+from .common import InfoExtractor
+from ..networking import HEADRequest
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    join_nonempty,
+    parse_qs,
+    update_url_query,
+)
+from ..utils.traversal import traverse_obj
+
+
+class RedCDNLivxIE(InfoExtractor):
+    _VALID_URL = r'https?://[^.]+\.(?:dcs\.redcdn|atmcdn)\.pl/(?:live(?:dash|hls|ss)|nvr)/o2/(?P<tenant>[^/?#]+)/(?P<id>[^?#]+)\.livx'
+    IE_NAME = 'redcdnlivx'
+
+    _TESTS = [{
+        'url': 'https://r.dcs.redcdn.pl/livedash/o2/senat/ENC02/channel.livx?indexMode=true&startTime=638272860000&stopTime=638292544000',
+        'info_dict': {
+            'id': 'ENC02-638272860000-638292544000',
+            'ext': 'mp4',
+            'title': 'ENC02',
+            'duration': 19683.982,
+            'live_status': 'was_live',
+        },
+    }, {
+        'url': 'https://r.dcs.redcdn.pl/livedash/o2/sejm/ENC18/live.livx?indexMode=true&startTime=722333096000&stopTime=722335562000',
+        'info_dict': {
+            'id': 'ENC18-722333096000-722335562000',
+            'ext': 'mp4',
+            'title': 'ENC18',
+            'duration': 2463.995,
+            'live_status': 'was_live',
+        },
+    }, {
+        'url': 'https://r.dcs.redcdn.pl/livehls/o2/sportevolution/live/triathlon2018/warsaw.livx/playlist.m3u8?startTime=550305000000&stopTime=550327620000',
+        'info_dict': {
+            'id': 'triathlon2018-warsaw-550305000000-550327620000',
+            'ext': 'mp4',
+            'title': 'triathlon2018/warsaw',
+            'duration': 22619.98,
+            'live_status': 'was_live',
+        },
+    }, {
+        'url': 'https://n-25-12.dcs.redcdn.pl/nvr/o2/sejm/Migacz-ENC01/1.livx?startTime=722347200000&stopTime=722367345000',
+        'only_matching': True,
+    }, {
+        'url': 'https://redir.atmcdn.pl/nvr/o2/sejm/ENC08/1.livx?startTime=503831270000&stopTime=503840040000',
+        'only_matching': True,
+    }]
+
+    """
+    Known methods (first in url path):
+    - `livedash` - DASH MPD
+    - `livehls` - HTTP Live Streaming
+    - `livess` - IIS Smooth Streaming
+    - `nvr` - CCTV mode, directly returns a file, typically flv, avc1, aac
+    - `sc` - shoutcast/icecast (audio streams, like radio)
+    """
+
+    def _real_extract(self, url):
+        tenant, path = self._match_valid_url(url).group('tenant', 'id')
+        qs = parse_qs(url)
+        start_time = traverse_obj(qs, ('startTime', 0, {int_or_none}))
+        stop_time = traverse_obj(qs, ('stopTime', 0, {int_or_none}))
+
+        def livx_mode(mode):
+            suffix = ''
+            if mode == 'livess':
+                suffix = '/manifest'
+            elif mode == 'livehls':
+                suffix = '/playlist.m3u8'
+            file_qs = {}
+            if start_time:
+                file_qs['startTime'] = start_time
+            if stop_time:
+                file_qs['stopTime'] = stop_time
+            if mode == 'nvr':
+                file_qs['nolimit'] = 1
+            elif mode != 'sc':
+                file_qs['indexMode'] = 'true'
+            return update_url_query(f'https://r.dcs.redcdn.pl/{mode}/o2/{tenant}/{path}.livx{suffix}', file_qs)
+
+        # no id or title for a transmission. making ones up.
+        title = path \
+            .replace('/live', '').replace('live/', '') \
+            .replace('/channel', '').replace('channel/', '') \
+            .strip('/')
+        video_id = join_nonempty(title.replace('/', '-'), start_time, stop_time)
+
+        formats = []
+        # downloading the manifest separately here instead of _extract_ism_formats to also get some stream metadata
+        ism_res = self._download_xml_handle(
+            livx_mode('livess'), video_id,
+            note='Downloading ISM manifest',
+            errnote='Failed to download ISM manifest',
+            fatal=False)
+        ism_doc = None
+        if ism_res is not False:
+            ism_doc, ism_urlh = ism_res
+            formats, _ = self._parse_ism_formats_and_subtitles(ism_doc, ism_urlh.url, 'ss')
+
+        nvr_urlh = self._request_webpage(
+            HEADRequest(livx_mode('nvr')), video_id, 'Follow flv file redirect', fatal=False,
+            expected_status=lambda _: True)
+        if nvr_urlh and nvr_urlh.status == 200:
+            formats.append({
+                'url': nvr_urlh.url,
+                'ext': 'flv',
+                'format_id': 'direct-0',
+                'preference': -1,   # might be slow
+            })
+        formats.extend(self._extract_mpd_formats(livx_mode('livedash'), video_id, mpd_id='dash', fatal=False))
+        formats.extend(self._extract_m3u8_formats(
+            livx_mode('livehls'), video_id, m3u8_id='hls', ext='mp4', fatal=False))
+
+        time_scale = traverse_obj(ism_doc, ('@TimeScale', {int_or_none})) or 10000000
+        duration = traverse_obj(
+            ism_doc, ('@Duration', {functools.partial(float_or_none, scale=time_scale)})) or None
+
+        live_status = None
+        if traverse_obj(ism_doc, '@IsLive') == 'TRUE':
+            live_status = 'is_live'
+        elif duration:
+            live_status = 'was_live'
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'duration': duration,
+            'live_status': live_status,
+        }
diff --git a/yt_dlp/extractor/sejmpl.py b/yt_dlp/extractor/sejmpl.py
new file mode 100644
index 000000000..29cb0152a
--- /dev/null
+++ b/yt_dlp/extractor/sejmpl.py
@@ -0,0 +1,218 @@
+import datetime
+
+from .common import InfoExtractor
+from .redge import RedCDNLivxIE
+from ..utils import (
+    clean_html,
+    join_nonempty,
+    js_to_json,
+    strip_or_none,
+    update_url_query,
+)
+from ..utils.traversal import traverse_obj
+
+
+def is_dst(date):
+    last_march = datetime.datetime(date.year, 3, 31)
+    last_october = datetime.datetime(date.year, 10, 31)
+    last_sunday_march = last_march - datetime.timedelta(days=last_march.isoweekday() % 7)
+    last_sunday_october = last_october - datetime.timedelta(days=last_october.isoweekday() % 7)
+    return last_sunday_march.replace(hour=2) <= date <= last_sunday_october.replace(hour=3)
+
+
+def rfc3339_to_atende(date):
+    date = datetime.datetime.fromisoformat(date)
+    date = date + datetime.timedelta(hours=1 if is_dst(date) else 0)
+    return int((date.timestamp() - 978307200) * 1000)
+
+
+class SejmIE(InfoExtractor):
+    _VALID_URL = (
+        r'https?://(?:www\.)?sejm\.gov\.pl/[Ss]ejm(?P<term>\d+)\.nsf/transmisje(?:_arch)?\.xsp(?:\?[^#]*)?#(?P<id>[\dA-F]+)',
+        r'https?://(?:www\.)?sejm\.gov\.pl/[Ss]ejm(?P<term>\d+)\.nsf/transmisje(?:_arch)?\.xsp\?(?:[^#]+&)?unid=(?P<id>[\dA-F]+)',
+        r'https?://sejm-embed\.redcdn\.pl/[Ss]ejm(?P<term>\d+)\.nsf/VideoFrame\.xsp/(?P<id>[\dA-F]+)',
+    )
+    IE_NAME = 'sejm'
+
+    _TESTS = [{
+        # multiple cameras, polish SL iterpreter
+        'url': 'https://www.sejm.gov.pl/Sejm10.nsf/transmisje_arch.xsp#6181EF1AD9CEEBB5C1258A6D006452B5',
+        'info_dict': {
+            'id': '6181EF1AD9CEEBB5C1258A6D006452B5',
+            'title': '1. posiedzenie Sejmu X kadencji',
+            'duration': 20145,
+            'live_status': 'was_live',
+            'location': 'Sala Posiedzeń',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'ENC01-722340000000-722360145000',
+                'ext': 'mp4',
+                'duration': 20145,
+                'title': '1. posiedzenie Sejmu X kadencji - ENC01',
+                'live_status': 'was_live',
+            },
+        }, {
+            'info_dict': {
+                'id': 'ENC30-722340000000-722360145000',
+                'ext': 'mp4',
+                'duration': 20145,
+                'title': '1. posiedzenie Sejmu X kadencji - ENC30',
+                'live_status': 'was_live',
+            },
+        }, {
+            'info_dict': {
+                'id': 'ENC31-722340000000-722360145000',
+                'ext': 'mp4',
+                'duration': 20145,
+                'title': '1. posiedzenie Sejmu X kadencji - ENC31',
+                'live_status': 'was_live',
+            },
+        }, {
+            'info_dict': {
+                'id': 'ENC32-722340000000-722360145000',
+                'ext': 'mp4',
+                'duration': 20145,
+                'title': '1. posiedzenie Sejmu X kadencji - ENC32',
+                'live_status': 'was_live',
+            },
+        }, {
+            # sign lang interpreter
+            'info_dict': {
+                'id': 'Migacz-ENC01-1-722340000000-722360145000',
+                'ext': 'mp4',
+                'duration': 20145,
+                'title': '1. posiedzenie Sejmu X kadencji - Migacz-ENC01',
+                'live_status': 'was_live',
+            },
+        }],
+    }, {
+        'url': 'https://www.sejm.gov.pl/Sejm8.nsf/transmisje.xsp?unid=9377A9D65518E9A5C125808E002E9FF2',
+        'info_dict': {
+            'id': '9377A9D65518E9A5C125808E002E9FF2',
+            'title': 'Debata "Lepsza Polska: obywatelska"',
+            'description': 'KP .Nowoczesna',
+            'duration': 8770,
+            'live_status': 'was_live',
+            'location': 'sala kolumnowa im. Kazimierza Pużaka (bud. C-D)',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'ENC08-1-503831270000-503840040000',
+                'ext': 'mp4',
+                'duration': 8770,
+                'title': 'Debata "Lepsza Polska: obywatelska" - ENC08',
+                'live_status': 'was_live',
+            },
+        }],
+    }, {
+        # 7th term is very special, since it does not use redcdn livx
+        'url': 'https://www.sejm.gov.pl/sejm7.nsf/transmisje_arch.xsp?rok=2015&month=11#A6E6D475ECCC6FE5C1257EF90034817F',
+        'info_dict': {
+            'id': 'A6E6D475ECCC6FE5C1257EF90034817F',
+            'title': 'Konferencja prasowa - Stanowisko SLD ws. składu nowego rządu',
+            'description': 'SLD - Biuro Prasowe Klubu',
+            'duration': 514,
+            'location': 'sala 101/bud. C',
+            'live_status': 'was_live',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'A6E6D475ECCC6FE5C1257EF90034817F',
+                'ext': 'mp4',
+                'title': 'Konferencja prasowa - Stanowisko SLD ws. składu nowego rządu',
+                'duration': 514,
+            },
+        }],
+    }, {
+        'url': 'https://sejm-embed.redcdn.pl/Sejm10.nsf/VideoFrame.xsp/FED58EABB97FBD53C1258A7400386492',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        term, video_id = self._match_valid_url(url).group('term', 'id')
+        frame = self._download_webpage(
+            f'https://sejm-embed.redcdn.pl/Sejm{term}.nsf/VideoFrame.xsp/{video_id}',
+            video_id)
+        # despite it says "transmisje_arch", it works for live streams too!
+        data = self._download_json(
+            f'https://www.sejm.gov.pl/Sejm{term}.nsf/transmisje_arch.xsp/json/{video_id}',
+            video_id)
+        params = data['params']
+
+        title = strip_or_none(data.get('title'))
+
+        if data.get('status') == 'VIDEO_ENDED':
+            live_status = 'was_live'
+        elif data.get('status') == 'VIDEO_PLAYING':
+            live_status = 'is_live'
+        else:
+            live_status = None
+            self.report_warning(f'unknown status: {data.get("status")}')
+
+        start_time = rfc3339_to_atende(params['start'])
+        # current streams have a stop time of *expected* end of session, but actual times
+        # can change during the transmission. setting a stop_time would artificially
+        # end the stream at that time, while the session actually keeps going.
+        if live_status == 'was_live':
+            stop_time = rfc3339_to_atende(params['stop'])
+            duration = (stop_time - start_time) // 1000
+        else:
+            stop_time, duration = None, None
+
+        entries = []
+
+        def add_entry(file, legacy_file=False):
+            if not file:
+                return
+            file = self._proto_relative_url(file)
+            if not legacy_file:
+                file = update_url_query(file, {'startTime': start_time})
+                if stop_time is not None:
+                    file = update_url_query(file, {'stopTime': stop_time})
+                stream_id = self._search_regex(r'/o2/sejm/([^/]+)/[^./]+\.livx', file, 'stream id')
+            common_info = {
+                'url': file,
+                'duration': duration,
+            }
+            if legacy_file:
+                entries.append({
+                    **common_info,
+                    'id': video_id,
+                    'title': title,
+                })
+            else:
+                entries.append({
+                    **common_info,
+                    '_type': 'url_transparent',
+                    'ie_key': RedCDNLivxIE.ie_key(),
+                    'id': stream_id,
+                    'title': join_nonempty(title, stream_id, delim=' - '),
+                })
+
+        cameras = self._search_json(
+            r'var\s+cameras\s*=', frame, 'camera list', video_id,
+            contains_pattern=r'\[(?s:.+)\]', transform_source=js_to_json,
+            fatal=False) or []
+        for camera_file in traverse_obj(cameras, (..., 'file', {dict})):
+            if camera_file.get('flv'):
+                add_entry(camera_file['flv'])
+            elif camera_file.get('mp4'):
+                # this is only a thing in 7th term. no streams before, and starting 8th it's redcdn livx
+                add_entry(camera_file['mp4'], legacy_file=True)
+            else:
+                self.report_warning('Unknown camera stream type found')
+
+        if params.get('mig'):
+            add_entry(self._search_regex(r"var sliUrl\s*=\s*'([^']+)'", frame, 'sign language interpreter url', fatal=False))
+
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': video_id,
+            'title': title,
+            'description': clean_html(data.get('desc')) or None,
+            'duration': duration,
+            'live_status': live_status,
+            'location': strip_or_none(data.get('location')),
+        }

From 5a63454b3637b3603434026cddfeac509218b90e Mon Sep 17 00:00:00 2001
From: Martin Renold <martin@log2.ch>
Date: Sun, 21 Jan 2024 03:45:38 +0100
Subject: [PATCH 803/871] [ie/mx3] Add extractors (#8736)

Authored by: martinxyz
---
 yt_dlp/extractor/_extractors.py |   5 +
 yt_dlp/extractor/mx3.py         | 171 ++++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+)
 create mode 100644 yt_dlp/extractor/mx3.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 31bef1eb5..c4f1ccb8e 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1137,6 +1137,11 @@
     MusicdexArtistIE,
     MusicdexPlaylistIE,
 )
+from .mx3 import (
+    Mx3IE,
+    Mx3NeoIE,
+    Mx3VolksmusikIE,
+)
 from .mxplayer import (
     MxplayerIE,
     MxplayerShowIE,
diff --git a/yt_dlp/extractor/mx3.py b/yt_dlp/extractor/mx3.py
new file mode 100644
index 000000000..cb9f50e0c
--- /dev/null
+++ b/yt_dlp/extractor/mx3.py
@@ -0,0 +1,171 @@
+import re
+
+from .common import InfoExtractor
+from ..networking import HEADRequest
+from ..utils import (
+    get_element_by_class,
+    int_or_none,
+    try_call,
+    url_or_none,
+    urlhandle_detect_ext,
+)
+from ..utils.traversal import traverse_obj
+
+
+class Mx3BaseIE(InfoExtractor):
+    _VALID_URL_TMPL = r'https?://(?:www\.)?%s/t/(?P<id>\w+)'
+    _FORMATS = [{
+        'url': 'player_asset',
+        'format_id': 'default',
+        'quality': 0,
+    }, {
+        'url': 'player_asset?quality=hd',
+        'format_id': 'hd',
+        'quality': 1,
+    }, {
+        'url': 'download',
+        'format_id': 'download',
+        'quality': 2,
+    }, {
+        'url': 'player_asset?quality=source',
+        'format_id': 'source',
+        'quality': 2,
+    }]
+
+    def _extract_formats(self, track_id):
+        formats = []
+        for fmt in self._FORMATS:
+            format_url = f'https://{self._DOMAIN}/tracks/{track_id}/{fmt["url"]}'
+            urlh = self._request_webpage(
+                HEADRequest(format_url), track_id, fatal=False, expected_status=404,
+                note=f'Checking for format {fmt["format_id"]}')
+            if urlh and urlh.status == 200:
+                formats.append({
+                    **fmt,
+                    'url': format_url,
+                    'ext': urlhandle_detect_ext(urlh),
+                    'filesize': int_or_none(urlh.headers.get('Content-Length')),
+                })
+        return formats
+
+    def _real_extract(self, url):
+        track_id = self._match_id(url)
+        webpage = self._download_webpage(url, track_id)
+        more_info = get_element_by_class('single-more-info', webpage)
+        data = self._download_json(f'https://{self._DOMAIN}/t/{track_id}.json', track_id, fatal=False)
+
+        def get_info_field(name):
+            return self._html_search_regex(
+                rf'<dt[^>]*>\s*{name}\s*</dt>\s*<dd[^>]*>(.*?)</dd>',
+                more_info, name, default=None, flags=re.DOTALL)
+
+        return {
+            'id': track_id,
+            'formats': self._extract_formats(track_id),
+            'genre': self._html_search_regex(
+                r'<div\b[^>]+class="single-band-genre"[^>]*>([^<]+)</div>', webpage, 'genre', default=None),
+            'release_year': int_or_none(get_info_field('Year of creation')),
+            'description': get_info_field('Description'),
+            'tags': try_call(lambda: get_info_field('Tag').split(', '), list),
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'artist': (('performer_name', 'artist'), {str}),
+                'album_artist': ('artist', {str}),
+                'composer': ('composer_name', {str}),
+                'thumbnail': (('picture_url_xlarge', 'picture_url'), {url_or_none}),
+            }, get_all=False),
+        }
+
+
+class Mx3IE(Mx3BaseIE):
+    _DOMAIN = 'mx3.ch'
+    _VALID_URL = Mx3BaseIE._VALID_URL_TMPL % re.escape(_DOMAIN)
+    _TESTS = [{
+        'url': 'https://mx3.ch/t/1Cru',
+        'md5': '7ba09e9826b4447d4e1ce9d69e0e295f',
+        'info_dict': {
+            'id': '1Cru',
+            'ext': 'wav',
+            'artist': 'Godina',
+            'album_artist': 'Tortue Tortue',
+            'composer': 'Olivier Godinat',
+            'genre': 'Rock',
+            'thumbnail': 'https://mx3.ch/pictures/mx3/file/0101/4643/square_xlarge/1-s-envoler-1.jpg?1630272813',
+            'title': "S'envoler",
+            'release_year': 2021,
+            'tags': [],
+        }
+    }, {
+        'url': 'https://mx3.ch/t/1LIY',
+        'md5': '48293cb908342547827f963a5a2e9118',
+        'info_dict': {
+            'id': '1LIY',
+            'ext': 'mov',
+            'artist': 'Tania Kimfumu',
+            'album_artist': 'The Broots',
+            'composer': 'Emmanuel Diserens',
+            'genre': 'Electro',
+            'thumbnail': 'https://mx3.ch/pictures/mx3/file/0110/0003/video_xlarge/frame_0000.png?1686963670',
+            'title': 'The Broots-Larytta remix "Begging For Help"',
+            'release_year': 2023,
+            'tags': ['the broots', 'cassata records', 'larytta'],
+            'description': '"Begging for Help" Larytta Remix Official Video\nRealized By Kali Donkilie in 2023',
+        }
+    }, {
+        'url': 'https://mx3.ch/t/1C6E',
+        'md5': '1afcd578493ddb8e5008e94bb6d97e25',
+        'info_dict': {
+            'id': '1C6E',
+            'ext': 'wav',
+            'artist': 'Alien Bubblegum',
+            'album_artist': 'Alien Bubblegum',
+            'composer': 'Alien Bubblegum',
+            'genre': 'Punk',
+            'thumbnail': 'https://mx3.ch/pictures/mx3/file/0101/1551/square_xlarge/pandora-s-box-cover-with-title.png?1627054733',
+            'title': 'Wide Awake',
+            'release_year': 2021,
+            'tags': ['alien bubblegum', 'bubblegum', 'alien', 'pop punk', 'poppunk'],
+        }
+    }]
+
+
+class Mx3NeoIE(Mx3BaseIE):
+    _DOMAIN = 'neo.mx3.ch'
+    _VALID_URL = Mx3BaseIE._VALID_URL_TMPL % re.escape(_DOMAIN)
+    _TESTS = [{
+        'url': 'https://neo.mx3.ch/t/1hpd',
+        'md5': '6d9986bbae5cac3296ec8813bf965eb2',
+        'info_dict': {
+            'id': '1hpd',
+            'ext': 'wav',
+            'artist': 'Baptiste Lopez',
+            'album_artist': 'Kammerorchester Basel',
+            'composer': 'Jannik Giger',
+            'genre': 'Composition, Orchestra',
+            'title': 'Troisième œil. Für Kammerorchester (2023)',
+            'thumbnail': 'https://neo.mx3.ch/pictures/neo/file/0000/0241/square_xlarge/kammerorchester-basel-group-photo-2_c_-lukasz-rajchert.jpg?1560341252',
+            'release_year': 2023,
+            'tags': [],
+        }
+    }]
+
+
+class Mx3VolksmusikIE(Mx3BaseIE):
+    _DOMAIN = 'volksmusik.mx3.ch'
+    _VALID_URL = Mx3BaseIE._VALID_URL_TMPL % re.escape(_DOMAIN)
+    _TESTS = [{
+        'url': 'https://volksmusik.mx3.ch/t/Zx',
+        'md5': 'dd967a7b0c1ef898f3e072cf9c2eae3c',
+        'info_dict': {
+            'id': 'Zx',
+            'ext': 'mp3',
+            'artist': 'Ländlerkapelle GrischArt',
+            'album_artist': 'Ländlerkapelle GrischArt',
+            'composer': 'Urs Glauser',
+            'genre': 'Instrumental, Graubünden',
+            'title': 'Chämilouf',
+            'thumbnail': 'https://volksmusik.mx3.ch/pictures/vxm/file/0000/3815/square_xlarge/grischart1.jpg?1450530120',
+            'release_year': 2012,
+            'tags': [],
+        }
+    }]

From 9f1e9dab21bbe651544c8f4663b0e615dc450e4d Mon Sep 17 00:00:00 2001
From: dasidiot <140998618+dasidiot@users.noreply.github.com>
Date: Sat, 20 Jan 2024 21:46:53 -0500
Subject: [PATCH 804/871] [ie/motherless] Support uploader playlists (#8994)

Authored by: dasidiot
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/motherless.py  | 31 ++++++++++++++++++++++++++++---
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index c4f1ccb8e..a273ae0d9 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1111,6 +1111,7 @@
     MotherlessIE,
     MotherlessGroupIE,
     MotherlessGalleryIE,
+    MotherlessUploaderIE,
 )
 from .motorsport import MotorsportIE
 from .moviepilot import MoviepilotIE
diff --git a/yt_dlp/extractor/motherless.py b/yt_dlp/extractor/motherless.py
index e359c44e9..160150a7b 100644
--- a/yt_dlp/extractor/motherless.py
+++ b/yt_dlp/extractor/motherless.py
@@ -177,6 +177,7 @@ def _real_extract(self, url):
 
 
 class MotherlessPaginatedIE(InfoExtractor):
+    _EXTRA_QUERY = {}
     _PAGE_SIZE = 60
 
     def _correct_path(self, url, item_id):
@@ -199,7 +200,7 @@ def _real_extract(self, url):
         def get_page(idx):
             page = idx + 1
             current_page = webpage if not idx else self._download_webpage(
-                real_url, item_id, note=f'Downloading page {page}', query={'page': page})
+                real_url, item_id, note=f'Downloading page {page}', query={'page': page, **self._EXTRA_QUERY})
             yield from self._extract_entries(current_page, real_url)
 
         return self.playlist_result(
@@ -213,7 +214,7 @@ class MotherlessGroupIE(MotherlessPaginatedIE):
         'url': 'http://motherless.com/gv/movie_scenes',
         'info_dict': {
             'id': 'movie_scenes',
-            'title': 'Movie Scenes',
+            'title': 'Movie Scenes - Videos - Hot and sexy scenes from "regular" movies... Beautiful actresses fully',
         },
         'playlist_mincount': 540,
     }, {
@@ -244,7 +245,7 @@ class MotherlessGalleryIE(MotherlessPaginatedIE):
             'id': '338999F',
             'title': 'Random',
         },
-        'playlist_mincount': 190,
+        'playlist_mincount': 171,
     }, {
         'url': 'https://motherless.com/GVABD6213',
         'info_dict': {
@@ -270,3 +271,27 @@ class MotherlessGalleryIE(MotherlessPaginatedIE):
 
     def _correct_path(self, url, item_id):
         return urllib.parse.urljoin(url, f'/GV{item_id}')
+
+
+class MotherlessUploaderIE(MotherlessPaginatedIE):
+    _VALID_URL = r'https?://(?:www\.)?motherless\.com/u/(?P<id>\w+)/?(?:$|[?#])'
+    _TESTS = [{
+        'url': 'https://motherless.com/u/Mrgo4hrs2023',
+        'info_dict': {
+            'id': 'Mrgo4hrs2023',
+            'title': "Mrgo4hrs2023's Uploads - Videos",
+        },
+        'playlist_mincount': 32,
+    }, {
+        'url': 'https://motherless.com/u/Happy_couple?t=v',
+        'info_dict': {
+            'id': 'Happy_couple',
+            'title': "Happy_couple's Uploads - Videos",
+        },
+        'playlist_mincount': 8,
+    }]
+
+    _EXTRA_QUERY = {'t': 'v'}
+
+    def _correct_path(self, url, item_id):
+        return urllib.parse.urljoin(url, f'/u/{item_id}?t=v')

From 3e083191cdc34dd8c482da9a9b4bc682f824cb9d Mon Sep 17 00:00:00 2001
From: u-spec-png <srdjankalaba@protonmail.ch>
Date: Sun, 21 Jan 2024 19:50:14 +0100
Subject: [PATCH 805/871] [ie/Newgrounds:user] Fix extractor (#9046)

Closes #7308
Authored by: u-spec-png
---
 yt_dlp/extractor/newgrounds.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/yt_dlp/extractor/newgrounds.py b/yt_dlp/extractor/newgrounds.py
index 9e3286dfe..9601cd10e 100644
--- a/yt_dlp/extractor/newgrounds.py
+++ b/yt_dlp/extractor/newgrounds.py
@@ -3,15 +3,15 @@
 
 from .common import InfoExtractor
 from ..utils import (
+    OnDemandPagedList,
     clean_html,
     extract_attributes,
     get_element_by_id,
     int_or_none,
     parse_count,
     parse_duration,
+    traverse_obj,
     unified_timestamp,
-    OnDemandPagedList,
-    try_get,
 )
 
 
@@ -263,19 +263,16 @@ class NewgroundsUserIE(InfoExtractor):
     def _fetch_page(self, channel_id, url, page):
         page += 1
         posts_info = self._download_json(
-            f'{url}/page/{page}', channel_id,
+            f'{url}?page={page}', channel_id,
             note=f'Downloading page {page}', headers={
                 'Accept': 'application/json, text/javascript, */*; q = 0.01',
                 'X-Requested-With': 'XMLHttpRequest',
             })
-        sequence = posts_info.get('sequence', [])
-        for year in sequence:
-            posts = try_get(posts_info, lambda x: x['years'][str(year)]['items'])
-            for post in posts:
-                path, media_id = self._search_regex(
-                    r'<a[^>]+\bhref=["\'][^"\']+((?:portal/view|audio/listen)/(\d+))[^>]+>',
-                    post, 'url', group=(1, 2))
-                yield self.url_result(f'https://www.newgrounds.com/{path}', NewgroundsIE.ie_key(), media_id)
+        for post in traverse_obj(posts_info, ('items', ..., ..., {str})):
+            path, media_id = self._search_regex(
+                r'<a[^>]+\bhref=["\'][^"\']+((?:portal/view|audio/listen)/(\d+))[^>]+>',
+                post, 'url', group=(1, 2))
+            yield self.url_result(f'https://www.newgrounds.com/{path}', NewgroundsIE.ie_key(), media_id)
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)

From c0ecceeefe6ebd27452d9d8f20658f83ae121d04 Mon Sep 17 00:00:00 2001
From: gmes78 <gmes.078@gmail.com>
Date: Sun, 21 Jan 2024 18:56:01 +0000
Subject: [PATCH 806/871] [ie/Rule34Video] Fix `_VALID_URL` (#9044)

Authored by: gmes78
---
 yt_dlp/extractor/rule34video.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/rule34video.py b/yt_dlp/extractor/rule34video.py
index e6bb4258e..85ad7e2ff 100644
--- a/yt_dlp/extractor/rule34video.py
+++ b/yt_dlp/extractor/rule34video.py
@@ -18,10 +18,10 @@
 
 
 class Rule34VideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rule34video\.com/videos/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rule34video\.com/videos?/(?P<id>\d+)'
     _TESTS = [
         {
-            'url': 'https://rule34video.com/videos/3065157/shot-it-mmd-hmv/',
+            'url': 'https://rule34video.com/video/3065157/shot-it-mmd-hmv/',
             'md5': 'ffccac2c23799dabbd192621ae4d04f3',
             'info_dict': {
                 'id': '3065157',

From c099ec9392b0283dde34b290d1a04158ad8eb882 Mon Sep 17 00:00:00 2001
From: Stefan Lobbenmeier <Stefan.Lobbenmeier@gmail.com>
Date: Sun, 21 Jan 2024 21:54:11 +0100
Subject: [PATCH 807/871] [ie/ard:mediathek] Support cookies to verify age
 (#9037)

Closes #9035
Authored by: StefanLobbenmeier
---
 yt_dlp/extractor/ard.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/ard.py b/yt_dlp/extractor/ard.py
index f4b1cd075..46e68d61e 100644
--- a/yt_dlp/extractor/ard.py
+++ b/yt_dlp/extractor/ard.py
@@ -8,6 +8,7 @@
     determine_ext,
     int_or_none,
     join_nonempty,
+    jwt_decode_hs256,
     make_archive_id,
     parse_duration,
     parse_iso8601,
@@ -238,6 +239,7 @@ class ARDBetaMediathekIE(InfoExtractor):
         (?P<id>[a-zA-Z0-9]+)
         /?(?:[?#]|$)'''
     _GEO_COUNTRIES = ['DE']
+    _TOKEN_URL = 'https://sso.ardmediathek.de/sso/token'
 
     _TESTS = [{
         'url': 'https://www.ardmediathek.de/video/filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen/Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
@@ -359,12 +361,27 @@ def _extract_episode_info(self, title):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        query = {'embedded': 'false', 'mcV6': 'true'}
+        headers = {}
+
+        if self._get_cookies(self._TOKEN_URL).get('ams'):
+            token = self._download_json(
+                self._TOKEN_URL, display_id, 'Fetching token for age verification',
+                'Unable to fetch age verification token', fatal=False)
+            id_token = traverse_obj(token, ('idToken', {str}))
+            decoded_token = traverse_obj(id_token, ({jwt_decode_hs256}, {dict}))
+            user_id = traverse_obj(decoded_token, (('user_id', 'sub'), {str}), get_all=False)
+            if not user_id:
+                self.report_warning('Unable to extract token, continuing without authentication')
+            else:
+                headers['x-authorization'] = f'Bearer {id_token}'
+                query['userId'] = user_id
+                if decoded_token.get('age_rating') != 18:
+                    self.report_warning('Account is not verified as 18+; video may be unavailable')
 
         page_data = self._download_json(
-            f'https://api.ardmediathek.de/page-gateway/pages/ard/item/{display_id}', display_id, query={
-                'embedded': 'false',
-                'mcV6': 'true',
-            })
+            f'https://api.ardmediathek.de/page-gateway/pages/ard/item/{display_id}',
+            display_id, query=query, headers=headers)
 
         # For user convenience we use the old contentId instead of the longer crid
         # Ref: https://github.com/yt-dlp/yt-dlp/issues/8731#issuecomment-1874398283
@@ -383,7 +400,7 @@ def _real_extract(self, url):
         media_data = traverse_obj(player_data, ('mediaCollection', 'embedded', {dict}))
 
         if player_data.get('blockedByFsk'):
-            self.raise_no_formats('This video is only available after 22:00', expected=True)
+            self.raise_login_required('This video is only available for age verified users or after 22:00')
 
         formats = []
         subtitles = {}

From f0e8bc7c60b61fe18b63116c975609d76b904771 Mon Sep 17 00:00:00 2001
From: John Victor <37747572+johnvictorfs@users.noreply.github.com>
Date: Sun, 21 Jan 2024 19:36:59 -0300
Subject: [PATCH 808/871] [ie/patreon] Fix embedded HLS extraction (#8993)

Closes #8973
Authored by: johnvictorfs
---
 yt_dlp/extractor/patreon.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/patreon.py b/yt_dlp/extractor/patreon.py
index 9316789df..d2ddb72cd 100644
--- a/yt_dlp/extractor/patreon.py
+++ b/yt_dlp/extractor/patreon.py
@@ -275,7 +275,7 @@ def _real_extract(self, url):
                     'ext': ext,
                     'url': post_file['url'],
                 }
-            elif name == 'video':
+            elif name == 'video' or determine_ext(post_file.get('url')) == 'm3u8':
                 formats, subtitles = self._extract_m3u8_formats_and_subtitles(post_file['url'], video_id)
                 return {
                     **info,

From 9cd90447907a59c8a2727583f4a755fb23ed8cd3 Mon Sep 17 00:00:00 2001
From: chtk <stefan@chtk.nl>
Date: Mon, 22 Jan 2024 06:57:52 +0100
Subject: [PATCH 809/871] [ie/Floatplane] Improve metadata extraction (#8934)

Authored by: chtk
---
 yt_dlp/extractor/floatplane.py | 103 +++++++++++++++++++++++++++------
 1 file changed, 84 insertions(+), 19 deletions(-)

diff --git a/yt_dlp/extractor/floatplane.py b/yt_dlp/extractor/floatplane.py
index 2cf4d4e64..8676d73f6 100644
--- a/yt_dlp/extractor/floatplane.py
+++ b/yt_dlp/extractor/floatplane.py
@@ -11,6 +11,7 @@
     join_nonempty,
     parse_codecs,
     parse_iso8601,
+    url_or_none,
     urljoin,
 )
 from ..utils.traversal import traverse_obj
@@ -108,6 +109,64 @@ class FloatplaneIE(InfoExtractor):
             'availability': 'subscriber_only',
         },
         'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.floatplane.com/post/65B5PNoBtf',
+        'info_dict': {
+            'id': '65B5PNoBtf',
+            'description': 'I recorded the inbuilt demo mode for your 90\'s enjoyment, thanks for being Floaties!',
+            'display_id': '65B5PNoBtf',
+            'like_count': int,
+            'release_timestamp': 1701249480,
+            'uploader': 'The Trash Network',
+            'availability': 'subscriber_only',
+            'uploader_id': '61bc20c9a131fb692bf2a513',
+            'uploader_url': 'https://www.floatplane.com/channel/TheTrashNetwork/home',
+            'channel_url': 'https://www.floatplane.com/channel/TheTrashNetwork/home/thedrumthing',
+            'comment_count': int,
+            'title': 'The $50 electronic drum kit.',
+            'channel_id': '64424fe73cd58cbcf8d8e131',
+            'thumbnail': 'https://pbs.floatplane.com/blogPost_thumbnails/65B5PNoBtf/725555379422705_1701247052743.jpeg',
+            'dislike_count': int,
+            'channel': 'The Drum Thing',
+            'release_date': '20231129',
+        },
+        'playlist_count': 2,
+        'playlist': [{
+            'info_dict': {
+                'id': 'ISPJjexylS',
+                'ext': 'mp4',
+                'release_date': '20231129',
+                'release_timestamp': 1701249480,
+                'title': 'The $50 electronic drum kit. .mov',
+                'channel_id': '64424fe73cd58cbcf8d8e131',
+                'thumbnail': 'https://pbs.floatplane.com/video_thumbnails/ISPJjexylS/335202812134041_1701249383392.jpeg',
+                'availability': 'subscriber_only',
+                'uploader': 'The Trash Network',
+                'duration': 622,
+                'channel': 'The Drum Thing',
+                'uploader_id': '61bc20c9a131fb692bf2a513',
+                'channel_url': 'https://www.floatplane.com/channel/TheTrashNetwork/home/thedrumthing',
+                'uploader_url': 'https://www.floatplane.com/channel/TheTrashNetwork/home',
+            },
+        }, {
+            'info_dict': {
+                'id': 'qKfxu6fEpu',
+                'ext': 'aac',
+                'release_date': '20231129',
+                'release_timestamp': 1701249480,
+                'title': 'Roland TD-7 Demo.m4a',
+                'channel_id': '64424fe73cd58cbcf8d8e131',
+                'availability': 'subscriber_only',
+                'uploader': 'The Trash Network',
+                'duration': 114,
+                'channel': 'The Drum Thing',
+                'uploader_id': '61bc20c9a131fb692bf2a513',
+                'channel_url': 'https://www.floatplane.com/channel/TheTrashNetwork/home/thedrumthing',
+                'uploader_url': 'https://www.floatplane.com/channel/TheTrashNetwork/home',
+            },
+        }],
+        'skip': 'requires subscription: "The Trash Network"',
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_initialize(self):
@@ -124,6 +183,22 @@ def _real_extract(self, url):
         if not any(traverse_obj(post_data, ('metadata', ('hasVideo', 'hasAudio')))):
             raise ExtractorError('Post does not contain a video or audio track', expected=True)
 
+        uploader_url = format_field(
+            post_data, [('creator', 'urlname')], 'https://www.floatplane.com/channel/%s/home') or None
+
+        common_info = {
+            'uploader_url': uploader_url,
+            'channel_url': urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname'))),
+            'availability': self._availability(needs_subscription=True),
+            **traverse_obj(post_data, {
+                'uploader': ('creator', 'title', {str}),
+                'uploader_id': ('creator', 'id', {str}),
+                'channel': ('channel', 'title', {str}),
+                'channel_id': ('channel', 'id', {str}),
+                'release_timestamp': ('releaseDate', {parse_iso8601}),
+            }),
+        }
+
         items = []
         for media in traverse_obj(post_data, (('videoAttachments', 'audioAttachments'), ...)):
             media_id = media['id']
@@ -150,11 +225,11 @@ def format_path(params):
             formats = []
             for quality in traverse_obj(stream, ('resource', 'data', 'qualityLevels', ...)):
                 url = urljoin(stream['cdn'], format_path(traverse_obj(
-                    stream, ('resource', 'data', 'qualityLevelParams', quality['name']))))
+                    stream, ('resource', 'data', 'qualityLevelParams', quality['name'], {dict}))))
                 formats.append({
                     **traverse_obj(quality, {
-                        'format_id': 'name',
-                        'format_note': 'label',
+                        'format_id': ('name', {str}),
+                        'format_note': ('label', {str}),
                         'width': ('width', {int}),
                         'height': ('height', {int}),
                     }),
@@ -164,38 +239,28 @@ def format_path(params):
                 })
 
             items.append({
+                **common_info,
                 'id': media_id,
                 **traverse_obj(metadata, {
-                    'title': 'title',
+                    'title': ('title', {str}),
                     'duration': ('duration', {int_or_none}),
-                    'thumbnail': ('thumbnail', 'path'),
+                    'thumbnail': ('thumbnail', 'path', {url_or_none}),
                 }),
                 'formats': formats,
             })
 
-        uploader_url = format_field(
-            post_data, [('creator', 'urlname')], 'https://www.floatplane.com/channel/%s/home') or None
-        channel_url = urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname')))
-
         post_info = {
+            **common_info,
             'id': post_id,
             'display_id': post_id,
             **traverse_obj(post_data, {
-                'title': 'title',
+                'title': ('title', {str}),
                 'description': ('text', {clean_html}),
-                'uploader': ('creator', 'title'),
-                'uploader_id': ('creator', 'id'),
-                'channel': ('channel', 'title'),
-                'channel_id': ('channel', 'id'),
                 'like_count': ('likes', {int_or_none}),
                 'dislike_count': ('dislikes', {int_or_none}),
                 'comment_count': ('comments', {int_or_none}),
-                'release_timestamp': ('releaseDate', {parse_iso8601}),
-                'thumbnail': ('thumbnail', 'path'),
+                'thumbnail': ('thumbnail', 'path', {url_or_none}),
             }),
-            'uploader_url': uploader_url,
-            'channel_url': channel_url,
-            'availability': self._availability(needs_subscription=True),
         }
 
         if len(items) > 1:

From a40b0070c2a00d3ed839897462171a82323aa875 Mon Sep 17 00:00:00 2001
From: kclauhk <78251477+kclauhk@users.noreply.github.com>
Date: Mon, 22 Jan 2024 14:28:11 +0800
Subject: [PATCH 810/871] [ie/facebook:ads] Add extractor (#8870)

Closes #8083
Authored by: kclauhk
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/facebook.py    | 112 ++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index a273ae0d9..f51045668 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -588,6 +588,7 @@
     FacebookPluginsVideoIE,
     FacebookRedirectURLIE,
     FacebookReelIE,
+    FacebookAdsIE,
 )
 from .fancode import (
     FancodeVodIE,
diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index a16a067ab..26cfda538 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -20,6 +20,7 @@
     get_element_by_id,
     get_first,
     int_or_none,
+    join_nonempty,
     js_to_json,
     merge_dicts,
     parse_count,
@@ -907,3 +908,114 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
         return self.url_result(
             f'https://m.facebook.com/watch/?v={video_id}&_rdr', FacebookIE, video_id)
+
+
+class FacebookAdsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/ads/library/?\?(?:[^#]+&)?id=(?P<id>\d+)'
+    IE_NAME = 'facebook:ads'
+
+    _TESTS = [{
+        'url': 'https://www.facebook.com/ads/library/?id=899206155126718',
+        'info_dict': {
+            'id': '899206155126718',
+            'ext': 'mp4',
+            'title': 'video by Kandao',
+            'uploader': 'Kandao',
+            'uploader_id': '774114102743284',
+            'uploader_url': r're:^https?://.*',
+            'timestamp': 1702548330,
+            'thumbnail': r're:^https?://.*',
+            'upload_date': '20231214',
+            'like_count': int,
+        }
+    }, {
+        'url': 'https://www.facebook.com/ads/library/?id=893637265423481',
+        'info_dict': {
+            'id': '893637265423481',
+            'title': 'Jusqu\u2019\u00e0 -25% sur une s\u00e9lection de vins p\u00e9tillants italiens ',
+            'uploader': 'Eataly Paris Marais',
+            'uploader_id': '2086668958314152',
+            'uploader_url': r're:^https?://.*',
+            'timestamp': 1703571529,
+            'upload_date': '20231226',
+            'like_count': int,
+        },
+        'playlist_count': 3,
+    }, {
+        'url': 'https://es-la.facebook.com/ads/library/?id=901230958115569',
+        'only_matching': True,
+    }, {
+        'url': 'https://m.facebook.com/ads/library/?id=901230958115569',
+        'only_matching': True,
+    }]
+
+    _FORMATS_MAP = {
+        'watermarked_video_sd_url': ('sd-wmk', 'SD, watermarked'),
+        'video_sd_url': ('sd', None),
+        'watermarked_video_hd_url': ('hd-wmk', 'HD, watermarked'),
+        'video_hd_url': ('hd', None),
+    }
+
+    def _extract_formats(self, video_dict):
+        formats = []
+        for format_key, format_url in traverse_obj(video_dict, (
+            {dict.items}, lambda _, v: v[0] in self._FORMATS_MAP and url_or_none(v[1])
+        )):
+            formats.append({
+                'format_id': self._FORMATS_MAP[format_key][0],
+                'format_note': self._FORMATS_MAP[format_key][1],
+                'url': format_url,
+                'ext': 'mp4',
+                'quality': qualities(tuple(self._FORMATS_MAP))(format_key),
+            })
+        return formats
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        post_data = [self._parse_json(j, video_id, fatal=False)
+                     for j in re.findall(r's\.handle\(({.*})\);requireLazy\(', webpage)]
+        data = traverse_obj(post_data, (
+            ..., 'require', ..., ..., ..., 'props', 'deeplinkAdCard', 'snapshot', {dict}), get_all=False)
+        if not data:
+            raise ExtractorError('Unable to extract ad data')
+
+        title = data.get('title')
+        if not title or title == '{{product.name}}':
+            title = join_nonempty('display_format', 'page_name', delim=' by ', from_dict=data)
+
+        info_dict = traverse_obj(data, {
+            'description': ('link_description', {str}, {lambda x: x if x != '{{product.description}}' else None}),
+            'uploader': ('page_name', {str}),
+            'uploader_id': ('page_id', {str_or_none}),
+            'uploader_url': ('page_profile_uri', {url_or_none}),
+            'timestamp': ('creation_time', {int_or_none}),
+            'like_count': ('page_like_count', {int_or_none}),
+        })
+
+        entries = []
+        for idx, entry in enumerate(traverse_obj(
+            data, (('videos', 'cards'), lambda _, v: any([url_or_none(v[f]) for f in self._FORMATS_MAP]))), 1
+        ):
+            entries.append({
+                'id': f'{video_id}_{idx}',
+                'title': entry.get('title') or title,
+                'description': entry.get('link_description') or info_dict.get('description'),
+                'thumbnail': url_or_none(entry.get('video_preview_image_url')),
+                'formats': self._extract_formats(entry),
+            })
+
+        if len(entries) == 1:
+            info_dict.update(entries[0])
+
+        elif len(entries) > 1:
+            info_dict.update({
+                'title': entries[0]['title'],
+                'entries': entries,
+                '_type': 'playlist',
+            })
+
+        info_dict['id'] = video_id
+
+        return info_dict

From 5f25f348f9eb5db842b1ec6799f95bebb7ba35a7 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 23 Jan 2024 23:20:13 +0100
Subject: [PATCH 811/871] [ie/pr0gramm] Enable POL filter and provide tags
 without login (#9051)

Authored by: Grub4K
---
 yt_dlp/extractor/pr0gramm.py | 41 ++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/yt_dlp/extractor/pr0gramm.py b/yt_dlp/extractor/pr0gramm.py
index 2a6794208..36e415f4a 100644
--- a/yt_dlp/extractor/pr0gramm.py
+++ b/yt_dlp/extractor/pr0gramm.py
@@ -18,7 +18,6 @@
 class Pr0grammIE(InfoExtractor):
     _VALID_URL = r'https?://pr0gramm\.com\/(?:[^/?#]+/)+(?P<id>[\d]+)(?:[/?#:]|$)'
     _TESTS = [{
-        # Tags require account
         'url': 'https://pr0gramm.com/new/video/5466437',
         'info_dict': {
             'id': '5466437',
@@ -36,7 +35,6 @@ class Pr0grammIE(InfoExtractor):
             '_old_archive_ids': ['pr0grammstatic 5466437'],
         },
     }, {
-        # Tags require account
         'url': 'https://pr0gramm.com/new/3052805:comment28391322',
         'info_dict': {
             'id': '3052805',
@@ -71,6 +69,23 @@ class Pr0grammIE(InfoExtractor):
             'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
             '_old_archive_ids': ['pr0grammstatic 5848332'],
         },
+    }, {
+        'url': 'https://pr0gramm.com/top/5895149',
+        'info_dict': {
+            'id': '5895149',
+            'ext': 'mp4',
+            'title': 'pr0gramm-5895149 by algoholigSeeManThrower',
+            'tags': 'count:19',
+            'uploader': 'algoholigSeeManThrower',
+            'uploader_id': 457556,
+            'upload_timestamp': 1697580902,
+            'upload_date': '20231018',
+            'like_count': int,
+            'dislike_count': int,
+            'age_limit': 0,
+            'thumbnail': 'https://thumb.pr0gramm.com/2023/10/18/db47bb3db5e1a1b3.jpg',
+            '_old_archive_ids': ['pr0grammstatic 5895149'],
+        },
     }, {
         'url': 'https://pr0gramm.com/static/5466437',
         'only_matching': True,
@@ -92,15 +107,15 @@ def _is_logged_in(self):
     def _maximum_flags(self):
         # We need to guess the flags for the content otherwise the api will raise an error
         # We can guess the maximum allowed flags for the account from the cookies
-        # Bitflags are (msbf): nsfp, nsfl, nsfw, sfw
-        flags = 0b0001
+        # Bitflags are (msbf): pol, nsfp, nsfl, nsfw, sfw
+        flags = 0b10001
         if self._is_logged_in:
-            flags |= 0b1000
+            flags |= 0b01000
             cookies = self._get_cookies(self.BASE_URL)
             if 'me' not in cookies:
                 self._download_webpage(self.BASE_URL, None, 'Refreshing verification information')
             if traverse_obj(cookies, ('me', {lambda x: x.value}, {unquote}, {json.loads}, 'verified')):
-                flags |= 0b0110
+                flags |= 0b00110
 
         return flags
 
@@ -134,14 +149,12 @@ def _real_extract(self, url):
         if not source or not source.endswith('mp4'):
             self.raise_no_formats('Could not extract a video', expected=bool(source), video_id=video_id)
 
-        tags = None
-        if self._is_logged_in:
-            metadata = self._call_api('info', video_id, {'itemId': video_id}, note='Downloading tags')
-            tags = traverse_obj(metadata, ('tags', ..., 'tag', {str}))
-            # Sorted by "confidence", higher confidence = earlier in list
-            confidences = traverse_obj(metadata, ('tags', ..., 'confidence', ({int}, {float})))
-            if confidences:
-                tags = [tag for _, tag in sorted(zip(confidences, tags), reverse=True)]
+        metadata = self._call_api('info', video_id, {'itemId': video_id}, note='Downloading tags')
+        tags = traverse_obj(metadata, ('tags', ..., 'tag', {str}))
+        # Sorted by "confidence", higher confidence = earlier in list
+        confidences = traverse_obj(metadata, ('tags', ..., 'confidence', ({int}, {float})))
+        if confidences:
+            tags = [tag for _, tag in sorted(zip(confidences, tags), reverse=True)]
 
         formats = traverse_obj(video_info, ('variants', ..., {
             'format_id': ('name', {str}),

From 5dda3b291f59f388f953337e9fb09a94b64aaf34 Mon Sep 17 00:00:00 2001
From: Caesim404 <caesim404@gmail.com>
Date: Sun, 28 Jan 2024 04:02:09 +0200
Subject: [PATCH 812/871] [ie/lsm,cloudycdn] Add extractors (#8643)

Closes #2978
Authored by: Caesim404
---
 yt_dlp/extractor/_extractors.py |   6 +
 yt_dlp/extractor/cloudycdn.py   |  79 +++++++++
 yt_dlp/extractor/lsm.py         | 282 ++++++++++++++++++++++++++++++++
 3 files changed, 367 insertions(+)
 create mode 100644 yt_dlp/extractor/cloudycdn.py
 create mode 100644 yt_dlp/extractor/lsm.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index f51045668..09565055c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -369,6 +369,7 @@
 from .cliprs import ClipRsIE
 from .closertotruth import CloserToTruthIE
 from .cloudflarestream import CloudflareStreamIE
+from .cloudycdn import CloudyCDNIE
 from .clubic import ClubicIE
 from .clyp import ClypIE
 from .cmt import CMTIE
@@ -1001,6 +1002,11 @@
     LRTVODIE,
     LRTStreamIE
 )
+from .lsm import (
+    LSMLREmbedIE,
+    LSMLTVEmbedIE,
+    LSMReplayIE
+)
 from .lumni import (
     LumniIE
 )
diff --git a/yt_dlp/extractor/cloudycdn.py b/yt_dlp/extractor/cloudycdn.py
new file mode 100644
index 000000000..e6e470e07
--- /dev/null
+++ b/yt_dlp/extractor/cloudycdn.py
@@ -0,0 +1,79 @@
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    url_or_none,
+    urlencode_postdata,
+)
+from ..utils.traversal import traverse_obj
+
+
+class CloudyCDNIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//embed\.cloudycdn\.services/(?P<site_id>[^/?#]+)/media/(?P<id>[\w-]+)'
+    _EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL})']
+    _TESTS = [{
+        'url': 'https://embed.cloudycdn.services/ltv/media/46k_d23-6000-105?',
+        'md5': '64f72a360ca530d5ed89c77646c9eee5',
+        'info_dict': {
+            'id': '46k_d23-6000-105',
+            'ext': 'mp4',
+            'timestamp': 1700589151,
+            'duration': 1442,
+            'upload_date': '20231121',
+            'title': 'D23-6000-105_cetstud',
+            'thumbnail': 'https://store.cloudycdn.services/tmsp00060/assets/media/660858/placeholder1700589200.jpg',
+        }
+    }, {
+        'url': 'https://embed.cloudycdn.services/izm/media/26e_lv-8-5-1',
+        'md5': '798828a479151e2444d8dcfbec76e482',
+        'info_dict': {
+            'id': '26e_lv-8-5-1',
+            'ext': 'mp4',
+            'title': 'LV-8-5-1',
+            'timestamp': 1669767167,
+            'thumbnail': 'https://store.cloudycdn.services/tmsp00120/assets/media/488306/placeholder1679423604.jpg',
+            'duration': 1205,
+            'upload_date': '20221130',
+        }
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://www.tavaklase.lv/video/es-esmu-mina-um-2/',
+        'md5': '63074e8e6c84ac2a01f2fb8bf03b8f43',
+        'info_dict': {
+            'id': 'cqd_lib-2',
+            'ext': 'mp4',
+            'upload_date': '20230223',
+            'duration': 629,
+            'thumbnail': 'https://store.cloudycdn.services/tmsp00120/assets/media/518407/placeholder1678748124.jpg',
+            'timestamp': 1677181513,
+            'title': 'LIB-2',
+        }
+    }]
+
+    def _real_extract(self, url):
+        site_id, video_id = self._match_valid_url(url).group('site_id', 'id')
+
+        data = self._download_json(
+            f'https://player.cloudycdn.services/player/{site_id}/media/{video_id}/',
+            video_id, data=urlencode_postdata({
+                'version': '6.4.0',
+                'referer': url,
+            }))
+
+        formats, subtitles = [], {}
+        for m3u8_url in traverse_obj(data, ('source', 'sources', ..., 'src', {url_or_none})):
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, fatal=False)
+            formats.extend(fmts)
+            self._merge_subtitles(subs, target=subtitles)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(data, {
+                'title': ('name', {str}),
+                'duration': ('duration', {int_or_none}),
+                'timestamp': ('upload_date', {parse_iso8601}),
+                'thumbnail': ('source', 'poster', {url_or_none}),
+            }),
+        }
diff --git a/yt_dlp/extractor/lsm.py b/yt_dlp/extractor/lsm.py
new file mode 100644
index 000000000..35a831fa2
--- /dev/null
+++ b/yt_dlp/extractor/lsm.py
@@ -0,0 +1,282 @@
+import re
+import urllib.parse
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    parse_iso8601,
+    parse_qs,
+    str_or_none,
+    url_or_none,
+    urljoin,
+)
+from ..utils.traversal import traverse_obj
+
+
+class LSMLREmbedIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+        https?://(?:
+            (?:latvijasradio|lr1|lr2|klasika|lr4|naba|radioteatris)\.lsm|
+            pieci
+        )\.lv/[^/?#]+/(?:
+            pleijeris|embed
+        )/?\?(?:[^#]+&)?(?:show|id)=(?P<id>\d+)'''
+    _TESTS = [{
+        'url': 'https://latvijasradio.lsm.lv/lv/embed/?theme=black&size=16x9&showCaptions=0&id=183522',
+        'md5': '719b33875cd1429846eeeaeec6df2830',
+        'info_dict': {
+            'id': 'a342781',
+            'ext': 'mp3',
+            'duration': 1823,
+            'title': '#138 Nepilnīgā kompensējamo zāļu sistēma pat mēnešiem dzenā pacientus pa aptiekām',
+            'thumbnail': 'https://pic.latvijasradio.lv/public/assets/media/9/d/gallery_fd4675ac.jpg',
+        }
+    }, {
+        'url': 'https://radioteatris.lsm.lv/lv/embed/?id=&show=1270&theme=white&size=16x9',
+        'info_dict': {
+            'id': '1270',
+        },
+        'playlist_count': 3,
+        'playlist': [{
+            'md5': '2e61b6eceff00d14d57fdbbe6ab24cac',
+            'info_dict': {
+                'id': 'a297397',
+                'ext': 'mp3',
+                'title': 'Eriks Emanuels Šmits "Pilāta evaņģēlijs". 1. daļa',
+                'thumbnail': 'https://radioteatris.lsm.lv/public/assets/shows/62f131ae81e3c.jpg',
+                'duration': 3300,
+            },
+        }],
+    }, {
+        'url': 'https://radioteatris.lsm.lv/lv/embed/?id=&show=1269&theme=white&size=16x9',
+        'md5': '24810d4a961da2295d9860afdcaf4f5a',
+        'info_dict': {
+            'id': 'a230690',
+            'ext': 'mp3',
+            'title': 'Jens Ahlboms "Spārni". Radioizrāde ar Mārtiņa Freimaņa mūziku',
+            'thumbnail': 'https://radioteatris.lsm.lv/public/assets/shows/62f13023a457c.jpg',
+            'duration': 1788,
+        }
+    }, {
+        'url': 'https://lr1.lsm.lv/lv/embed/?id=166557&show=0&theme=white&size=16x9',
+        'info_dict': {
+            'id': '166557',
+        },
+        'playlist_count': 2,
+        'playlist': [{
+            'md5': '6a8b0927572f443f09c6e50a3ad65f2d',
+            'info_dict': {
+                'id': 'a303104',
+                'ext': 'mp3',
+                'thumbnail': 'https://pic.latvijasradio.lv/public/assets/media/c/5/gallery_a83ad2c2.jpg',
+                'title': 'Krustpunktā Lielā intervija: Valsts prezidents Egils Levits',
+                'duration': 3222,
+            },
+        }, {
+            'md5': '5d5e191e718b7644e5118b7b4e093a6d',
+            'info_dict': {
+                'id': 'v303104',
+                'ext': 'mp4',
+                'thumbnail': 'https://pic.latvijasradio.lv/public/assets/media/c/5/gallery_a83ad2c2.jpg',
+                'title': 'Krustpunktā Lielā intervija: Valsts prezidents Egils Levits - Video Version',
+                'duration': 3222,
+            },
+        }],
+    }, {
+        'url': 'https://lr1.lsm.lv/lv/embed/?id=183522&show=0&theme=white&size=16x9',
+        'only_matching': True,
+    }, {
+        'url': 'https://lr2.lsm.lv/lv/embed/?id=182126&show=0&theme=white&size=16x9',
+        'only_matching': True,
+    }, {
+        'url': 'https://klasika.lsm.lv/lv/embed/?id=110806&show=0&theme=white&size=16x9',
+        'only_matching': True,
+    }, {
+        'url': 'https://lr4.lsm.lv/lv/embed/?id=184282&show=0&theme=white&size=16x9',
+        'only_matching': True,
+    }, {
+        'url': 'https://pieci.lv/lv/embed/?id=168896&show=0&theme=white&size=16x9',
+        'only_matching': True,
+    }, {
+        'url': 'https://naba.lsm.lv/lv/embed/?id=182901&show=0&theme=white&size=16x9',
+        'only_matching': True,
+    }, {
+        'url': 'https://radioteatris.lsm.lv/lv/embed/?id=176439&show=0&theme=white&size=16x9',
+        'only_matching': True,
+    }, {
+        'url': 'https://lr1.lsm.lv/lv/pleijeris/?embed=0&id=48205&time=00%3A00&idx=0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        query = parse_qs(url)
+        video_id = traverse_obj(query, (
+            ('show', 'id'), 0, {int_or_none}, {lambda x: x or None}, {str_or_none}), get_all=False)
+        webpage = self._download_webpage(url, video_id)
+
+        player_data, media_data = self._search_regex(
+            r'LR\.audio\.Player\s*\([^{]*(?P<player>\{.*?\}),(?P<media>\{.*\})\);',
+            webpage, 'player json', group=('player', 'media'))
+
+        player_json = self._parse_json(
+            player_data, video_id, transform_source=js_to_json, fatal=False) or {}
+        media_json = self._parse_json(media_data, video_id, transform_source=js_to_json)
+
+        entries = []
+        for item in traverse_obj(media_json, (('audio', 'video'), lambda _, v: v['id'])):
+            formats = []
+            for source_url in traverse_obj(item, ('sources', ..., 'file', {url_or_none})):
+                if determine_ext(source_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(source_url, video_id, fatal=False))
+                else:
+                    formats.append({'url': source_url})
+
+            id_ = item['id']
+            title = item.get('title')
+            if id_.startswith('v') and not title:
+                title = traverse_obj(
+                    media_json, ('audio', lambda _, v: v['id'][1:] == id_[1:], 'title',
+                                 {lambda x: x and f'{x} - Video Version'}), get_all=False)
+
+            entries.append({
+                'formats': formats,
+                'thumbnail': urljoin(url, player_json.get('poster')),
+                'id': id_,
+                'title': title,
+                'duration': traverse_obj(item, ('duration', {int_or_none})),
+            })
+
+        if len(entries) == 1:
+            return entries[0]
+
+        return self.playlist_result(entries, video_id)
+
+
+class LSMLTVEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://ltv\.lsm\.lv/embed\?(?:[^#]+&)?c=(?P<id>[^#&]+)'
+    _TESTS = [{
+        'url': 'https://ltv.lsm.lv/embed?c=eyJpdiI6IjQzbHVUeHAyaDJiamFjcjdSUUFKdnc9PSIsInZhbHVlIjoiMHl3SnJNRmd2TmFIdnZwOGtGUUpzODFzUEZ4SVVsN2xoRjliSW9vckUyMWZIWG8vbWVzaFFkY0lhNmRjbjRpaCIsIm1hYyI6ImMzNjdhMzFhNTFhZmY1ZmE0NWI5YmFjZGI1YmJiNGEyNjgzNDM4MjUzMWEwM2FmMDMyZDMwYWM1MDFjZmM5MGIiLCJ0YWciOiIifQ==',
+        'md5': '64f72a360ca530d5ed89c77646c9eee5',
+        'info_dict': {
+            'id': '46k_d23-6000-105',
+            'ext': 'mp4',
+            'timestamp': 1700589151,
+            'duration': 1442,
+            'upload_date': '20231121',
+            'title': 'D23-6000-105_cetstud',
+            'thumbnail': 'https://store.cloudycdn.services/tmsp00060/assets/media/660858/placeholder1700589200.jpg',
+        }
+    }, {
+        'url': 'https://ltv.lsm.lv/embed?enablesdkjs=1&c=eyJpdiI6IncwVzZmUFk2MU12enVWK1I3SUcwQ1E9PSIsInZhbHVlIjoid3FhV29vamc3T2sxL1RaRmJ5Rm1GTXozU0o2dVczdUtLK0cwZEZJMDQ2a3ZIRG5DK2pneGlnbktBQy9uazVleHN6VXhxdWIweWNvcHRDSnlISlNYOHlVZ1lpcTUrcWZSTUZPQW14TVdkMW9aOUtRWVNDcFF4eWpHNGcrT0VZbUNFQStKQk91cGpndW9FVjJIa0lpbkh3PT0iLCJtYWMiOiIyZGI1NDJlMWRlM2QyMGNhOGEwYTM2MmNlN2JlOGRhY2QyYjdkMmEzN2RlOTEzYTVkNzI1ODlhZDlhZjU4MjQ2IiwidGFnIjoiIn0=',
+        'md5': 'a1711e190fe680fdb68fd8413b378e87',
+        'info_dict': {
+            'id': 'wUnFArIPDSY',
+            'ext': 'mp4',
+            'uploader': 'LTV_16plus',
+            'release_date': '20220514',
+            'channel_url': 'https://www.youtube.com/channel/UCNMrnafwXD2XKeeQOyfkFCw',
+            'view_count': int,
+            'availability': 'public',
+            'thumbnail': 'https://i.ytimg.com/vi/wUnFArIPDSY/maxresdefault.jpg',
+            'release_timestamp': 1652544074,
+            'title': 'EIROVĪZIJA SALĀTOS',
+            'live_status': 'was_live',
+            'uploader_id': '@LTV16plus',
+            'comment_count': int,
+            'channel_id': 'UCNMrnafwXD2XKeeQOyfkFCw',
+            'channel_follower_count': int,
+            'categories': ['Entertainment'],
+            'duration': 5269,
+            'upload_date': '20220514',
+            'age_limit': 0,
+            'channel': 'LTV_16plus',
+            'playable_in_embed': True,
+            'tags': [],
+            'uploader_url': 'https://www.youtube.com/@LTV16plus',
+            'like_count': int,
+            'description': 'md5:7ff0c42ba971e3c13e4b8a2ff03b70b5',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = urllib.parse.unquote(self._match_id(url))
+        webpage = self._download_webpage(url, video_id)
+        data = self._search_json(
+            r'window\.ltvEmbedPayload\s*=', webpage, 'embed json', video_id)
+        embed_type = traverse_obj(data, ('source', 'name', {str}))
+
+        if embed_type == 'telia':
+            ie_key = 'CloudyCDN'
+            embed_url = traverse_obj(data, ('source', 'embed_url', {url_or_none}))
+        elif embed_type == 'youtube':
+            ie_key = 'Youtube'
+            embed_url = traverse_obj(data, ('source', 'id', {str}))
+        else:
+            raise ExtractorError(f'Unsupported embed type {embed_type!r}')
+
+        return self.url_result(
+            embed_url, ie_key, video_id, **traverse_obj(data, {
+                'title': ('parentInfo', 'title'),
+                'duration': ('parentInfo', 'duration', {int_or_none}),
+                'thumbnail': ('source', 'poster', {url_or_none}),
+            }))
+
+
+class LSMReplayIE(InfoExtractor):
+    _VALID_URL = r'https?://replay\.lsm\.lv/[^/?#]+/(?:ieraksts|statja)/[^/?#]+/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://replay.lsm.lv/lv/ieraksts/ltv/311130/4-studija-zolitudes-tragedija-un-incupes-stacija',
+        'md5': '64f72a360ca530d5ed89c77646c9eee5',
+        'info_dict': {
+            'id': '46k_d23-6000-105',
+            'ext': 'mp4',
+            'timestamp': 1700586300,
+            'description': 'md5:0f1b14798cc39e1ae578bd0eb268f759',
+            'duration': 1442,
+            'upload_date': '20231121',
+            'title': '4. studija. Zolitūdes traģēdija un Inčupes stacija',
+            'thumbnail': 'https://ltv.lsm.lv/storage/media/8/7/large/5/1f9604e1.jpg',
+        }
+    }, {
+        'url': 'https://replay.lsm.lv/lv/ieraksts/lr/183522/138-nepilniga-kompensejamo-zalu-sistema-pat-menesiem-dzena-pacientus-pa-aptiekam',
+        'md5': '719b33875cd1429846eeeaeec6df2830',
+        'info_dict': {
+            'id': 'a342781',
+            'ext': 'mp3',
+            'duration': 1823,
+            'title': '#138 Nepilnīgā kompensējamo zāļu sistēma pat mēnešiem dzenā pacientus pa aptiekām',
+            'thumbnail': 'https://pic.latvijasradio.lv/public/assets/media/9/d/large_fd4675ac.jpg',
+            'upload_date': '20231102',
+            'timestamp': 1698921060,
+            'description': 'md5:7bac3b2dd41e44325032943251c357b1',
+        }
+    }, {
+        'url': 'https://replay.lsm.lv/ru/statja/ltv/311130/4-studija-zolitudes-tragedija-un-incupes-stacija',
+        'only_matching': True,
+    }]
+
+    def _fix_nuxt_data(self, webpage):
+        return re.sub(r'Object\.create\(null(?:,(\{.+\}))?\)', lambda m: m.group(1) or 'null', webpage)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        data = self._search_nuxt_data(
+            self._fix_nuxt_data(webpage), video_id, context_name='__REPLAY__')
+
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            **traverse_obj(data, {
+                'url': ('playback', 'service', 'url', {url_or_none}),
+                'title': ('mediaItem', 'title'),
+                'description': ('mediaItem', ('lead', 'body')),
+                'duration': ('mediaItem', 'duration', {int_or_none}),
+                'timestamp': ('mediaItem', 'aired_at', {parse_iso8601}),
+                'thumbnail': ('mediaItem', 'largeThumbnail', {url_or_none}),
+            }, get_all=False),
+        }

From d79c7e9937c388c68b722ab7450960e43ef776d6 Mon Sep 17 00:00:00 2001
From: shmohawk <hawkshmo@gmail.com>
Date: Sun, 28 Jan 2024 03:10:20 +0100
Subject: [PATCH 813/871] [ie/Txxx] Extract thumbnails (#9063)

Authored by: shmohawk
---
 yt_dlp/extractor/txxx.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/yt_dlp/extractor/txxx.py b/yt_dlp/extractor/txxx.py
index fff7a5d76..77dabbc82 100644
--- a/yt_dlp/extractor/txxx.py
+++ b/yt_dlp/extractor/txxx.py
@@ -10,6 +10,7 @@
     parse_duration,
     traverse_obj,
     try_call,
+    url_or_none,
     urljoin,
     variadic,
 )
@@ -83,6 +84,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.txxx.tube/contents/videos_sources/16574000/16574965/screenshots/1.jpg',
         }
     }, {
         'url': 'https://txxx.tube/videos/16574965/digital-desire-malena-morgan/',
@@ -98,6 +100,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.txxx.tube/contents/videos_sources/16574000/16574965/screenshots/1.jpg',
         }
     }, {
         'url': 'https://vxxx.com/video-68925/',
@@ -113,6 +116,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.vxxx.com/contents/videos_sources/68000/68925/screenshots/1.jpg',
         }
     }, {
         'url': 'https://hclips.com/videos/6291073/malena-morgan-masturbates-her-sweet/',
@@ -128,6 +132,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://hctn.nv7s.com/contents/videos_sources/6291000/6291073/screenshots/1.jpg',
         }
     }, {
         'url': 'https://hdzog.com/videos/67063/gorgeous-malena-morgan-will-seduce-you-at-the-first-glance/',
@@ -143,6 +148,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.hdzog.com/contents/videos_sources/67000/67063/screenshots/1.jpg',
         }
     }, {
         'url': 'https://hdzog.tube/videos/67063/gorgeous-malena-morgan-will-seduce-you-at-the-first-glance/',
@@ -158,6 +164,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.hdzog.com/contents/videos_sources/67000/67063/screenshots/1.jpg',
         }
     }, {
         'url': 'https://hotmovs.com/videos/8789287/unbelievable-malena-morgan-performing-in-incredible-masturantion/',
@@ -173,6 +180,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.hotmovs.com/contents/videos_sources/8789000/8789287/screenshots/10.jpg',
         }
     }, {
         'url': 'https://hotmovs.tube/videos/8789287/unbelievable-malena-morgan-performing-in-incredible-masturantion/',
@@ -188,6 +196,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.hotmovs.com/contents/videos_sources/8789000/8789287/screenshots/10.jpg',
         }
     }, {
         'url': 'https://inporn.com/video/517897/malena-morgan-solo/',
@@ -203,6 +212,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://iptn.m3pd.com/media/tn/sources/517897_1.jpg',
         }
     }, {
         'url': 'https://privatehomeclips.com/videos/3630599/malena-morgan-cam-show/',
@@ -218,6 +228,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://hctn.nv7s.com/contents/videos_sources/3630000/3630599/screenshots/15.jpg',
         }
     }, {
         'url': 'https://tubepornclassic.com/videos/1015455/mimi-rogers-full-body-massage-nude-compilation/',
@@ -233,6 +244,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.tubepornclassic.com/contents/videos_sources/1015000/1015455/screenshots/6.jpg',
         }
     }, {
         'url': 'https://upornia.com/videos/1498858/twistys-malena-morgan-starring-at-dr-morgan-baller/',
@@ -248,6 +260,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.upornia.com/contents/videos_sources/1498000/1498858/screenshots/1.jpg',
         }
     }, {
         'url': 'https://upornia.tube/videos/1498858/twistys-malena-morgan-starring-at-dr-morgan-baller/',
@@ -263,6 +276,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.upornia.com/contents/videos_sources/1498000/1498858/screenshots/1.jpg',
         }
     }, {
         'url': 'https://vjav.com/videos/11761/yui-hatano-in-if-yui-was-my-girlfriend2/',
@@ -278,6 +292,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.vjav.com/contents/videos_sources/11000/11761/screenshots/23.jpg',
         }
     }, {
         'url': 'https://vjav.tube/videos/11761/yui-hatano-in-if-yui-was-my-girlfriend2/',
@@ -293,6 +308,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.vjav.com/contents/videos_sources/11000/11761/screenshots/23.jpg',
         }
     }, {
         'url': 'https://voyeurhit.com/videos/332875/charlotte-stokely-elle-alexandra-malena-morgan-lingerie/',
@@ -308,6 +324,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.voyeurhit.com/contents/videos_sources/332000/332875/screenshots/1.jpg',
         }
     }, {
         'url': 'https://voyeurhit.tube/videos/332875/charlotte-stokely-elle-alexandra-malena-morgan-lingerie/',
@@ -323,6 +340,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://tn.voyeurhit.com/contents/videos_sources/332000/332875/screenshots/1.jpg',
         }
     }]
     _WEBPAGE_TESTS = [{
@@ -338,6 +356,7 @@ class TxxxIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'age_limit': 18,
+            'thumbnail': 'https://hctn.nv7s.com/contents/videos_sources/5119000/5119660/screenshots/1.jpg',
         }
     }]
 
@@ -371,6 +390,7 @@ def _real_extract(self, url):
             'like_count': int_or_none(traverse_obj(video_info, ('video', 'statistics', 'likes'))),
             'dislike_count': int_or_none(traverse_obj(video_info, ('video', 'statistics', 'dislikes'))),
             'age_limit': 18,
+            'thumbnail': traverse_obj(video_info, ('video', 'thumbsrc', {url_or_none})),
             'formats': get_formats(host, video_file),
         }
 

From 77c2472ca1ef9050a66aa68bc5fa1bee88706c66 Mon Sep 17 00:00:00 2001
From: jazz1611 <jazz1611@users.noreply.github.com>
Date: Sun, 28 Jan 2024 09:12:40 +0700
Subject: [PATCH 814/871] [ie/Gofile] Fix extraction (#9074)

Closes #9073
Authored by: jazz1611
---
 yt_dlp/extractor/gofile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/gofile.py b/yt_dlp/extractor/gofile.py
index ef14b57d0..eb1dcf85f 100644
--- a/yt_dlp/extractor/gofile.py
+++ b/yt_dlp/extractor/gofile.py
@@ -66,7 +66,7 @@ def _entries(self, file_id):
         query_params = {
             'contentId': file_id,
             'token': self._TOKEN,
-            'websiteToken': '7fd94ds12fds4',  # From https://gofile.io/dist/js/alljs.js
+            'wt': '4fd6sg89d7s6',  # From https://gofile.io/dist/js/alljs.js
         }
         password = self.get_param('videopassword')
         if password:

From c91d8b1899403daff6fc15206ad32de8db17fb8f Mon Sep 17 00:00:00 2001
From: jazz1611 <jazz1611@users.noreply.github.com>
Date: Sun, 28 Jan 2024 09:15:29 +0700
Subject: [PATCH 815/871] [ie/redtube] Fix formats extraction (#9076)

Authored by: jazz1611
---
 yt_dlp/extractor/redtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/redtube.py b/yt_dlp/extractor/redtube.py
index 172c31b39..36d530daf 100644
--- a/yt_dlp/extractor/redtube.py
+++ b/yt_dlp/extractor/redtube.py
@@ -7,6 +7,7 @@
     str_to_int,
     unified_strdate,
     url_or_none,
+    urljoin,
 )
 
 
@@ -79,7 +80,7 @@ def _real_extract(self, url):
                 'media definitions', default='{}'),
             video_id, fatal=False)
         for media in medias if isinstance(medias, list) else []:
-            format_url = url_or_none(media.get('videoUrl'))
+            format_url = urljoin('https://www.redtube.com', media.get('videoUrl'))
             if not format_url:
                 continue
             format_id = media.get('format')

From cae6e461073fb7c32fd32052a3e6721447c469bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=BCndig?= <christian@kuendig.info>
Date: Sun, 28 Jan 2024 03:19:54 +0100
Subject: [PATCH 816/871] [ie/PlaySuisse] Add login support (#9077)

Closes #7974
Authored by: chkuendig
---
 yt_dlp/extractor/playsuisse.py | 53 ++++++++++++++++++++++++++++++++--
 1 file changed, 50 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/playsuisse.py b/yt_dlp/extractor/playsuisse.py
index 76288c778..7c5cad1be 100644
--- a/yt_dlp/extractor/playsuisse.py
+++ b/yt_dlp/extractor/playsuisse.py
@@ -1,10 +1,18 @@
 import json
 
 from .common import InfoExtractor
-from ..utils import int_or_none, traverse_obj
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_qs,
+    traverse_obj,
+    update_url_query,
+    urlencode_postdata,
+)
 
 
 class PlaySuisseIE(InfoExtractor):
+    _NETRC_MACHINE = 'playsuisse'
     _VALID_URL = r'https?://(?:www\.)?playsuisse\.ch/(?:watch|detail)/(?:[^#]*[?&]episodeId=)?(?P<id>[0-9]+)'
     _TESTS = [
         {
@@ -134,12 +142,47 @@ class PlaySuisseIE(InfoExtractor):
             id
             url
         }'''
+    _LOGIN_BASE_URL = 'https://login.srgssr.ch/srgssrlogin.onmicrosoft.com'
+    _LOGIN_PATH = 'B2C_1A__SignInV2'
+    _ID_TOKEN = None
+
+    def _perform_login(self, username, password):
+        login_page = self._download_webpage(
+            'https://www.playsuisse.ch/api/sso/login', None, note='Downloading login page',
+            query={'x': 'x', 'locale': 'de', 'redirectUrl': 'https://www.playsuisse.ch/'})
+        settings = self._search_json(r'var\s+SETTINGS\s*=', login_page, 'settings', None)
+
+        csrf_token = settings['csrf']
+        query = {'tx': settings['transId'], 'p': self._LOGIN_PATH}
+
+        status = traverse_obj(self._download_json(
+            f'{self._LOGIN_BASE_URL}/{self._LOGIN_PATH}/SelfAsserted', None, 'Logging in',
+            query=query, headers={'X-CSRF-TOKEN': csrf_token}, data=urlencode_postdata({
+                'request_type': 'RESPONSE',
+                'signInName': username,
+                'password': password
+            }), expected_status=400), ('status', {int_or_none}))
+        if status == 400:
+            raise ExtractorError('Invalid username or password', expected=True)
+
+        urlh = self._request_webpage(
+            f'{self._LOGIN_BASE_URL}/{self._LOGIN_PATH}/api/CombinedSigninAndSignup/confirmed',
+            None, 'Downloading ID token', query={
+                'rememberMe': 'false',
+                'csrf_token': csrf_token,
+                **query,
+                'diags': '',
+            })
+
+        self._ID_TOKEN = traverse_obj(parse_qs(urlh.url), ('id_token', 0))
+        if not self._ID_TOKEN:
+            raise ExtractorError('Login failed')
 
     def _get_media_data(self, media_id):
         # NOTE In the web app, the "locale" header is used to switch between languages,
         # However this doesn't seem to take effect when passing the header here.
         response = self._download_json(
-            'https://4bbepzm4ef.execute-api.eu-central-1.amazonaws.com/prod/graphql',
+            'https://www.playsuisse.ch/api/graphql',
             media_id, data=json.dumps({
                 'operationName': 'AssetWatch',
                 'query': self._GRAPHQL_QUERY,
@@ -150,6 +193,9 @@ def _get_media_data(self, media_id):
         return response['data']['assetV2']
 
     def _real_extract(self, url):
+        if not self._ID_TOKEN:
+            self.raise_login_required(method='password')
+
         media_id = self._match_id(url)
         media_data = self._get_media_data(media_id)
         info = self._extract_single(media_data)
@@ -168,7 +214,8 @@ def _extract_single(self, media_data):
             if not media.get('url') or media.get('type') != 'HLS':
                 continue
             f, subs = self._extract_m3u8_formats_and_subtitles(
-                media['url'], media_data['id'], 'mp4', m3u8_id='HLS', fatal=False)
+                update_url_query(media['url'], {'id_token': self._ID_TOKEN}),
+                media_data['id'], 'mp4', m3u8_id='HLS', fatal=False)
             formats.extend(f)
             self._merge_subtitles(subs, target=subtitles)
 

From 0023af81fbce01984f35b34ecaf8562739831227 Mon Sep 17 00:00:00 2001
From: vista-narvas <vista-narvas@users.noreply.github.com>
Date: Sun, 28 Jan 2024 16:32:19 +0100
Subject: [PATCH 817/871] [ie/RumbleChannel] Fix extractor (#9092)

Closes #8782
Authored by: vista-narvas, Pranaxcau
---
 yt_dlp/extractor/rumble.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py
index 85567d9a2..1dc049ac8 100644
--- a/yt_dlp/extractor/rumble.py
+++ b/yt_dlp/extractor/rumble.py
@@ -383,7 +383,7 @@ def entries(self, url, playlist_id):
                 if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                     break
                 raise
-            for video_url in re.findall(r'class=video-item--a\s?href=([^>]+\.html)', webpage):
+            for video_url in re.findall(r'class="[^>"]*videostream__link[^>]+href="([^"]+\.html)"', webpage):
                 yield self.url_result('https://rumble.com' + video_url)
 
     def _real_extract(self, url):

From 9526b1f179d19f75284eceaa5e0ee381af18cf19 Mon Sep 17 00:00:00 2001
From: Christopher Schreiner <git@infanf.com>
Date: Sun, 28 Jan 2024 17:03:19 +0100
Subject: [PATCH 818/871] [ie/adn] Improve auth error handling (#9068)

Closes #9067
Authored by: infanf
---
 yt_dlp/extractor/adn.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/adn.py b/yt_dlp/extractor/adn.py
index ed23226a3..898d37298 100644
--- a/yt_dlp/extractor/adn.py
+++ b/yt_dlp/extractor/adn.py
@@ -3,6 +3,7 @@
 import json
 import os
 import random
+import time
 
 from .common import InfoExtractor
 from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
@@ -17,6 +18,7 @@
     int_or_none,
     intlist_to_bytes,
     long_to_bytes,
+    parse_iso8601,
     pkcs1pad,
     strip_or_none,
     str_or_none,
@@ -185,7 +187,10 @@ def _real_extract(self, url):
 
         user = options['user']
         if not user.get('hasAccess'):
-            self.raise_login_required()
+            start_date = traverse_obj(options, ('video', 'startDate', {str}))
+            if (parse_iso8601(start_date) or 0) > time.time():
+                raise ExtractorError(f'This video is not available yet. Release date: {start_date}', expected=True)
+            self.raise_login_required('This video requires a subscription', method='password')
 
         token = self._download_json(
             user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),
@@ -267,6 +272,9 @@ def _real_extract(self, url):
                         f['language'] = 'de'
                 formats.extend(m3u8_formats)
 
+        if not formats:
+            self.raise_login_required('This video requires a subscription', method='password')
+
         video = (self._download_json(
             self._API_BASE_URL + 'video/%s' % video_id, video_id,
             'Downloading additional video metadata', fatal=False) or {}).get('video') or {}

From 5b68c478fb0b93ea6b8fac23f50e12217fa063db Mon Sep 17 00:00:00 2001
From: kclauhk <78251477+kclauhk@users.noreply.github.com>
Date: Mon, 29 Jan 2024 02:39:14 +0800
Subject: [PATCH 819/871] [ie/facebook] Set format HTTP chunk size (#9058)

Closes #8197
Authored by: bashonly, kclauhk
---
 yt_dlp/extractor/facebook.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 26cfda538..84856abe1 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -564,7 +564,11 @@ def process_formats(info):
             # Downloads with browser's User-Agent are rate limited. Working around
             # with non-browser User-Agent.
             for f in info['formats']:
+                # Downloads with browser's User-Agent are rate limited. Working around
+                # with non-browser User-Agent.
                 f.setdefault('http_headers', {})['User-Agent'] = 'facebookexternalhit/1.1'
+                # Formats larger than ~500MB will return error 403 unless chunk size is regulated
+                f.setdefault('downloader_options', {})['http_chunk_size'] = 250 << 20
 
         def extract_relay_data(_filter):
             return self._parse_json(self._search_regex(

From 3c4d3ee491b0ec22ed3cade51d943d3d27141ba7 Mon Sep 17 00:00:00 2001
From: kclauhk <78251477+kclauhk@users.noreply.github.com>
Date: Mon, 29 Jan 2024 02:41:56 +0800
Subject: [PATCH 820/871] [ie/facebook] Improve thumbnail extraction (#9060)

Authored by: kclauhk
---
 yt_dlp/extractor/facebook.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 84856abe1..2fbdf1c37 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -682,6 +682,9 @@ def parse_attachment(attachment, key='media'):
                 # honor precise duration in video info
                 if video_info.get('duration'):
                     webpage_info['duration'] = video_info['duration']
+                # preserve preferred_thumbnail in video info
+                if video_info.get('thumbnail'):
+                    webpage_info['thumbnail'] = video_info['thumbnail']
                 return merge_dicts(webpage_info, video_info)
 
         if not video_data:

From 87286e93af949c4e6a0f8ba34af6a1ab5aa102b6 Mon Sep 17 00:00:00 2001
From: kclauhk <78251477+kclauhk@users.noreply.github.com>
Date: Mon, 29 Jan 2024 02:50:03 +0800
Subject: [PATCH 821/871] [ie/facebook] Support permalink URLs (#9061)

Authored by: kclauhk
---
 yt_dlp/extractor/facebook.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 2fbdf1c37..d186b57bf 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -44,6 +44,7 @@ class FacebookIE(InfoExtractor):
                         (?:[^#]*?\#!/)?
                         (?:
                             (?:
+                                permalink\.php|
                                 video/video\.php|
                                 photo\.php|
                                 video\.php|
@@ -249,6 +250,7 @@ class FacebookIE(InfoExtractor):
             'duration': 148.435,
         },
     }, {
+        # data.node.comet_sections.content.story.attachments[].styles.attachment.media
         'url': 'https://www.facebook.com/attn/posts/pfbid0j1Czf2gGDVqeQ8KiMLFm3pWN8GxsQmeRrVhimWDzMuKQoR8r4b1knNsejELmUgyhl',
         'info_dict': {
             'id': '6968553779868435',
@@ -263,6 +265,22 @@ class FacebookIE(InfoExtractor):
             'thumbnail': r're:^https?://.*',
             'timestamp': 1701975646,
         },
+    }, {
+        # data.node.comet_sections.content.story.attachments[].styles.attachment.media
+        'url': 'https://www.facebook.com/permalink.php?story_fbid=pfbid0fqQuVEQyXRa9Dp4RcaTR14KHU3uULHV1EK7eckNXSH63JMuoALsAvVCJ97zAGitil&id=100068861234290',
+        'info_dict': {
+            'id': '270103405756416',
+            'ext': 'mp4',
+            'title': 'Lela Evans',
+            'description': 'Today Makkovik\'s own Pilot Mandy Smith made her inaugural landing on the airstrip in her hometown. What a proud moment as we all cheered and...',
+            'thumbnail': r're:^https?://.*',
+            'uploader': 'Lela Evans',
+            'uploader_id': 'pfbid0shZJipuigyy5mqrUJn9ub5LJFWNHvan5prtyi3LrDuuuJ4NwrURgnQHYR9fywBepl',
+            'upload_date': '20231228',
+            'timestamp': 1703804085,
+            'duration': 394.347,
+            'view_count': int,
+        },
     }, {
         'url': 'https://www.facebook.com/story.php?story_fbid=pfbid0Fnzhm8UuzjBYpPMNFzaSpFE9UmLdU4fJN8qTANi1Dmtj5q7DNrL5NERXfsAzDEV7l&id=100073071055552',
         'only_matching': True,

From a514cc2feb1c3b265b19acab11487acad8bb3ab0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Sun, 28 Jan 2024 20:58:34 +0200
Subject: [PATCH 822/871] [ie/ERRJupiter] Add extractor (#8549)

Authored by: glensc
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/err.py         | 199 ++++++++++++++++++++++++++++++++
 2 files changed, 200 insertions(+)
 create mode 100644 yt_dlp/extractor/err.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 09565055c..2fc1e116b 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -565,6 +565,7 @@
     EroProfileIE,
     EroProfileAlbumIE,
 )
+from .err import ERRJupiterIE
 from .ertgr import (
     ERTFlixCodenameIE,
     ERTFlixIE,
diff --git a/yt_dlp/extractor/err.py b/yt_dlp/extractor/err.py
new file mode 100644
index 000000000..129f39ad6
--- /dev/null
+++ b/yt_dlp/extractor/err.py
@@ -0,0 +1,199 @@
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+    str_or_none,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class ERRJupiterIE(InfoExtractor):
+    _VALID_URL = r'https?://jupiter(?:pluss)?\.err\.ee/(?P<id>\d+)'
+    _TESTS = [{
+        'note': 'Jupiter: Movie: siin-me-oleme',
+        'url': 'https://jupiter.err.ee/1211107/siin-me-oleme',
+        'md5': '9b45d1682a98853acaa1e1b0c791f425',
+        'info_dict': {
+            'id': '1211107',
+            'ext': 'mp4',
+            'title': 'Siin me oleme!',
+            'alt_title': '',
+            'description': 'md5:1825b795f5f7584241aeb59e5bbb4f70',
+            'release_date': '20231226',
+            'upload_date': '20201217',
+            'modified_date': '20201217',
+            'release_timestamp': 1703577600,
+            'timestamp': 1608210000,
+            'modified_timestamp': 1608220800,
+            'release_year': 1978,
+        },
+    }, {
+        'note': 'Jupiter: Series: Impulss',
+        'url': 'https://jupiter.err.ee/1609145945/impulss',
+        'md5': 'a378486df07ed1ba74e46cc861886243',
+        'info_dict': {
+            'id': '1609145945',
+            'ext': 'mp4',
+            'title': 'Impulss',
+            'alt_title': 'Loteriipilet hooldekodusse',
+            'description': 'md5:fa8a2ed0cdccb130211513443ee4d571',
+            'release_date': '20231107',
+            'upload_date': '20231026',
+            'modified_date': '20231118',
+            'release_timestamp': 1699380000,
+            'timestamp': 1698327601,
+            'modified_timestamp': 1700311802,
+            'series': 'Impulss',
+            'season': 'Season 1',
+            'season_number': 1,
+            'episode': 'Loteriipilet hooldekodusse',
+            'episode_number': 6,
+            'series_id': '1609108187',
+            'release_year': 2023,
+            'episode_id': '1609145945',
+        },
+    }, {
+        'note': 'Jupiter: Radio Show: mnemoturniir episode',
+        'url': 'https://jupiter.err.ee/1037919/mnemoturniir',
+        'md5': 'f1eb95fe66f9620ff84e81bbac37076a',
+        'info_dict': {
+            'id': '1037919',
+            'ext': 'm4a',
+            'title': 'Mnemoturniir',
+            'alt_title': '',
+            'description': 'md5:626db52394e7583c26ab74d6a34d9982',
+            'release_date': '20240121',
+            'upload_date': '20240108',
+            'modified_date': '20240121',
+            'release_timestamp': 1705827900,
+            'timestamp': 1704675602,
+            'modified_timestamp': 1705827601,
+            'series': 'Mnemoturniir',
+            'season': 'Season 0',
+            'season_number': 0,
+            'episode': 'Episode 0',
+            'episode_number': 0,
+            'series_id': '1037919',
+            'release_year': 2024,
+            'episode_id': '1609215101',
+        },
+    }, {
+        'note': 'Jupiter+: Clip: bolee-zelenyj-tallinn',
+        'url': 'https://jupiterpluss.err.ee/1609180445/bolee-zelenyj-tallinn',
+        'md5': '1b812270c4daf6ce51c06bfeaf33ed95',
+        'info_dict': {
+            'id': '1609180445',
+            'ext': 'mp4',
+            'title': 'Более зеленый Таллинн',
+            'alt_title': '',
+            'description': 'md5:fd34d9bf939c28c4a725b19a7f0d6320',
+            'release_date': '20231224',
+            'upload_date': '20231130',
+            'modified_date': '20231207',
+            'release_timestamp': 1703423400,
+            'timestamp': 1701338400,
+            'modified_timestamp': 1701967200,
+            'release_year': 2023,
+        },
+    }, {
+        'note': 'Jupiter+: Series: The Sniffer',
+        'url': 'https://jupiterpluss.err.ee/1608311387/njuhach',
+        'md5': '2abdeb7131ce551bce49e8d0cea08536',
+        'info_dict': {
+            'id': '1608311387',
+            'ext': 'mp4',
+            'title': 'Нюхач',
+            'alt_title': '',
+            'description': 'md5:8c5c7d8f32ec6e54cd498c9e59ca83bc',
+            'release_date': '20230601',
+            'upload_date': '20210818',
+            'modified_date': '20210903',
+            'release_timestamp': 1685633400,
+            'timestamp': 1629318000,
+            'modified_timestamp': 1630686000,
+            'release_year': 2013,
+            'episode': 'Episode 1',
+            'episode_id': '1608311390',
+            'episode_number': 1,
+            'season': 'Season 1',
+            'season_number': 1,
+            'series': 'Нюхач',
+            'series_id': '1608311387',
+        },
+    }, {
+        'note': 'Jupiter+: Podcast: lesnye-istorii-aisty',
+        'url': 'https://jupiterpluss.err.ee/1608990335/lesnye-istorii-aisty',
+        'md5': '8b46d7e4510b254a14b7a52211b5bf96',
+        'info_dict': {
+            'id': '1608990335',
+            'ext': 'm4a',
+            'title': 'Лесные истории | Аисты',
+            'alt_title': '',
+            'description': 'md5:065e721623e271e7a63e6540d409ca6b',
+            'release_date': '20230609',
+            'upload_date': '20230527',
+            'modified_date': '20230608',
+            'release_timestamp': 1686308700,
+            'timestamp': 1685145600,
+            'modified_timestamp': 1686252600,
+            'release_year': 2023,
+            'episode': 'Episode 0',
+            'episode_id': '1608990335',
+            'episode_number': 0,
+            'season': 'Season 0',
+            'season_number': 0,
+            'series': 'Лесные истории | Аисты',
+            'series_id': '1037497',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._download_json(
+            'https://services.err.ee/api/v2/vodContent/getContentPageData', video_id,
+            query={'contentId': video_id})['data']['mainContent']
+
+        media_data = traverse_obj(data, ('medias', ..., {dict}), get_all=False)
+        if traverse_obj(media_data, ('restrictions', 'drm', {bool})):
+            self.report_drm(video_id)
+
+        formats, subtitles = [], {}
+        for format_url in set(traverse_obj(media_data, ('src', ('hls', 'hls2', 'hlsNew'), {url_or_none}))):
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                format_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+            formats.extend(fmts)
+            self._merge_subtitles(subs, target=subtitles)
+        for format_url in set(traverse_obj(media_data, ('src', ('dash', 'dashNew'), {url_or_none}))):
+            fmts, subs = self._extract_mpd_formats_and_subtitles(
+                format_url, video_id, mpd_id='dash', fatal=False)
+            formats.extend(fmts)
+            self._merge_subtitles(subs, target=subtitles)
+        if format_url := traverse_obj(media_data, ('src', 'file', {url_or_none})):
+            formats.append({
+                'url': format_url,
+                'format_id': 'http',
+            })
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(data, {
+                'title': ('heading', {str}),
+                'alt_title': ('subHeading', {str}),
+                'description': (('lead', 'body'), {clean_html}, {lambda x: x or None}),
+                'timestamp': ('created', {int_or_none}),
+                'modified_timestamp': ('updated', {int_or_none}),
+                'release_timestamp': (('scheduleStart', 'publicStart'), {int_or_none}),
+                'release_year': ('year', {int_or_none}),
+            }, get_all=False),
+            **(traverse_obj(data, {
+                'series': ('heading', {str}),
+                'series_id': ('rootContentId', {str_or_none}),
+                'episode': ('subHeading', {str}),
+                'season_number': ('season', {int_or_none}),
+                'episode_number': ('episode', {int_or_none}),
+                'episode_id': ('id', {str_or_none}),
+            }) if data.get('type') == 'episode' else {}),
+        }

From 02e343f6ef6d7b3f9087ff69e4a1db0b4b4a5c5d Mon Sep 17 00:00:00 2001
From: Danish Humair <me@danishhumair.com>
Date: Mon, 29 Jan 2024 02:23:52 +0500
Subject: [PATCH 823/871] [ie/MedalTV] Fix extraction (#9098)

Closes #8766
Authored by: Danish-H
---
 yt_dlp/extractor/medaltv.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/medaltv.py b/yt_dlp/extractor/medaltv.py
index 9e57ee21a..eeb5b85f3 100644
--- a/yt_dlp/extractor/medaltv.py
+++ b/yt_dlp/extractor/medaltv.py
@@ -8,7 +8,8 @@
     float_or_none,
     int_or_none,
     str_or_none,
-    traverse_obj
+    traverse_obj,
+    update_url_query,
 )
 
 
@@ -16,7 +17,7 @@ class MedalTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?medal\.tv/games/[^/?#&]+/clips/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://medal.tv/games/valorant/clips/jTBFnLKdLy15K',
-        'md5': '6930f8972914b6b9fdc2bb3918098ba0',
+        'md5': '03e4911fdcf7fce563090705c2e79267',
         'info_dict': {
             'id': 'jTBFnLKdLy15K',
             'ext': 'mp4',
@@ -33,8 +34,8 @@ class MedalTVIE(InfoExtractor):
             'duration': 13,
         }
     }, {
-        'url': 'https://medal.tv/games/cod%20cold%20war/clips/2mA60jWAGQCBH',
-        'md5': '3d19d426fe0b2d91c26e412684e66a06',
+        'url': 'https://medal.tv/games/cod-cold-war/clips/2mA60jWAGQCBH',
+        'md5': 'fc7a3e4552ae8993c1c4006db46be447',
         'info_dict': {
             'id': '2mA60jWAGQCBH',
             'ext': 'mp4',
@@ -52,7 +53,7 @@ class MedalTVIE(InfoExtractor):
             'duration': 23,
         }
     }, {
-        'url': 'https://medal.tv/games/cod%20cold%20war/clips/2um24TWdty0NA',
+        'url': 'https://medal.tv/games/cod-cold-war/clips/2um24TWdty0NA',
         'md5': 'b6dc76b78195fff0b4f8bf4a33ec2148',
         'info_dict': {
             'id': '2um24TWdty0NA',
@@ -81,7 +82,7 @@ class MedalTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(update_url_query(url, {'mobilebypass': 'true'}), video_id)
 
         hydration_data = self._search_json(
             r'<script[^>]*>[^<]*\bhydrationData\s*=', webpage,

From 41b6cdb4197aaf7ad82bdad6885eb5d5c64acd74 Mon Sep 17 00:00:00 2001
From: Nur Mahmud Ul Alam Tasin <62534505+NurTasin@users.noreply.github.com>
Date: Mon, 29 Jan 2024 04:33:44 +0600
Subject: [PATCH 824/871] [ie/viewlift] Add support for chorki.com (#9095)

Closes #3369
Authored by: NurTasin
---
 yt_dlp/extractor/viewlift.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/viewlift.py b/yt_dlp/extractor/viewlift.py
index 8f686f05d..c93be5f3d 100644
--- a/yt_dlp/extractor/viewlift.py
+++ b/yt_dlp/extractor/viewlift.py
@@ -12,7 +12,7 @@
 
 class ViewLiftBaseIE(InfoExtractor):
     _API_BASE = 'https://prod-api.viewlift.com/'
-    _DOMAINS_REGEX = r'(?:(?:main\.)?snagfilms|snagxtreme|funnyforfree|kiddovid|winnersview|(?:monumental|lax)sportsnetwork|vayafilm|failarmy|ftfnext|lnppass\.legapallacanestro|moviespree|app\.myoutdoortv|neoufitness|pflmma|theidentitytb)\.com|(?:hoichoi|app\.horseandcountry|kronon|marquee|supercrosslive)\.tv'
+    _DOMAINS_REGEX = r'(?:(?:main\.)?snagfilms|snagxtreme|funnyforfree|kiddovid|winnersview|(?:monumental|lax)sportsnetwork|vayafilm|failarmy|ftfnext|lnppass\.legapallacanestro|moviespree|app\.myoutdoortv|neoufitness|pflmma|theidentitytb|chorki)\.com|(?:hoichoi|app\.horseandcountry|kronon|marquee|supercrosslive)\.tv'
     _SITE_MAP = {
         'ftfnext': 'lax',
         'funnyforfree': 'snagfilms',
@@ -27,6 +27,7 @@ class ViewLiftBaseIE(InfoExtractor):
         'snagxtreme': 'snagfilms',
         'theidentitytb': 'tampabay',
         'vayafilm': 'snagfilms',
+        'chorki': 'prothomalo',
     }
     _TOKENS = {}
 
@@ -296,6 +297,33 @@ class ViewLiftIE(ViewLiftBaseIE):
     }, {  # Premium movie
         'url': 'https://www.hoichoi.tv/movies/detective-2020',
         'only_matching': True
+    }, {  # Chorki Premium series
+        'url': 'https://www.chorki.com/bn/series/sinpaat',
+        'playlist_mincount': 7,
+        'info_dict': {
+            'id': 'bn/series/sinpaat',
+        },
+    }, {  # Chorki free movie
+        'url': 'https://www.chorki.com/bn/videos/bangla-movie-bikkhov',
+        'info_dict': {
+            'id': '564e755b-f5c7-4515-aee6-8959bee18c93',
+            'title': 'Bikkhov',
+            'ext': 'mp4',
+            'upload_date': '20230824',
+            'timestamp': 1692860553,
+            'categories': ['Action Movies', 'Salman Special'],
+            'tags': 'count:14',
+            'thumbnail': 'https://snagfilms-a.akamaihd.net/dd078ff5-b16e-45e4-9723-501b56b9df0a/images/2023/08/24/1692860450729_1920x1080_16x9Images.jpg',
+            'display_id': 'bn/videos/bangla-movie-bikkhov',
+            'description': 'md5:71492b086450625f4374a3eb824f27dc',
+            'duration': 8002,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {  # Chorki Premium movie
+        'url': 'https://www.chorki.com/bn/videos/something-like-an-autobiography',
+        'only_matching': True,
     }]
 
     @classmethod

From 999ea80beb053491089d256104c4188aced3110f Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Mon, 29 Jan 2024 20:38:25 +0100
Subject: [PATCH 825/871] [ie/art19] Add extractors (#9099)

Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py |   4 +
 yt_dlp/extractor/art19.py       | 303 ++++++++++++++++++++++++++++++++
 2 files changed, 307 insertions(+)
 create mode 100644 yt_dlp/extractor/art19.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 2fc1e116b..f8488d304 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -138,6 +138,10 @@
     ARDMediathekCollectionIE,
     ARDIE,
 )
+from .art19 import (
+    Art19IE,
+    Art19ShowIE,
+)
 from .arte import (
     ArteTVIE,
     ArteTVEmbedIE,
diff --git a/yt_dlp/extractor/art19.py b/yt_dlp/extractor/art19.py
new file mode 100644
index 000000000..271c505da
--- /dev/null
+++ b/yt_dlp/extractor/art19.py
@@ -0,0 +1,303 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import float_or_none, int_or_none, parse_iso8601, url_or_none
+from ..utils.traversal import traverse_obj
+
+
+class Art19IE(InfoExtractor):
+    _UUID_REGEX = r'[\da-f]{8}-?[\da-f]{4}-?[\da-f]{4}-?[\da-f]{4}-?[\da-f]{12}'
+    _VALID_URL = [
+        rf'https?://(?:www\.)?art19\.com/shows/[^/#?]+/episodes/(?P<id>{_UUID_REGEX})',
+        rf'https?://rss\.art19\.com/episodes/(?P<id>{_UUID_REGEX})\.mp3',
+    ]
+    _EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL[0]})']
+
+    _TESTS = [{
+        'url': 'https://rss.art19.com/episodes/5ba1413c-48b8-472b-9cc3-cfd952340bdb.mp3',
+        'info_dict': {
+            'id': '5ba1413c-48b8-472b-9cc3-cfd952340bdb',
+            'ext': 'mp3',
+            'title': 'Why Did DeSantis Drop Out?',
+            'series': 'The Daily Briefing',
+            'release_timestamp': 1705941275,
+            'description': 'md5:da38961da4a3f7e419471365e3c6b49f',
+            'episode': 'Episode 582',
+            'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$',
+            'series_id': 'ed52a0ab-08b1-4def-8afc-549e4d93296d',
+            'upload_date': '20240122',
+            'timestamp': 1705940815,
+            'episode_number': 582,
+            'modified_date': '20240122',
+            'episode_id': '5ba1413c-48b8-472b-9cc3-cfd952340bdb',
+            'modified_timestamp': 1705941275,
+            'release_date': '20240122',
+            'duration': 527.4,
+        },
+    }, {
+        'url': 'https://art19.com/shows/scamfluencers/episodes/8319b776-4153-4d22-8630-631f204a03dd',
+        'info_dict': {
+            'id': '8319b776-4153-4d22-8630-631f204a03dd',
+            'ext': 'mp3',
+            'title': 'Martha Stewart: The Homemaker Hustler Part 2',
+            'modified_date': '20240116',
+            'upload_date': '20240105',
+            'modified_timestamp': 1705435802,
+            'episode_id': '8319b776-4153-4d22-8630-631f204a03dd',
+            'series_id': 'd3c9b8ca-26b3-42f4-9bd8-21d1a9031e75',
+            'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$',
+            'description': 'md5:4aa7cfd1358dc57e729835bc208d7893',
+            'release_timestamp': 1705305660,
+            'release_date': '20240115',
+            'timestamp': 1704481536,
+            'episode_number': 88,
+            'series': 'Scamfluencers',
+            'duration': 2588.37501,
+            'episode': 'Episode 88',
+        },
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://www.nu.nl/formule-1/6291456/verstappen-wordt-een-synoniem-voor-formule-1.html',
+        'info_dict': {
+            'id': '7d42626a-7301-47db-bb8a-3b6f054d77d7',
+            'ext': 'mp3',
+            'title': "'Verstappen wordt een synoniem voor Formule 1'",
+            'season': 'Seizoen 6',
+            'description': 'md5:39a7159a31c4cda312b2e893bdd5c071',
+            'episode_id': '7d42626a-7301-47db-bb8a-3b6f054d77d7',
+            'duration': 3061.82111,
+            'series_id': '93f4e113-2a60-4609-a564-755058fa40d8',
+            'release_date': '20231126',
+            'modified_timestamp': 1701156004,
+            'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$',
+            'season_number': 6,
+            'episode_number': 52,
+            'modified_date': '20231128',
+            'upload_date': '20231126',
+            'timestamp': 1701025981,
+            'season_id': '36097c1e-7455-490d-a2fe-e2f10b4d5f26',
+            'series': 'De Boordradio',
+            'release_timestamp': 1701026308,
+            'episode': 'Episode 52',
+        },
+    }, {
+        'url': 'https://www.wishtv.com/podcast-episode/larry-bucshon-announces-retirement-from-congress/',
+        'info_dict': {
+            'id': '8da368bd-08d1-46d0-afaa-c134a4af7dc0',
+            'ext': 'mp3',
+            'title': 'Larry Bucshon announces retirement from congress',
+            'upload_date': '20240115',
+            'episode_number': 148,
+            'episode': 'Episode 148',
+            'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$',
+            'release_date': '20240115',
+            'timestamp': 1705328205,
+            'release_timestamp': 1705329275,
+            'series': 'All INdiana Politics',
+            'modified_date': '20240117',
+            'modified_timestamp': 1705458901,
+            'series_id': 'c4af6c27-b10f-4ff2-9f84-0f407df86ff1',
+            'episode_id': '8da368bd-08d1-46d0-afaa-c134a4af7dc0',
+            'description': 'md5:53b5239e4d14973a87125c217c255b2a',
+            'duration': 1256.18848,
+        },
+    }]
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        yield from super()._extract_embed_urls(url, webpage)
+        for episode_id in re.findall(
+                rf'<div[^>]+\bclass=[\'"][^\'"]*art19-web-player[^\'"]*[\'"][^>]+\bdata-episode-id=[\'"]({cls._UUID_REGEX})[\'"]', webpage):
+            yield f'https://rss.art19.com/episodes/{episode_id}.mp3'
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+
+        player_metadata = self._download_json(
+            f'https://art19.com/episodes/{episode_id}', episode_id,
+            note='Downloading player metadata', fatal=False,
+            headers={'Accept': 'application/vnd.art19.v0+json'})
+        rss_metadata = self._download_json(
+            f'https://rss.art19.com/episodes/{episode_id}.json', episode_id, fatal=False,
+            note='Downloading RSS metadata')
+
+        formats = [{
+            'format_id': 'direct',
+            'url': f'https://rss.art19.com/episodes/{episode_id}.mp3',
+            'vcodec': 'none',
+            'acodec': 'mp3',
+        }]
+        for fmt_id, fmt_data in traverse_obj(rss_metadata, ('content', 'media', {dict.items}, ...)):
+            if fmt_id == 'waveform_bin':
+                continue
+            fmt_url = traverse_obj(fmt_data, ('url', {url_or_none}))
+            if not fmt_url:
+                continue
+            formats.append({
+                'format_id': fmt_id,
+                'url': fmt_url,
+                'vcodec': 'none',
+                'acodec': fmt_id,
+                'quality': -2 if fmt_id == 'ogg' else -1,
+            })
+
+        return {
+            'id': episode_id,
+            'formats': formats,
+            **traverse_obj(player_metadata, ('episode', {
+                'title': ('title', {str}),
+                'description': ('description_plain', {str}),
+                'episode_id': ('id', {str}),
+                'episode_number': ('episode_number', {int_or_none}),
+                'season_id': ('season_id', {str}),
+                'series_id': ('series_id', {str}),
+                'timestamp': ('created_at', {parse_iso8601}),
+                'release_timestamp': ('released_at', {parse_iso8601}),
+                'modified_timestamp': ('updated_at', {parse_iso8601})
+            })),
+            **traverse_obj(rss_metadata, ('content', {
+                'title': ('episode_title', {str}),
+                'description': ('episode_description_plain', {str}),
+                'episode_id': ('episode_id', {str}),
+                'episode_number': ('episode_number', {int_or_none}),
+                'season': ('season_title', {str}),
+                'season_id': ('season_id', {str}),
+                'season_number': ('season_number', {int_or_none}),
+                'series': ('series_title', {str}),
+                'series_id': ('series_id', {str}),
+                'thumbnail': ('cover_image', {url_or_none}),
+                'duration': ('duration', {float_or_none}),
+            })),
+        }
+
+
+class Art19ShowIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?art19\.com/shows/(?P<id>[\w-]+)(?:/embed)?/?'
+    _VALID_URL = [
+        rf'{_VALID_URL_BASE}(?:$|[#?])',
+        r'https?://rss\.art19\.com/(?P<id>[\w-]+)/?(?:$|[#?])',
+    ]
+    _EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL_BASE}[^\'"])']
+
+    _TESTS = [{
+        'url': 'https://www.art19.com/shows/5898c087-a14f-48dc-b6fc-a2280a1ff6e0/',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': '5898c087-a14f-48dc-b6fc-a2280a1ff6e0',
+            'display_id': 'echt-gebeurd',
+            'title': 'Echt Gebeurd',
+            'description': 'md5:5fd11dc80b76e51ffd34b6067fd5e560',
+            'timestamp': 1492642167,
+            'upload_date': '20170419',
+            'modified_timestamp': int,
+            'modified_date': str,
+            'tags': 'count:7',
+        },
+        'playlist_mincount': 425,
+    }, {
+        'url': 'https://www.art19.com/shows/echt-gebeurd',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': '5898c087-a14f-48dc-b6fc-a2280a1ff6e0',
+            'display_id': 'echt-gebeurd',
+            'title': 'Echt Gebeurd',
+            'description': 'md5:5fd11dc80b76e51ffd34b6067fd5e560',
+            'timestamp': 1492642167,
+            'upload_date': '20170419',
+            'modified_timestamp': int,
+            'modified_date': str,
+            'tags': 'count:7',
+        },
+        'playlist_mincount': 425,
+    }, {
+        'url': 'https://rss.art19.com/scamfluencers',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'd3c9b8ca-26b3-42f4-9bd8-21d1a9031e75',
+            'display_id': 'scamfluencers',
+            'title': 'Scamfluencers',
+            'description': 'md5:7d239d670c0ced6dadbf71c4caf764b7',
+            'timestamp': 1647368573,
+            'upload_date': '20220315',
+            'modified_timestamp': int,
+            'modified_date': str,
+            'tags': [],
+        },
+        'playlist_mincount': 90,
+    }, {
+        'url': 'https://art19.com/shows/enthuellt/embed',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'e2cacf57-bb8a-4263-aa81-719bcdd4f80c',
+            'display_id': 'enthuellt',
+            'title': 'Enthüllt',
+            'description': 'md5:17752246643414a2fd51744fc9a1c08e',
+            'timestamp': 1601645860,
+            'upload_date': '20201002',
+            'modified_timestamp': int,
+            'modified_date': str,
+            'tags': 'count:10',
+        },
+        'playlist_mincount': 10,
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://deconstructingyourself.com/deconstructing-yourself-podcast',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'cfbb9b01-c295-4adb-8726-adde7c03cf21',
+            'display_id': 'deconstructing-yourself',
+            'title': 'Deconstructing Yourself',
+            'description': 'md5:dab5082b28b248a35476abf64768854d',
+            'timestamp': 1570581181,
+            'upload_date': '20191009',
+            'modified_timestamp': int,
+            'modified_date': str,
+            'tags': 'count:5',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'https://chicagoreader.com/columns-opinion/podcasts/ben-joravsky-show-podcast-episodes/',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': '9dfa2c37-ab87-4c13-8388-4897914313ec',
+            'display_id': 'the-ben-joravsky-show',
+            'title': 'The Ben Joravsky Show',
+            'description': 'md5:c0f3ec0ee0dbea764390e521adc8780a',
+            'timestamp': 1550875095,
+            'upload_date': '20190222',
+            'modified_timestamp': int,
+            'modified_date': str,
+            'tags': ['Chicago Politics', 'chicago', 'Ben Joravsky'],
+        },
+        'playlist_mincount': 1900,
+    }]
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        yield from super()._extract_embed_urls(url, webpage)
+        for series_id in re.findall(
+                r'<div[^>]+\bclass=[\'"][^\'"]*art19-web-player[^\'"]*[\'"][^>]+\bdata-series-id=[\'"]([\w-]+)[\'"]', webpage):
+            yield f'https://art19.com/shows/{series_id}'
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        series_metadata = self._download_json(
+            f'https://art19.com/series/{series_id}', series_id, note='Downloading series metadata',
+            headers={'Accept': 'application/vnd.art19.v0+json'})
+
+        return {
+            '_type': 'playlist',
+            'entries': [
+                self.url_result(f'https://rss.art19.com/episodes/{episode_id}.mp3', Art19IE)
+                for episode_id in traverse_obj(series_metadata, ('series', 'episode_ids', ..., {str}))
+            ],
+            **traverse_obj(series_metadata, ('series', {
+                'id': ('id', {str}),
+                'display_id': ('slug', {str}),
+                'title': ('title', {str}),
+                'description': ('description_plain', {str}),
+                'timestamp': ('created_at', {parse_iso8601}),
+                'modified_timestamp': ('updated_at', {parse_iso8601}),
+            })),
+            'tags': traverse_obj(series_metadata, ('tags', ..., 'name', {str})),
+        }

From 9b5efaf86b99a2664fff9fc725d275f766c3221d Mon Sep 17 00:00:00 2001
From: kclauhk <78251477+kclauhk@users.noreply.github.com>
Date: Tue, 30 Jan 2024 03:43:41 +0800
Subject: [PATCH 826/871] [ie/facebook] Support events (#9055)

Closes #5355
Authored by: kclauhk
---
 yt_dlp/extractor/facebook.py | 77 +++++++++++++++++++++---------------
 1 file changed, 45 insertions(+), 32 deletions(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index d186b57bf..830bbcc3c 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -54,6 +54,7 @@ class FacebookIE(InfoExtractor):
                             )\?(?:.*?)(?:v|video_id|story_fbid)=|
                             [^/]+/videos/(?:[^/]+/)?|
                             [^/]+/posts/|
+                            events/(?:[^/]+/)?|
                             groups/[^/]+/(?:permalink|posts)/|
                             watchparty/
                         )|
@@ -399,6 +400,18 @@ class FacebookIE(InfoExtractor):
         },
         'playlist_count': 1,
         'skip': 'Requires logging in',
+    }, {
+        # data.event.cover_media_renderer.cover_video
+        'url': 'https://m.facebook.com/events/1509582499515440',
+        'info_dict': {
+            'id': '637246984455045',
+            'ext': 'mp4',
+            'title': 'ANALISI IN CAMPO OSCURO " Coaguli nel sangue dei vaccinati"',
+            'description': 'Other event by Comitato Liberi Pensatori on Tuesday, October 18 2022',
+            'thumbnail': r're:^https?://.*',
+            'uploader': 'Comitato Liberi Pensatori',
+            'uploader_id': '100065709540881',
+        },
     }]
     _SUPPORTED_PAGLETS_REGEX = r'(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_[0-9a-f]+)'
     _api_config = {
@@ -473,38 +486,10 @@ def extract_metadata(webpage):
                 r'data-sjs>({.*?ScheduledServerJS.*?})</script>', webpage)]
             post = traverse_obj(post_data, (
                 ..., 'require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., '__bbox', 'result', 'data'), expected_type=dict) or []
-
-            automatic_captions, subtitles = {}, {}
-            subs_data = traverse_obj(post, (..., 'video', ..., 'attachments', ..., lambda k, v: (
-                k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')))
-            is_video_broadcast = get_first(subs_data, 'is_video_broadcast', expected_type=bool)
-            captions = get_first(subs_data, 'video_available_captions_locales', 'captions_url')
-            if url_or_none(captions):  # if subs_data only had a 'captions_url'
-                locale = self._html_search_meta(['og:locale', 'twitter:locale'], webpage, 'locale', default='en_US')
-                subtitles[locale] = [{'url': captions}]
-            # or else subs_data had 'video_available_captions_locales', a list of dicts
-            for caption in traverse_obj(captions, (
-                {lambda x: sorted(x, key=lambda c: c['locale'])}, lambda _, v: v['captions_url'])
-            ):
-                lang = caption.get('localized_language') or ''
-                subs = {
-                    'url': caption['captions_url'],
-                    'name': format_field(caption, 'localized_country', f'{lang} (%s)', default=lang),
-                }
-                if caption.get('localized_creation_method') or is_video_broadcast:
-                    automatic_captions.setdefault(caption['locale'], []).append(subs)
-                else:
-                    subtitles.setdefault(caption['locale'], []).append(subs)
-
             media = traverse_obj(post, (..., 'attachments', ..., lambda k, v: (
                 k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')), expected_type=dict)
             title = get_first(media, ('title', 'text'))
             description = get_first(media, ('creation_story', 'comet_sections', 'message', 'story', 'message', 'text'))
-            uploader_data = (
-                get_first(media, ('owner', {dict}))
-                or get_first(post, (..., 'video', lambda k, v: k == 'owner' and v['name']))
-                or get_first(post, ('node', 'actors', ..., {dict})) or {})
-
             page_title = title or self._html_search_regex((
                 r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>(?P<content>[^<]*)</h2>',
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(?P<content>.*?)</span>',
@@ -513,11 +498,15 @@ def extract_metadata(webpage):
             description = description or self._html_search_meta(
                 ['description', 'og:description', 'twitter:description'],
                 webpage, 'description', default=None)
+            uploader_data = (
+                get_first(media, ('owner', {dict}))
+                or get_first(post, (..., 'video', lambda k, v: k == 'owner' and v['name']))
+                or get_first(post, ('node', 'actors', ..., {dict}))
+                or get_first(post, ('event', 'event_creator', {dict})) or {})
             uploader = uploader_data.get('name') or (
                 clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
                 or self._search_regex(
                     (r'ownerName\s*:\s*"([^"]+)"', *self._og_regexes('title')), webpage, 'uploader', fatal=False))
-
             timestamp = int_or_none(self._search_regex(
                 r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
                 'timestamp', default=None))
@@ -539,8 +528,6 @@ def extract_metadata(webpage):
                     webpage, 'view count', default=None)),
                 'concurrent_view_count': get_first(post, (
                     ('video', (..., ..., 'attachments', ..., 'media')), 'liveViewerCount', {int_or_none})),
-                'automatic_captions': automatic_captions,
-                'subtitles': subtitles,
             }
 
             info_json_ld = self._search_json_ld(webpage, video_id, default={})
@@ -638,6 +625,29 @@ def parse_graphql_video(video):
                                 'url': playable_url,
                             })
                     extract_dash_manifest(video, formats)
+
+                    automatic_captions, subtitles = {}, {}
+                    is_broadcast = traverse_obj(video, ('is_video_broadcast', {bool}))
+                    for caption in traverse_obj(video, (
+                        'video_available_captions_locales',
+                        {lambda x: sorted(x, key=lambda c: c['locale'])},
+                        lambda _, v: url_or_none(v['captions_url'])
+                    )):
+                        lang = caption.get('localized_language') or 'und'
+                        subs = {
+                            'url': caption['captions_url'],
+                            'name': format_field(caption, 'localized_country', f'{lang} (%s)', default=lang),
+                        }
+                        if caption.get('localized_creation_method') or is_broadcast:
+                            automatic_captions.setdefault(caption['locale'], []).append(subs)
+                        else:
+                            subtitles.setdefault(caption['locale'], []).append(subs)
+                    captions_url = traverse_obj(video, ('captions_url', {url_or_none}))
+                    if captions_url and not automatic_captions and not subtitles:
+                        locale = self._html_search_meta(
+                            ['og:locale', 'twitter:locale'], webpage, 'locale', default='en_US')
+                        (automatic_captions if is_broadcast else subtitles)[locale] = [{'url': captions_url}]
+
                     info = {
                         'id': v_id,
                         'formats': formats,
@@ -647,6 +657,8 @@ def parse_graphql_video(video):
                         'timestamp': traverse_obj(video, 'publish_time', 'creation_time', expected_type=int_or_none),
                         'duration': (float_or_none(video.get('playable_duration_in_ms'), 1000)
                                      or float_or_none(video.get('length_in_second'))),
+                        'automatic_captions': automatic_captions,
+                        'subtitles': subtitles,
                     }
                     process_formats(info)
                     description = try_get(video, lambda x: x['savable_description']['text'])
@@ -681,7 +693,8 @@ def parse_attachment(attachment, key='media'):
                 for edge in edges:
                     parse_attachment(edge, key='node')
 
-                video = data.get('video') or {}
+                video = traverse_obj(data, (
+                    'event', 'cover_media_renderer', 'cover_video'), 'video', expected_type=dict) or {}
                 if video:
                     attachments = try_get(video, [
                         lambda x: x['story']['attachments'],

From 67bb70cd700c8d4c3149cd9e0539a5f32c3d1ce6 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Mon, 29 Jan 2024 21:16:46 +0100
Subject: [PATCH 827/871] [ie/Vbox7] Fix extractor (#9100)

Closes #1098, Closes #5661
Authored by: seproDev
---
 yt_dlp/extractor/vbox7.py | 82 ++++++++++++++++++++-------------------
 1 file changed, 43 insertions(+), 39 deletions(-)

diff --git a/yt_dlp/extractor/vbox7.py b/yt_dlp/extractor/vbox7.py
index be35dad1c..21bf4232b 100644
--- a/yt_dlp/extractor/vbox7.py
+++ b/yt_dlp/extractor/vbox7.py
@@ -1,5 +1,6 @@
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import ExtractorError, base_url, int_or_none, url_basename
+from ..utils.traversal import traverse_obj
 
 
 class Vbox7IE(InfoExtractor):
@@ -19,7 +20,7 @@ class Vbox7IE(InfoExtractor):
     _GEO_COUNTRIES = ['BG']
     _TESTS = [{
         'url': 'http://vbox7.com/play:0946fff23c',
-        'md5': 'a60f9ab3a3a2f013ef9a967d5f7be5bf',
+        'md5': '50ca1f78345a9c15391af47d8062d074',
         'info_dict': {
             'id': '0946fff23c',
             'ext': 'mp4',
@@ -29,19 +30,25 @@ class Vbox7IE(InfoExtractor):
             'timestamp': 1470982814,
             'upload_date': '20160812',
             'uploader': 'zdraveibulgaria',
-        },
-        'params': {
-            'proxy': '127.0.0.1:8118',
+            'view_count': int,
+            'duration': 2640,
         },
     }, {
         'url': 'http://vbox7.com/play:249bb972c2',
-        'md5': '99f65c0c9ef9b682b97313e052734c3f',
+        'md5': 'da1dd2eb245200cb86e6d09d43232116',
         'info_dict': {
             'id': '249bb972c2',
             'ext': 'mp4',
             'title': 'Смях! Чудо - чист за секунди - Скрита камера',
+            'uploader': 'svideteliat_ot_varshava',
+            'view_count': int,
+            'timestamp': 1360215023,
+            'thumbnail': 'https://i49.vbox7.com/design/iconci/png/noimg6.png',
+            'description': 'Смях! Чудо - чист за секунди - Скрита камера',
+            'upload_date': '20130207',
+            'duration': 83,
         },
-        'skip': 'georestricted',
+        'expected_warnings': ['Failed to download m3u8 information'],
     }, {
         'url': 'http://vbox7.com/emb/external.php?vid=a240d20f9c&autoplay=1',
         'only_matching': True,
@@ -53,41 +60,38 @@ class Vbox7IE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        response = self._download_json(
-            'https://www.vbox7.com/ajax/video/nextvideo.php?vid=%s' % video_id,
-            video_id)
+        data = self._download_json(
+            'https://www.vbox7.com/aj/player/item/options', video_id,
+            query={'vid': video_id})['options']
 
-        if 'error' in response:
-            raise ExtractorError(
-                '%s said: %s' % (self.IE_NAME, response['error']), expected=True)
+        src_url = data.get('src')
+        if src_url in (None, '', 'blank'):
+            raise ExtractorError('Video is unavailable', expected=True)
 
-        video = response['options']
+        fmt_base = url_basename(src_url).rsplit('.', 1)[0].rsplit('_', 1)[0]
+        if fmt_base == 'vn':
+            self.raise_geo_restricted()
 
-        title = video['title']
-        video_url = video['src']
+        fmt_base = base_url(src_url) + fmt_base
 
-        if '/na.mp4' in video_url:
-            self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+        formats = self._extract_m3u8_formats(
+            f'{fmt_base}.m3u8', video_id, m3u8_id='hls', fatal=False)
+        # TODO: Add MPD formats, when dash range support is added
+        for res in traverse_obj(data, ('resolutions', lambda _, v: v != 0, {int})):
+            formats.append({
+                'url': f'{fmt_base}_{res}.mp4',
+                'format_id': f'http-{res}',
+                'height': res,
+            })
 
-        uploader = video.get('uploader')
-
-        webpage = self._download_webpage(
-            'http://vbox7.com/play:%s' % video_id, video_id, fatal=None)
-
-        info = {}
-
-        if webpage:
-            info = self._search_json_ld(
-                webpage.replace('"/*@context"', '"@context"'), video_id,
-                fatal=False)
-
-        info.update({
+        return {
             'id': video_id,
-            'title': title,
-            'url': video_url,
-            'uploader': uploader,
-            'thumbnail': self._proto_relative_url(
-                info.get('thumbnail') or self._og_search_thumbnail(webpage),
-                'http:'),
-        })
-        return info
+            'formats': formats,
+            **self._search_json_ld(self._download_webpage(
+                f'https://www.vbox7.com/play:{video_id}', video_id, fatal=False) or '', video_id, fatal=False),
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'uploader': ('uploader', {str}),
+                'duration': ('duration', {int_or_none}),
+            }),
+        }

From 3725b4f0c93ca3943e6300013a9670e4ab757fda Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 31 Jan 2024 09:35:35 +0100
Subject: [PATCH 828/871] [core] Add `--compat-options 2023` (#9084)

Authored by: Grub4K
---
 README.md         | 3 ++-
 yt_dlp/options.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b6a79667c..7dc3bb2f6 100644
--- a/README.md
+++ b/README.md
@@ -167,7 +167,8 @@ ### Differences in default behavior
 * `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx`
 * `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx`
 * `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date`
-* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx`. Use this to enable all future compat options
+* `--compat-options 2022`: Same as `--compat-options 2023,playlist-match-filter,no-external-downloader-progress`
+* `--compat-options 2023`: Same as `--compat-options prefer-legacy-http-handler,manifest-filesize-approx`. Use this to enable all future compat options
 
 
 # INSTALLATION
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index e9d927717..9bea6549d 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -476,7 +476,8 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
                 'youtube-dl': ['all', '-multistreams', '-playlist-match-filter', '-manifest-filesize-approx'],
                 'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter', '-manifest-filesize-approx'],
                 '2021': ['2022', 'no-certifi', 'filename-sanitization', 'no-youtube-prefer-utc-upload-date'],
-                '2022': ['no-external-downloader-progress', 'playlist-match-filter', 'prefer-legacy-http-handler', 'manifest-filesize-approx'],
+                '2022': ['2023', 'no-external-downloader-progress', 'playlist-match-filter'],
+                '2023': ['prefer-legacy-http-handler', 'manifest-filesize-approx'],
             }
         }, help=(
             'Options that can help keep compatibility with youtube-dl or youtube-dlc '

From cbed249aaa053a3f425b9bafc97f8dbd71c44487 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 31 Jan 2024 09:43:52 +0100
Subject: [PATCH 829/871] [cookies] Fix `--cookies-from-browser` for `snap`
 Firefox (#9016)

Authored by: Grub4K
---
 yt_dlp/cookies.py | 46 ++++++++++++++++++++++++++++++----------------
 1 file changed, 30 insertions(+), 16 deletions(-)

diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index eac033e39..a92ab4164 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -1,6 +1,7 @@
 import base64
 import collections
 import contextlib
+import glob
 import http.cookiejar
 import http.cookies
 import io
@@ -122,13 +123,14 @@ def _extract_firefox_cookies(profile, container, logger):
         return YoutubeDLCookieJar()
 
     if profile is None:
-        search_root = _firefox_browser_dir()
+        search_roots = list(_firefox_browser_dirs())
     elif _is_path(profile):
-        search_root = profile
+        search_roots = [profile]
     else:
-        search_root = os.path.join(_firefox_browser_dir(), profile)
+        search_roots = [os.path.join(path, profile) for path in _firefox_browser_dirs()]
+    search_root = ', '.join(map(repr, search_roots))
 
-    cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite', logger)
+    cookie_database_path = _newest(_firefox_cookie_dbs(search_roots))
     if cookie_database_path is None:
         raise FileNotFoundError(f'could not find firefox cookies database in {search_root}')
     logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
@@ -182,12 +184,21 @@ def _extract_firefox_cookies(profile, container, logger):
                 cursor.connection.close()
 
 
-def _firefox_browser_dir():
+def _firefox_browser_dirs():
     if sys.platform in ('cygwin', 'win32'):
-        return os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles')
+        yield os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles')
+
     elif sys.platform == 'darwin':
-        return os.path.expanduser('~/Library/Application Support/Firefox/Profiles')
-    return os.path.expanduser('~/.mozilla/firefox')
+        yield os.path.expanduser('~/Library/Application Support/Firefox/Profiles')
+
+    else:
+        yield from map(os.path.expanduser, ('~/.mozilla/firefox', '~/snap/firefox/common/.mozilla/firefox'))
+
+
+def _firefox_cookie_dbs(roots):
+    for root in map(os.path.abspath, roots):
+        for pattern in ('', '*/', 'Profiles/*/'):
+            yield from glob.iglob(os.path.join(root, pattern, 'cookies.sqlite'))
 
 
 def _get_chromium_based_browser_settings(browser_name):
@@ -268,7 +279,7 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
             logger.error(f'{browser_name} does not support profiles')
             search_root = config['browser_dir']
 
-    cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies', logger)
+    cookie_database_path = _newest(_find_files(search_root, 'Cookies', logger))
     if cookie_database_path is None:
         raise FileNotFoundError(f'could not find {browser_name} cookies database in "{search_root}"')
     logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
@@ -947,7 +958,7 @@ def _get_windows_v10_key(browser_root, logger):
     References:
         - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
     """
-    path = _find_most_recently_used_file(browser_root, 'Local State', logger)
+    path = _newest(_find_files(browser_root, 'Local State', logger))
     if path is None:
         logger.error('could not find local state file')
         return None
@@ -1049,17 +1060,20 @@ def _get_column_names(cursor, table_name):
     return [row[1].decode() for row in table_info]
 
 
-def _find_most_recently_used_file(root, filename, logger):
+def _newest(files):
+    return max(files, key=lambda path: os.lstat(path).st_mtime, default=None)
+
+
+def _find_files(root, filename, logger):
     # if there are multiple browser profiles, take the most recently used one
-    i, paths = 0, []
+    i = 0
     with _create_progress_bar(logger) as progress_bar:
-        for curr_root, dirs, files in os.walk(root):
+        for curr_root, _, files in os.walk(root):
             for file in files:
                 i += 1
                 progress_bar.print(f'Searching for "{filename}": {i: 6d} files searched')
                 if file == filename:
-                    paths.append(os.path.join(curr_root, file))
-    return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
+                    yield os.path.join(curr_root, file)
 
 
 def _merge_cookie_jars(jars):
@@ -1073,7 +1087,7 @@ def _merge_cookie_jars(jars):
 
 
 def _is_path(value):
-    return os.path.sep in value
+    return any(sep in value for sep in (os.path.sep, os.path.altsep) if sep)
 
 
 def _parse_browser_specification(browser_name, profile=None, keyring=None, container=None):

From 2792092afd367e39251ace1fb2819c855ab8919f Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Wed, 31 Jan 2024 09:56:14 +0100
Subject: [PATCH 830/871] [cookies] Improve error message for Windows
 `--cookies-from-browser chrome` issue (#9080)

Authored by: Grub4K
---
 yt_dlp/cookies.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index a92ab4164..deb2e35f2 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -24,7 +24,8 @@
     aes_gcm_decrypt_and_verify_bytes,
     unpad_pkcs7,
 )
-from .compat import functools
+from .compat import functools  # isort: split
+from .compat import compat_os_name
 from .dependencies import (
     _SECRETSTORAGE_UNAVAILABLE_REASON,
     secretstorage,
@@ -32,6 +33,7 @@
 )
 from .minicurses import MultilinePrinter, QuietMultilinePrinter
 from .utils import (
+    DownloadError,
     Popen,
     error_to_str,
     expand_path,
@@ -318,6 +320,12 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
             counts['unencrypted'] = unencrypted_cookies
             logger.debug(f'cookie version breakdown: {counts}')
             return jar
+        except PermissionError as error:
+            if compat_os_name == 'nt' and error.errno == 13:
+                message = 'Could not copy Chrome cookie database. See  https://github.com/yt-dlp/yt-dlp/issues/7271  for more info'
+                logger.error(message)
+                raise DownloadError(message)  # force exit
+            raise
         finally:
             if cursor is not None:
                 cursor.connection.close()

From d63eae7e7ffb1f3e733e552b9e5e82355bfba214 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Wed, 31 Jan 2024 03:11:41 -0600
Subject: [PATCH 831/871] [core] Don't select storyboard formats as fallback

Closes #7715
Authored by: bashonly
---
 yt_dlp/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 5dcefb5b8..e7d654d0f 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2451,7 +2451,7 @@ def selector_function(ctx):
                                 # for extractors with incomplete formats (audio only (soundcloud)
                                 # or video only (imgur)) best/worst will fallback to
                                 # best/worst {video,audio}-only format
-                                matches = formats
+                                matches = list(filter(lambda f: f.get('vcodec') != 'none' or f.get('acodec') != 'none', formats))
                             elif seperate_fallback and not ctx['has_merged_format']:
                                 # for compatibility with youtube-dl when there is no pre-merged format
                                 matches = list(filter(seperate_fallback, formats))

From 62c65bfaf81e04e6746f6fdbafe384eb3edddfbc Mon Sep 17 00:00:00 2001
From: Radu Manole <radu.v.manole@gmail.com>
Date: Wed, 31 Jan 2024 19:41:31 +0200
Subject: [PATCH 832/871] [ie/NinaProtocol] Add extractor (#8946)

Closes #8709, Closes #8764
Authored by: RaduManole, seproDev

Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 yt_dlp/extractor/_extractors.py  |   1 +
 yt_dlp/extractor/ninaprotocol.py | 225 +++++++++++++++++++++++++++++++
 2 files changed, 226 insertions(+)
 create mode 100644 yt_dlp/extractor/ninaprotocol.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index f8488d304..69deaf15a 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1284,6 +1284,7 @@
     NicovideoTagURLIE,
     NiconicoLiveIE,
 )
+from .ninaprotocol import NinaProtocolIE
 from .ninecninemedia import (
     NineCNineMediaIE,
     CPTwentyFourIE,
diff --git a/yt_dlp/extractor/ninaprotocol.py b/yt_dlp/extractor/ninaprotocol.py
new file mode 100644
index 000000000..ea57c5f38
--- /dev/null
+++ b/yt_dlp/extractor/ninaprotocol.py
@@ -0,0 +1,225 @@
+from .common import InfoExtractor
+from ..utils import int_or_none, mimetype2ext, parse_iso8601, url_or_none
+from ..utils.traversal import traverse_obj
+
+
+class NinaProtocolIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ninaprotocol\.com/releases/(?P<id>[^/#?]+)'
+    _TESTS = [{
+        'url': 'https://www.ninaprotocol.com/releases/3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ',
+        'info_dict': {
+            'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ',
+            'title': 'The Spatulas - March Chant',
+            'tags': ['punk', 'postpresentmedium', 'cambridge'],
+            'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+            'channel': 'ppm',
+            'description': 'md5:bb9f9d39d8f786449cd5d0ff7c5772db',
+            'album': 'The Spatulas - March Chant',
+            'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+            'timestamp': 1701417610,
+            'uploader': 'ppmrecs',
+            'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+            'display_id': 'the-spatulas-march-chant',
+            'upload_date': '20231201',
+            'album_artist': 'Post Present Medium ',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_1',
+                'title': 'March Chant In April',
+                'track': 'March Chant In April',
+                'ext': 'mp3',
+                'duration': 152,
+                'track_number': 1,
+                'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+                'uploader': 'ppmrecs',
+                'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+                'timestamp': 1701417610,
+                'channel': 'ppm',
+                'album': 'The Spatulas - March Chant',
+                'tags': ['punk', 'postpresentmedium', 'cambridge'],
+                'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+                'upload_date': '20231201',
+                'album_artist': 'Post Present Medium ',
+            }
+        }, {
+            'info_dict': {
+                'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_2',
+                'title': 'Rescue Mission',
+                'track': 'Rescue Mission',
+                'ext': 'mp3',
+                'duration': 212,
+                'track_number': 2,
+                'album_artist': 'Post Present Medium ',
+                'uploader': 'ppmrecs',
+                'tags': ['punk', 'postpresentmedium', 'cambridge'],
+                'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+                'channel': 'ppm',
+                'upload_date': '20231201',
+                'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+                'timestamp': 1701417610,
+                'album': 'The Spatulas - March Chant',
+                'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+            }
+        }, {
+            'info_dict': {
+                'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_3',
+                'title': 'Slinger Style',
+                'track': 'Slinger Style',
+                'ext': 'mp3',
+                'duration': 179,
+                'track_number': 3,
+                'timestamp': 1701417610,
+                'upload_date': '20231201',
+                'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+                'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+                'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+                'album_artist': 'Post Present Medium ',
+                'album': 'The Spatulas - March Chant',
+                'tags': ['punk', 'postpresentmedium', 'cambridge'],
+                'uploader': 'ppmrecs',
+                'channel': 'ppm',
+            }
+        }, {
+            'info_dict': {
+                'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_4',
+                'title': 'Psychic Signal',
+                'track': 'Psychic Signal',
+                'ext': 'mp3',
+                'duration': 220,
+                'track_number': 4,
+                'tags': ['punk', 'postpresentmedium', 'cambridge'],
+                'upload_date': '20231201',
+                'album': 'The Spatulas - March Chant',
+                'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+                'timestamp': 1701417610,
+                'album_artist': 'Post Present Medium ',
+                'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+                'channel': 'ppm',
+                'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+                'uploader': 'ppmrecs',
+            }
+        }, {
+            'info_dict': {
+                'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_5',
+                'title': 'Curvy Color',
+                'track': 'Curvy Color',
+                'ext': 'mp3',
+                'duration': 148,
+                'track_number': 5,
+                'timestamp': 1701417610,
+                'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+                'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+                'album': 'The Spatulas - March Chant',
+                'album_artist': 'Post Present Medium ',
+                'channel': 'ppm',
+                'tags': ['punk', 'postpresentmedium', 'cambridge'],
+                'uploader': 'ppmrecs',
+                'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+                'upload_date': '20231201',
+            }
+        }, {
+            'info_dict': {
+                'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_6',
+                'title': 'Caveman Star',
+                'track': 'Caveman Star',
+                'ext': 'mp3',
+                'duration': 121,
+                'track_number': 6,
+                'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+                'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+                'tags': ['punk', 'postpresentmedium', 'cambridge'],
+                'album_artist': 'Post Present Medium ',
+                'uploader': 'ppmrecs',
+                'timestamp': 1701417610,
+                'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+                'album': 'The Spatulas - March Chant',
+                'channel': 'ppm',
+                'upload_date': '20231201',
+            },
+        }],
+    }, {
+        'url': 'https://www.ninaprotocol.com/releases/f-g-s-american-shield',
+        'info_dict': {
+            'id': '76PZnJwaMgViQHYfA4NYJXds7CmW6vHQKAtQUxGene6J',
+            'description': 'md5:63f08d5db558b4b36e1896f317062721',
+            'title': 'F.G.S. - American Shield',
+            'uploader_id': 'Ej3rozs11wYqFk1Gs6oggGCkGLz8GzBhmJfnUxf6gPci',
+            'channel_id': '6JuksCZPXuP16wJ1BUfwuukJzh42C7guhLrFPPkVJfyE',
+            'channel': 'tinkscough',
+            'tags': [],
+            'album_artist': 'F.G.S.',
+            'album': 'F.G.S. - American Shield',
+            'thumbnail': 'https://www.arweave.net/YJpgImkXLT9SbpFb576KuZ5pm6bdvs452LMs3Rx6lm8',
+            'display_id': 'f-g-s-american-shield',
+            'uploader': 'flannerysilva',
+            'timestamp': 1702395858,
+            'upload_date': '20231212',
+        },
+        'playlist_count': 1,
+    }, {
+        'url': 'https://www.ninaprotocol.com/releases/time-to-figure-things-out',
+        'info_dict': {
+            'id': '6Zi1nC5hj6b13NkpxVYwRhFy6mYA7oLBbe9DMrgGDcYh',
+            'display_id': 'time-to-figure-things-out',
+            'description': 'md5:960202ed01c3134bb8958f1008527e35',
+            'timestamp': 1706283607,
+            'title': 'DJ STEPDAD - time to figure things out',
+            'album_artist': 'DJ STEPDAD',
+            'uploader': 'tddvsss',
+            'upload_date': '20240126',
+            'album': 'time to figure things out',
+            'uploader_id': 'AXQNRgTyYsySyAMFDwxzumuGjfmoXshorCesjpquwCBi',
+            'thumbnail': 'https://www.arweave.net/O4i8bcKVqJVZvNeHHFp6r8knpFGh9ZwEgbeYacr4nss',
+            'tags': [],
+        },
+        'playlist_count': 4,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        release = self._download_json(
+            f'https://api.ninaprotocol.com/v1/releases/{video_id}', video_id)['release']
+
+        video_id = release.get('publicKey') or video_id
+
+        common_info = traverse_obj(release, {
+            'album': ('metadata', 'properties', 'title', {str}),
+            'album_artist': ((('hub', 'data'), 'publisherAccount'), 'displayName', {str}),
+            'timestamp': ('datetime', {parse_iso8601}),
+            'thumbnail': ('metadata', 'image', {url_or_none}),
+            'uploader': ('publisherAccount', 'handle', {str}),
+            'uploader_id': ('publisherAccount', 'publicKey', {str}),
+            'channel': ('hub', 'handle', {str}),
+            'channel_id': ('hub', 'publicKey', {str}),
+        }, get_all=False)
+        common_info['tags'] = traverse_obj(release, ('metadata', 'properties', 'tags', ..., {str}))
+
+        entries = []
+        for track_num, track in enumerate(traverse_obj(release, (
+                'metadata', 'properties', 'files', lambda _, v: url_or_none(v['uri']))), 1):
+            entries.append({
+                'id': f'{video_id}_{track_num}',
+                'url': track['uri'],
+                **traverse_obj(track, {
+                    'title': ('track_title', {str}),
+                    'track': ('track_title', {str}),
+                    'ext': ('type', {mimetype2ext}),
+                    'track_number': ('track', {int_or_none}),
+                    'duration': ('duration', {int_or_none}),
+                }),
+                'vcodec': 'none',
+                **common_info,
+            })
+
+        return {
+            '_type': 'playlist',
+            'id': video_id,
+            'entries': entries,
+            **traverse_obj(release, {
+                'display_id': ('slug', {str}),
+                'title': ('metadata', 'name', {str}),
+                'description': ('metadata', 'description', {str}),
+            }),
+            **common_info,
+        }

From 4a6ff0b47a700dee3ee5c54804c31965308479ae Mon Sep 17 00:00:00 2001
From: jazz1611 <jazz1611@users.noreply.github.com>
Date: Thu, 1 Feb 2024 00:56:29 +0700
Subject: [PATCH 833/871] [ie/redtube] Support redtube.com.br URLs (#9103)

Authored by: jazz1611
---
 yt_dlp/extractor/redtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/redtube.py b/yt_dlp/extractor/redtube.py
index 36d530daf..965abbee8 100644
--- a/yt_dlp/extractor/redtube.py
+++ b/yt_dlp/extractor/redtube.py
@@ -12,7 +12,7 @@
 
 
 class RedTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:\w+\.)?redtube\.com/|embed\.redtube\.com/\?.*?\bid=)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:(?:\w+\.)?redtube\.com(?:\.br)?/|embed\.redtube\.com/\?.*?\bid=)(?P<id>[0-9]+)'
     _EMBED_REGEX = [r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//embed\.redtube\.com/\?.*?\bid=\d+)']
     _TESTS = [{
         'url': 'https://www.redtube.com/38864951',
@@ -35,6 +35,9 @@ class RedTubeIE(InfoExtractor):
     }, {
         'url': 'http://it.redtube.com/66418',
         'only_matching': True,
+    }, {
+        'url': 'https://www.redtube.com.br/103224331',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 4b8b0dded8c65cd5b2ab2e858058ba98c9bf49ff Mon Sep 17 00:00:00 2001
From: rrgomes <rrg@panix.com>
Date: Wed, 31 Jan 2024 13:00:15 -0500
Subject: [PATCH 834/871] [ie/nfb] Add support for onf.ca and series (#8997)

Closes #8198
Authored by: bashonly, rrgomes

Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
---
 yt_dlp/extractor/_extractors.py |   5 +-
 yt_dlp/extractor/nfb.py         | 288 +++++++++++++++++++++++++++++---
 2 files changed, 269 insertions(+), 24 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 69deaf15a..82d3004ba 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1247,7 +1247,10 @@
     NexxIE,
     NexxEmbedIE,
 )
-from .nfb import NFBIE
+from .nfb import (
+    NFBIE,
+    NFBSeriesIE,
+)
 from .nfhsnetwork import NFHSNetworkIE
 from .nfl import (
     NFLIE,
diff --git a/yt_dlp/extractor/nfb.py b/yt_dlp/extractor/nfb.py
index 38e068af4..6f7872825 100644
--- a/yt_dlp/extractor/nfb.py
+++ b/yt_dlp/extractor/nfb.py
@@ -1,10 +1,54 @@
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    join_nonempty,
+    merge_dicts,
+    parse_count,
+    url_or_none,
+    urljoin,
+)
+from ..utils.traversal import traverse_obj
 
 
-class NFBIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nfb\.ca/film/(?P<id>[^/?#&]+)'
+class NFBBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?(?P<site>nfb|onf)\.ca'
+    _GEO_COUNTRIES = ['CA']
+
+    def _extract_ep_data(self, webpage, video_id, fatal=False):
+        return self._search_json(
+            r'const\s+episodesData\s*=', webpage, 'episode data', video_id,
+            contains_pattern=r'\[\s*{(?s:.+)}\s*\]', fatal=fatal) or []
+
+    def _extract_ep_info(self, data, video_id, slug=None):
+        info = traverse_obj(data, (lambda _, v: video_id in v['embed_url'], {
+            'description': ('description', {str}),
+            'thumbnail': ('thumbnail_url', {url_or_none}),
+            'uploader': ('data_layer', 'episodeMaker', {str}),
+            'release_year': ('data_layer', 'episodeYear', {int_or_none}),
+            'episode': ('data_layer', 'episodeTitle', {str}),
+            'season': ('data_layer', 'seasonTitle', {str}),
+            'season_number': ('data_layer', 'seasonTitle', {parse_count}),
+            'series': ('data_layer', 'seriesTitle', {str}),
+        }), get_all=False)
+
+        return {
+            **info,
+            'id': video_id,
+            'title': join_nonempty('series', 'episode', from_dict=info, delim=' - '),
+            'episode_number': int_or_none(self._search_regex(
+                r'[/-]e(?:pisode)?-?(\d+)(?:[/-]|$)', slug or video_id, 'episode number', default=None)),
+        }
+
+
+class NFBIE(NFBBaseIE):
+    IE_NAME = 'nfb'
+    IE_DESC = 'nfb.ca and onf.ca films and episodes'
+    _VALID_URL = [
+        rf'{NFBBaseIE._VALID_URL_BASE}/(?P<type>film)/(?P<id>[^/?#&]+)',
+        rf'{NFBBaseIE._VALID_URL_BASE}/(?P<type>series?)/(?P<id>[^/?#&]+/s(?:ea|ai)son\d+/episode\d+)',
+    ]
     _TESTS = [{
+        'note': 'NFB film',
         'url': 'https://www.nfb.ca/film/trafficopter/',
         'info_dict': {
             'id': 'trafficopter',
@@ -14,29 +58,192 @@ class NFBIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Barrie Howells',
             'release_year': 1972,
+            'duration': 600.0,
         },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'ONF film',
+        'url': 'https://www.onf.ca/film/mal-du-siecle/',
+        'info_dict': {
+            'id': 'mal-du-siecle',
+            'ext': 'mp4',
+            'title': 'Le mal du siècle',
+            'description': 'md5:1abf774d77569ebe603419f2d344102b',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'Catherine Lepage',
+            'release_year': 2019,
+            'duration': 300.0,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'NFB episode with English title',
+        'url': 'https://www.nfb.ca/series/true-north-inside-the-rise-of-toronto-basketball/season1/episode9/',
+        'info_dict': {
+            'id': 'true-north-episode9-true-north-finale-making-it',
+            'ext': 'mp4',
+            'title': 'True North: Inside the Rise of Toronto Basketball - Finale: Making It',
+            'description': 'We catch up with each player in the midst of their journey as they reflect on their road ahead.',
+            'series': 'True North: Inside the Rise of Toronto Basketball',
+            'release_year': 2018,
+            'season': 'Season 1',
+            'season_number': 1,
+            'episode': 'Finale: Making It',
+            'episode_number': 9,
+            'uploader': 'Ryan Sidhoo',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'ONF episode with French title',
+        'url': 'https://www.onf.ca/serie/direction-nord-la-montee-du-basketball-a-toronto/saison1/episode9/',
+        'info_dict': {
+            'id': 'direction-nord-episode-9',
+            'ext': 'mp4',
+            'title': 'Direction nord – La montée du basketball à Toronto - Finale : Réussir',
+            'description': 'md5:349a57419b71432b97bf6083d92b029d',
+            'series': 'Direction nord – La montée du basketball à Toronto',
+            'release_year': 2018,
+            'season': 'Saison 1',
+            'season_number': 1,
+            'episode': 'Finale : Réussir',
+            'episode_number': 9,
+            'uploader': 'Ryan Sidhoo',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'NFB episode with French title (needs geo-bypass)',
+        'url': 'https://www.nfb.ca/series/etoile-du-nord/saison1/episode1/',
+        'info_dict': {
+            'id': 'etoile-du-nord-episode-1-lobservation',
+            'ext': 'mp4',
+            'title': 'Étoile du Nord - L\'observation',
+            'description': 'md5:161a4617260dee3de70f509b2c9dd21b',
+            'series': 'Étoile du Nord',
+            'release_year': 2023,
+            'season': 'Saison 1',
+            'season_number': 1,
+            'episode': 'L\'observation',
+            'episode_number': 1,
+            'uploader': 'Patrick Bossé',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'ONF episode with English title (needs geo-bypass)',
+        'url': 'https://www.onf.ca/serie/north-star/season1/episode1/',
+        'info_dict': {
+            'id': 'north-star-episode-1-observation',
+            'ext': 'mp4',
+            'title': 'North Star - Observation',
+            'description': 'md5:c727f370839d8a817392b9e3f23655c7',
+            'series': 'North Star',
+            'release_year': 2023,
+            'season': 'Season 1',
+            'season_number': 1,
+            'episode': 'Observation',
+            'episode_number': 1,
+            'uploader': 'Patrick Bossé',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'NFB episode with /film/ URL and English title (needs geo-bypass)',
+        'url': 'https://www.nfb.ca/film/north-star-episode-1-observation/',
+        'info_dict': {
+            'id': 'north-star-episode-1-observation',
+            'ext': 'mp4',
+            'title': 'North Star - Observation',
+            'description': 'md5:c727f370839d8a817392b9e3f23655c7',
+            'series': 'North Star',
+            'release_year': 2023,
+            'season': 'Season 1',
+            'season_number': 1,
+            'episode': 'Observation',
+            'episode_number': 1,
+            'uploader': 'Patrick Bossé',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'ONF episode with /film/ URL and French title (needs geo-bypass)',
+        'url': 'https://www.onf.ca/film/etoile-du-nord-episode-1-lobservation/',
+        'info_dict': {
+            'id': 'etoile-du-nord-episode-1-lobservation',
+            'ext': 'mp4',
+            'title': 'Étoile du Nord - L\'observation',
+            'description': 'md5:161a4617260dee3de70f509b2c9dd21b',
+            'series': 'Étoile du Nord',
+            'release_year': 2023,
+            'season': 'Saison 1',
+            'season_number': 1,
+            'episode': 'L\'observation',
+            'episode_number': 1,
+            'uploader': 'Patrick Bossé',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'Season 2 episode w/o episode num in id, extract from json ld',
+        'url': 'https://www.onf.ca/film/liste-des-choses-qui-existent-saison-2-ours',
+        'info_dict': {
+            'id': 'liste-des-choses-qui-existent-saison-2-ours',
+            'ext': 'mp4',
+            'title': 'La liste des choses qui existent - L\'ours en peluche',
+            'description': 'md5:d5e8d8fc5f3a7385a9cf0f509b37e28a',
+            'series': 'La liste des choses qui existent',
+            'release_year': 2022,
+            'season': 'Saison 2',
+            'season_number': 2,
+            'episode': 'L\'ours en peluche',
+            'episode_number': 12,
+            'uploader': 'Francis Papillon',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'NFB film /embed/player/ page',
+        'url': 'https://www.nfb.ca/film/afterlife/embed/player/',
+        'info_dict': {
+            'id': 'afterlife',
+            'ext': 'mp4',
+            'title': 'Afterlife',
+            'description': 'md5:84951394f594f1fb1e62d9c43242fdf5',
+            'release_year': 1978,
+            'duration': 420.0,
+            'uploader': 'Ishu Patel',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        site, type_, slug = self._match_valid_url(url).group('site', 'type', 'id')
+        # Need to construct the URL since we match /embed/player/ URLs as well
+        webpage, urlh = self._download_webpage_handle(f'https://www.{site}.ca/{type_}/{slug}/', slug)
+        # type_ can change from film to serie(s) after redirect; new slug may have episode number
+        type_, slug = self._match_valid_url(urlh.url).group('type', 'id')
 
-        webpage = self._download_webpage('https://www.nfb.ca/film/%s/' % video_id, video_id)
+        embed_url = urljoin(f'https://www.{site}.ca', self._html_search_regex(
+            r'<[^>]+\bid=["\']player-iframe["\'][^>]*\bsrc=["\']([^"\']+)', webpage, 'embed url'))
+        video_id = self._match_id(embed_url)  # embed url has unique slug
+        player = self._download_webpage(embed_url, video_id, 'Downloading player page')
+        if 'MESSAGE_GEOBLOCKED' in player:
+            self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
 
-        iframe = self._html_search_regex(
-            r'<[^>]+\bid=["\']player-iframe["\'][^>]*src=["\']([^"\']+)',
-            webpage, 'iframe', default=None, fatal=True)
-        if iframe.startswith('/'):
-            iframe = f'https://www.nfb.ca{iframe}'
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            self._html_search_regex(r'source:\s*\'([^\']+)', player, 'm3u8 url'),
+            video_id, 'mp4', m3u8_id='hls')
 
-        player = self._download_webpage(iframe, video_id)
+        if dv_source := self._html_search_regex(r'dvSource:\s*\'([^\']+)', player, 'dv', default=None):
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                dv_source, video_id, 'mp4', m3u8_id='dv', preference=-2, fatal=False)
+            for fmt in fmts:
+                fmt['format_note'] = 'described video'
+            formats.extend(fmts)
+            self._merge_subtitles(subs, target=subtitles)
 
-        source = self._html_search_regex(
-            r'source:\s*\'([^\']+)',
-            player, 'source', default=None, fatal=True)
-
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(source, video_id, ext='mp4')
-
-        return {
+        info = {
             'id': video_id,
             'title': self._html_search_regex(
                 r'<[^>]+\bid=["\']titleHeader["\'][^>]*>\s*<h1[^>]*>\s*([^<]+?)\s*</h1>',
@@ -45,14 +252,49 @@ def _real_extract(self, url):
                 r'<[^>]+\bid=["\']tabSynopsis["\'][^>]*>\s*<p[^>]*>\s*([^<]+)',
                 webpage, 'description', default=None),
             'thumbnail': self._html_search_regex(
-                r'poster:\s*\'([^\']+)',
-                player, 'thumbnail', default=None),
+                r'poster:\s*\'([^\']+)', player, 'thumbnail', default=None),
             'uploader': self._html_search_regex(
-                r'<[^>]+\bitemprop=["\']name["\'][^>]*>([^<]+)',
-                webpage, 'uploader', default=None),
+                r'<[^>]+\bitemprop=["\']name["\'][^>]*>([^<]+)', webpage, 'uploader', default=None),
             'release_year': int_or_none(self._html_search_regex(
                 r'<[^>]+\bitemprop=["\']datePublished["\'][^>]*>([^<]+)',
                 webpage, 'release_year', default=None)),
+        } if type_ == 'film' else self._extract_ep_info(self._extract_ep_data(webpage, video_id, slug), video_id)
+
+        return merge_dicts({
             'formats': formats,
             'subtitles': subtitles,
-        }
+        }, info, self._search_json_ld(webpage, video_id, default={}))
+
+
+class NFBSeriesIE(NFBBaseIE):
+    IE_NAME = 'nfb:series'
+    IE_DESC = 'nfb.ca and onf.ca series'
+    _VALID_URL = rf'{NFBBaseIE._VALID_URL_BASE}/(?P<type>series?)/(?P<id>[^/?#&]+)/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://www.nfb.ca/series/true-north-inside-the-rise-of-toronto-basketball/',
+        'playlist_mincount': 9,
+        'info_dict': {
+            'id': 'true-north-inside-the-rise-of-toronto-basketball',
+        },
+    }, {
+        'url': 'https://www.onf.ca/serie/la-liste-des-choses-qui-existent-serie/',
+        'playlist_mincount': 26,
+        'info_dict': {
+            'id': 'la-liste-des-choses-qui-existent-serie',
+        },
+    }]
+
+    def _entries(self, episodes):
+        for episode in traverse_obj(episodes, lambda _, v: NFBIE.suitable(v['embed_url'])):
+            mobj = NFBIE._match_valid_url(episode['embed_url'])
+            yield self.url_result(
+                mobj[0], NFBIE, **self._extract_ep_info([episode], mobj.group('id')))
+
+    def _real_extract(self, url):
+        site, type_, series_id = self._match_valid_url(url).group('site', 'type', 'id')
+        season_path = 'saison' if type_ == 'serie' else 'season'
+        webpage = self._download_webpage(
+            f'https://www.{site}.ca/{type_}/{series_id}/{season_path}1/episode1', series_id)
+        episodes = self._extract_ep_data(webpage, series_id, fatal=True)
+
+        return self.playlist_result(self._entries(episodes), series_id)

From a2bac6b7adb7b0e955125838e20bb39eece630ce Mon Sep 17 00:00:00 2001
From: columndeeply <106948293+columndeeply@users.noreply.github.com>
Date: Wed, 31 Jan 2024 20:16:07 +0000
Subject: [PATCH 835/871] [ie/PrankCastPost] Add extractor (#8933)

Authored by: columndeeply
---
 yt_dlp/extractor/_extractors.py |  2 +-
 yt_dlp/extractor/prankcast.py   | 73 ++++++++++++++++++++++++++++++++-
 2 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 82d3004ba..4c8604099 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1518,7 +1518,7 @@
     PuhuTVSerieIE,
 )
 from .pr0gramm import Pr0grammIE
-from .prankcast import PrankCastIE
+from .prankcast import PrankCastIE, PrankCastPostIE
 from .premiershiprugby import PremiershipRugbyIE
 from .presstv import PressTVIE
 from .projectveritas import ProjectVeritasIE
diff --git a/yt_dlp/extractor/prankcast.py b/yt_dlp/extractor/prankcast.py
index b2ec5bbb8..562aca0ff 100644
--- a/yt_dlp/extractor/prankcast.py
+++ b/yt_dlp/extractor/prankcast.py
@@ -1,5 +1,8 @@
+import json
+
 from .common import InfoExtractor
-from ..utils import parse_iso8601, traverse_obj, try_call
+from ..utils import float_or_none, parse_iso8601, str_or_none, try_call
+from ..utils.traversal import traverse_obj
 
 
 class PrankCastIE(InfoExtractor):
@@ -64,3 +67,71 @@ def _real_extract(self, url):
             'categories': [json_info.get('broadcast_category')],
             'tags': try_call(lambda: json_info['broadcast_tags'].split(','))
         }
+
+
+class PrankCastPostIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?prankcast\.com/[^/?#]+/posts/(?P<id>\d+)-(?P<display_id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://prankcast.com/devonanustart/posts/6214-happy-national-rachel-day-',
+        'info_dict': {
+            'id': '6214',
+            'ext': 'mp3',
+            'title': 'Happy National Rachel Day!',
+            'display_id': 'happy-national-rachel-day-',
+            'timestamp': 1704333938,
+            'uploader': 'Devonanustart',
+            'channel_id': '4',
+            'duration': 13175,
+            'cast': ['Devonanustart'],
+            'description': '',
+            'categories': ['prank call'],
+            'upload_date': '20240104'
+        }
+    }, {
+        'url': 'https://prankcast.com/despicabledogs/posts/6217-jake-the-work-crow-',
+        'info_dict': {
+            'id': '6217',
+            'ext': 'mp3',
+            'title': 'Jake the Work Crow!',
+            'display_id': 'jake-the-work-crow-',
+            'timestamp': 1704346592,
+            'uploader': 'despicabledogs',
+            'channel_id': '957',
+            'duration': 263.287,
+            'cast': ['despicabledogs'],
+            'description': 'https://imgur.com/a/vtxLvKU',
+            'categories': [],
+            'upload_date': '20240104'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
+
+        webpage = self._download_webpage(url, video_id)
+        post = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['ssr_data_posts']
+        content = self._parse_json(post['post_contents_json'], video_id)[0]
+
+        uploader = post.get('user_name')
+        guests_json = traverse_obj(content, ('guests_json', {json.loads}, {dict})) or {}
+
+        return {
+            'id': video_id,
+            'title': post.get('post_title') or self._og_search_title(webpage),
+            'display_id': display_id,
+            'url': content.get('url'),
+            'timestamp': parse_iso8601(content.get('start_date') or content.get('crdate'), ' '),
+            'uploader': uploader,
+            'channel_id': str_or_none(post.get('user_id')),
+            'duration': float_or_none(content.get('duration')),
+            'cast': list(filter(None, [uploader] + traverse_obj(guests_json, (..., 'name')))),
+            'description': post.get('post_body'),
+            'categories': list(filter(None, [content.get('category')])),
+            'tags': try_call(lambda: list(filter('', post['post_tags'].split(',')))),
+            'subtitles': {
+                'live_chat': [{
+                    'url': f'https://prankcast.com/api/private/chat/select-broadcast?id={post["content_id"]}&cache=',
+                    'ext': 'json',
+                }],
+            } if post.get('content_id') else None
+        }

From fc2cc626f07328a6c71b5e21853e4cfa7b1e6256 Mon Sep 17 00:00:00 2001
From: garret <garret1317@yandex.com>
Date: Wed, 31 Jan 2024 20:21:59 +0000
Subject: [PATCH 836/871] [ie/cineverse] Detect when login required (#9081)

Partially addresses #9072
Authored by: garret1317
---
 yt_dlp/extractor/cineverse.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/cineverse.py b/yt_dlp/extractor/cineverse.py
index c9fa789b7..032c4334b 100644
--- a/yt_dlp/extractor/cineverse.py
+++ b/yt_dlp/extractor/cineverse.py
@@ -67,7 +67,10 @@ def _real_extract(self, url):
         html = self._download_webpage(url, video_id)
         idetails = self._search_nextjs_data(html, video_id)['props']['pageProps']['idetails']
 
-        if idetails.get('err_code') == 1200:
+        err_code = idetails.get('err_code')
+        if err_code == 1002:
+            self.raise_login_required()
+        elif err_code == 1200:
             self.raise_geo_restricted(
                 'This video is not available from your location due to geo restriction. '
                 'You may be able to bypass it by using the /details/ page instead of the /watch/ page',

From 2f4b57594673035a59d72f7667588da848820034 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Sat, 3 Feb 2024 05:56:29 +0900
Subject: [PATCH 837/871] [ie/zetland] Add extractor (#9116)

Closes #9024
Authored by: HobbyistDev
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/zetland.py     | 71 +++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 yt_dlp/extractor/zetland.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 4c8604099..7726fe359 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -2496,6 +2496,7 @@
     Zee5SeriesIE,
 )
 from .zeenews import ZeeNewsIE
+from .zetland import ZetlandDKArticleIE
 from .zhihu import ZhihuIE
 from .zingmp3 import (
     ZingMp3IE,
diff --git a/yt_dlp/extractor/zetland.py b/yt_dlp/extractor/zetland.py
new file mode 100644
index 000000000..055a643b3
--- /dev/null
+++ b/yt_dlp/extractor/zetland.py
@@ -0,0 +1,71 @@
+from .common import InfoExtractor
+from ..utils import merge_dicts, unified_timestamp, url_or_none
+from ..utils.traversal import traverse_obj
+
+
+class ZetlandDKArticleIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.zetland\.dk/\w+/(?P<id>(?P<story_id>\w{8})-(?P<uploader_id>\w{8})-(?:\w{5}))'
+    _TESTS = [{
+        'url': 'https://www.zetland.dk/historie/sO9aq2MY-a81VP3BY-66e69?utm_source=instagram&utm_medium=linkibio&utm_campaign=artikel',
+        'info_dict': {
+            'id': 'sO9aq2MY-a81VP3BY-66e69',
+            'ext': 'mp3',
+            'modified_date': '20240118',
+            'title': 'Afsnit 1: “Det føltes som en kidnapning.” ',
+            'upload_date': '20240116',
+            'uploader_id': 'a81VP3BY',
+            'modified_timestamp': 1705568739,
+            'release_timestamp': 1705377592,
+            'uploader_url': 'https://www.zetland.dk/skribent/a81VP3BY',
+            'uploader': 'Helle Fuusager',
+            'release_date': '20240116',
+            'thumbnail': r're:https://zetland\.imgix\.net/2aafe500-b14e-11ee-bf83-65d5e1283a57/Zetland_Image_1\.jpg',
+            'description': 'md5:9619d426772c133f5abb26db27f26a01',
+            'timestamp': 1705377592,
+            'series_id': '62d54630-e87b-4ab1-a255-8de58dbe1b14',
+        }
+
+    }]
+
+    def _real_extract(self, url):
+        display_id, uploader_id = self._match_valid_url(url).group('id', 'uploader_id')
+        webpage = self._download_webpage(url, display_id)
+
+        next_js_data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']
+        story_data = traverse_obj(next_js_data, ('initialState', 'consume', 'story', 'story'))
+
+        formats = []
+        for audio_url in traverse_obj(story_data, ('story_content', 'meta', 'audioFiles', ..., {url_or_none})):
+            formats.append({
+                'url': audio_url,
+                'vcodec': 'none',
+            })
+
+        return merge_dicts({
+            'id': display_id,
+            'formats': formats,
+            'uploader_id': uploader_id
+        }, traverse_obj(story_data, {
+            'title': ((('story_content', 'content', 'title'), 'title'), {str}),
+            'uploader': ('sharer', 'name'),
+            'uploader_id': ('sharer', 'sharer_id'),
+            'description': ('story_content', 'content', 'socialDescription'),
+            'series_id': ('story_content', 'meta', 'seriesId'),
+            'release_timestamp': ('published_at', {unified_timestamp}),
+            'modified_timestamp': ('revised_at', {unified_timestamp}),
+        }, get_all=False), traverse_obj(next_js_data, ('metaInfo', {
+            'title': ((('meta', 'title'), ('ld', 'headline'), ('og', 'og:title'), ('og', 'twitter:title')), {str}),
+            'description': ((('meta', 'description'), ('ld', 'description'), ('og', 'og:description'), ('og', 'twitter:description')), {str}),
+            'uploader': ((('meta', 'author'), ('ld', 'author', 'name')), {str}),
+            'uploader_url': ('ld', 'author', 'url', {url_or_none}),
+            'thumbnail': ((('ld', 'image'), ('og', 'og:image'), ('og', 'twitter:image')), {url_or_none}),
+            'modified_timestamp': ('ld', 'dateModified', {unified_timestamp}),
+            'release_timestamp': ('ld', 'datePublished', {unified_timestamp}),
+            'timestamp': ('ld', 'dateCreated', {unified_timestamp}),
+        }), get_all=False), {
+            'title': self._html_search_meta(['title', 'og:title', 'twitter:title'], webpage),
+            'description': self._html_search_meta(['description', 'og:description', 'twitter:description'], webpage),
+            'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage),
+            'uploader': self._html_search_meta(['author'], webpage),
+            'release_timestamp': unified_timestamp(self._html_search_meta(['article:published_time'], webpage)),
+        }, self._search_json_ld(webpage, display_id, fatal=False))

From a0d50aabc5462aee302bd3f2663d3a3554875789 Mon Sep 17 00:00:00 2001
From: HobbyistDev <105957301+HobbyistDev@users.noreply.github.com>
Date: Sat, 3 Feb 2024 05:57:53 +0900
Subject: [PATCH 838/871] [ie/orf:on] Add extractor (#9113)

Closes #8903
Authored by: HobbyistDev
---
 yt_dlp/extractor/_extractors.py |  1 +
 yt_dlp/extractor/orf.py         | 64 +++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 7726fe359..04318a716 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1394,6 +1394,7 @@
 from .orf import (
     ORFTVthekIE,
     ORFFM4StoryIE,
+    ORFONIE,
     ORFRadioIE,
     ORFPodcastIE,
     ORFIPTVIE,
diff --git a/yt_dlp/extractor/orf.py b/yt_dlp/extractor/orf.py
index 9a48ae1b3..1b2a79a62 100644
--- a/yt_dlp/extractor/orf.py
+++ b/yt_dlp/extractor/orf.py
@@ -1,3 +1,4 @@
+import base64
 import functools
 import re
 
@@ -565,3 +566,66 @@ def _real_extract(self, url):
             })
 
         return self.playlist_result(entries)
+
+
+class ORFONIE(InfoExtractor):
+    IE_NAME = 'orf:on'
+    _VALID_URL = r'https?://on\.orf\.at/video/(?P<id>\d{8})/(?P<slug>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://on.orf.at/video/14210000/school-of-champions-48',
+        'info_dict': {
+            'id': '14210000',
+            'ext': 'mp4',
+            'duration': 2651.08,
+            'thumbnail': 'https://api-tvthek.orf.at/assets/segments/0167/98/thumb_16697671_segments_highlight_teaser.jpeg',
+            'title': 'School of Champions (4/8)',
+            'description': 'md5:d09ad279fc2e8502611e7648484b6afd',
+            'media_type': 'episode',
+            'timestamp': 1706472362,
+            'upload_date': '20240128',
+        }
+    }]
+
+    def _extract_video(self, video_id, display_id):
+        encrypted_id = base64.b64encode(f'3dSlfek03nsLKdj4Jsd{video_id}'.encode()).decode()
+        api_json = self._download_json(
+            f'https://api-tvthek.orf.at/api/v4.3/public/episode/encrypted/{encrypted_id}', display_id)
+
+        formats, subtitles = [], {}
+        for manifest_type in traverse_obj(api_json, ('sources', {dict.keys}, ...)):
+            for manifest_url in traverse_obj(api_json, ('sources', manifest_type, ..., 'src', {url_or_none})):
+                if manifest_type == 'hls':
+                    fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                        manifest_url, display_id, fatal=False, m3u8_id='hls')
+                elif manifest_type == 'dash':
+                    fmts, subs = self._extract_mpd_formats_and_subtitles(
+                        manifest_url, display_id, fatal=False, mpd_id='dash')
+                else:
+                    continue
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(api_json, {
+                'duration': ('duration_second', {float_or_none}),
+                'title': (('title', 'headline'), {str}),
+                'description': (('description', 'teaser_text'), {str}),
+                'media_type': ('video_type', {str}),
+            }, get_all=False),
+        }
+
+    def _real_extract(self, url):
+        video_id, display_id = self._match_valid_url(url).group('id', 'slug')
+        webpage = self._download_webpage(url, display_id)
+
+        return {
+            'id': video_id,
+            'title': self._html_search_meta(['og:title', 'twitter:title'], webpage, default=None),
+            'description': self._html_search_meta(
+                ['description', 'og:description', 'twitter:description'], webpage, default=None),
+            **self._search_json_ld(webpage, display_id, fatal=False),
+            **self._extract_video(video_id, display_id),
+        }

From ffa017cfc5973b265c92248546fcf5020dc43eaf Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Fri, 2 Feb 2024 16:08:29 -0500
Subject: [PATCH 839/871] [ie/BiliBiliSearch] Set cookie to fix extraction
 (#9119)

Closes #5083
Authored by: c-basalt
---
 yt_dlp/extractor/bilibili.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index cd7df69ef..4ed9e2af7 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -7,6 +7,7 @@
 import re
 import time
 import urllib.parse
+import uuid
 
 from .common import InfoExtractor, SearchInfoExtractor
 from ..dependencies import Cryptodome
@@ -1464,8 +1465,37 @@ class BiliBiliSearchIE(SearchInfoExtractor):
     IE_DESC = 'Bilibili video search'
     _MAX_RESULTS = 100000
     _SEARCH_KEY = 'bilisearch'
+    _TESTS = [{
+        'url': 'bilisearch3:靡烟 出道一年，我怎么还在等你单推的女人睡觉后开播啊',
+        'playlist_count': 3,
+        'info_dict': {
+            'id': '靡烟 出道一年，我怎么还在等你单推的女人睡觉后开播啊',
+            'title': '靡烟 出道一年，我怎么还在等你单推的女人睡觉后开播啊',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'BV1n44y1Q7sc',
+                'ext': 'mp4',
+                'title': '“出道一年，我怎么还在等你单推的女人睡觉后开播啊？”【一分钟了解靡烟miya】',
+                'timestamp': 1669889987,
+                'upload_date': '20221201',
+                'description': 'md5:43343c0973defff527b5a4b403b4abf9',
+                'tags': list,
+                'uploader': '靡烟miya',
+                'duration': 123.156,
+                'uploader_id': '1958703906',
+                'comment_count': int,
+                'view_count': int,
+                'like_count': int,
+                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+                '_old_archive_ids': ['bilibili 988222410_part1'],
+            },
+        }],
+    }]
 
     def _search_results(self, query):
+        if not self._get_cookies('https://api.bilibili.com').get('buvid3'):
+            self._set_cookie('.bilibili.com', 'buvid3', f'{uuid.uuid4()}infoc')
         for page_num in itertools.count(1):
             videos = self._download_json(
                 'https://api.bilibili.com/x/web-interface/search/type', query,

From 8e765755f7f4909e1b535e61b7376b2d66e1ba6a Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Fri, 2 Feb 2024 15:15:04 -0600
Subject: [PATCH 840/871] [ie/vimeo] Fix API headers (#9125)

Closes #9124
Authored by: bashonly
---
 yt_dlp/extractor/vimeo.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py
index e5e8144bb..208e11184 100644
--- a/yt_dlp/extractor/vimeo.py
+++ b/yt_dlp/extractor/vimeo.py
@@ -269,7 +269,7 @@ def _extract_original_format(self, url, video_id, unlisted_hash=None):
             'https://vimeo.com/_rv/viewer', video_id, note='Downloading jwt token', fatal=False) or {}
         if not jwt_response.get('jwt'):
             return
-        headers = {'Authorization': 'jwt %s' % jwt_response['jwt']}
+        headers = {'Authorization': 'jwt %s' % jwt_response['jwt'], 'Accept': 'application/json'}
         original_response = self._download_json(
             f'https://api.vimeo.com/videos/{video_id}', video_id,
             headers=headers, fatal=False, expected_status=(403, 404)) or {}
@@ -751,6 +751,7 @@ def _extract_from_api(self, video_id, unlisted_hash=None):
         video = self._download_json(
             api_url, video_id, headers={
                 'Authorization': 'jwt ' + token,
+                'Accept': 'application/json',
             }, query={
                 'fields': 'config_url,created_time,description,license,metadata.connections.comments.total,metadata.connections.likes.total,release_time,stats.plays',
             })
@@ -785,7 +786,7 @@ def _try_album_password(self, url):
         jwt = viewer['jwt']
         album = self._download_json(
             'https://api.vimeo.com/albums/' + album_id,
-            album_id, headers={'Authorization': 'jwt ' + jwt},
+            album_id, headers={'Authorization': 'jwt ' + jwt, 'Accept': 'application/json'},
             query={'fields': 'description,name,privacy'})
         if try_get(album, lambda x: x['privacy']['view']) == 'password':
             password = self.get_param('videopassword')
@@ -1147,10 +1148,12 @@ def _fetch_page(self, album_id, authorization, hashed_pass, page):
                 'https://api.vimeo.com/albums/%s/videos' % album_id,
                 album_id, 'Downloading page %d' % api_page, query=query, headers={
                     'Authorization': 'jwt ' + authorization,
+                    'Accept': 'application/json',
                 })['data']
         except ExtractorError as e:
             if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                 return
+            raise
         for video in videos:
             link = video.get('link')
             if not link:
@@ -1171,7 +1174,7 @@ def _real_extract(self, url):
         jwt = viewer['jwt']
         album = self._download_json(
             'https://api.vimeo.com/albums/' + album_id,
-            album_id, headers={'Authorization': 'jwt ' + jwt},
+            album_id, headers={'Authorization': 'jwt ' + jwt, 'Accept': 'application/json'},
             query={'fields': 'description,name,privacy'})
         hashed_pass = None
         if try_get(album, lambda x: x['privacy']['view']) == 'password':

From 4253e3b7f483127bd812bdac02466f4a5b47ff34 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sat, 3 Feb 2024 15:59:43 +0100
Subject: [PATCH 841/871] [ie/CCMA] Extract 1080p DASH formats (#9130)

Closes #5755
Authored by: seproDev
---
 yt_dlp/extractor/ccma.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/yt_dlp/extractor/ccma.py b/yt_dlp/extractor/ccma.py
index 88ff82f6e..ab840f301 100644
--- a/yt_dlp/extractor/ccma.py
+++ b/yt_dlp/extractor/ccma.py
@@ -1,6 +1,7 @@
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    determine_ext,
     int_or_none,
     parse_duration,
     parse_resolution,
@@ -60,6 +61,7 @@ def _real_extract(self, url):
             'http://dinamics.ccma.cat/pvideo/media.jsp', media_id, query={
                 'media': media_type,
                 'idint': media_id,
+                'format': 'dm',
             })
 
         formats = []
@@ -69,6 +71,10 @@ def _real_extract(self, url):
                 format_url = url_or_none(format_.get('file'))
                 if not format_url:
                     continue
+                if determine_ext(format_url) == 'mpd':
+                    formats.extend(self._extract_mpd_formats(
+                        format_url, media_id, mpd_id='dash', fatal=False))
+                    continue
                 label = format_.get('label')
                 f = parse_resolution(label)
                 f.update({

From e3ce2b385ec1f03fac9d4210c57fda77134495fc Mon Sep 17 00:00:00 2001
From: YoshichikaAAA <154937389+YoshichikaAAA@users.noreply.github.com>
Date: Sun, 4 Feb 2024 03:44:17 +0900
Subject: [PATCH 842/871] [ie/radiko] Extract more metadata (#9115)

Authored by: YoshichikaAAA
---
 yt_dlp/extractor/radiko.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/yt_dlp/extractor/radiko.py b/yt_dlp/extractor/radiko.py
index c363d9ba5..2b6405999 100644
--- a/yt_dlp/extractor/radiko.py
+++ b/yt_dlp/extractor/radiko.py
@@ -1,5 +1,6 @@
 import base64
 import random
+import re
 import urllib.parse
 
 from .common import InfoExtractor
@@ -11,6 +12,7 @@
     unified_timestamp,
     update_url_query,
 )
+from ..utils.traversal import traverse_obj
 
 
 class RadikoBaseIE(InfoExtractor):
@@ -159,6 +161,12 @@ def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token,
 
         return formats
 
+    def _extract_performers(self, prog):
+        performers = traverse_obj(prog, (
+            'pfm/text()', ..., {lambda x: re.split(r'[/／、　,，]', x)}, ..., {str.strip}))
+        # TODO: change 'artist' fields to 'artists' and return traversal list instead of str
+        return ', '.join(performers) or None
+
 
 class RadikoIE(RadikoBaseIE):
     _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<id>\d+)'
@@ -186,10 +194,12 @@ def _real_extract(self, url):
         return {
             'id': video_id,
             'title': try_call(lambda: prog.find('title').text),
+            'artist': self._extract_performers(prog),
             'description': clean_html(try_call(lambda: prog.find('info').text)),
             'uploader': try_call(lambda: station_program.find('.//name').text),
             'uploader_id': station,
             'timestamp': vid_int,
+            'duration': try_call(lambda: unified_timestamp(radio_end, False) - unified_timestamp(radio_begin, False)),
             'is_live': True,
             'formats': self._extract_formats(
                 video_id=video_id, station=station, is_onair=False,
@@ -243,6 +253,7 @@ def _real_extract(self, url):
         return {
             'id': station,
             'title': title,
+            'artist': self._extract_performers(prog),
             'description': description,
             'uploader': station_name,
             'uploader_id': station,

From 96d0f8c1cb8aec250c5614bfde6b5fb95f10819b Mon Sep 17 00:00:00 2001
From: Michal <salin87@gmail.com>
Date: Mon, 5 Feb 2024 00:25:13 +0100
Subject: [PATCH 843/871] [ie/eporner] Extract AV1 formats (#9028)

Authored by: michal-repo
---
 yt_dlp/extractor/eporner.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/eporner.py b/yt_dlp/extractor/eporner.py
index aee2dee58..b18a76c7c 100644
--- a/yt_dlp/extractor/eporner.py
+++ b/yt_dlp/extractor/eporner.py
@@ -1,8 +1,10 @@
 from .common import InfoExtractor
 from ..utils import (
-    encode_base_n,
     ExtractorError,
+    encode_base_n,
+    get_elements_by_class,
     int_or_none,
+    join_nonempty,
     merge_dicts,
     parse_duration,
     str_to_int,
@@ -81,6 +83,7 @@ def calc_hash(s):
         sources = video['sources']
 
         formats = []
+        has_av1 = bool(get_elements_by_class('download-av1', webpage))
         for kind, formats_dict in sources.items():
             if not isinstance(formats_dict, dict):
                 continue
@@ -106,6 +109,14 @@ def calc_hash(s):
                         'height': height,
                         'fps': fps,
                     })
+                    if has_av1:
+                        formats.append({
+                            'url': src.replace('.mp4', '-av1.mp4'),
+                            'format_id': join_nonempty('av1', format_id),
+                            'height': height,
+                            'fps': fps,
+                            'vcodec': 'av1',
+                        })
 
         json_ld = self._search_json_ld(webpage, display_id, default={})
 

From e439693f729daf6fb15457baea1bca10ef5da34d Mon Sep 17 00:00:00 2001
From: c-basalt <117849907+c-basalt@users.noreply.github.com>
Date: Sun, 4 Feb 2024 18:28:45 -0500
Subject: [PATCH 844/871] [ie/bilibili] Support `--no-playlist` (#9139)

Addresses #8499
Authored by: c-basalt
---
 yt_dlp/extractor/bilibili.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index 4ed9e2af7..c138bde3a 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -1305,6 +1305,26 @@ class BilibiliPlaylistIE(BilibiliSpaceListBaseIE):
             'upload_date': '20211127',
         },
         'playlist_mincount': 513,
+    }, {
+        'url': 'https://www.bilibili.com/list/1958703906?sid=547718&oid=687146339&bvid=BV1DU4y1r7tz',
+        'info_dict': {
+            'id': 'BV1DU4y1r7tz',
+            'ext': 'mp4',
+            'title': '【直播回放】8.20晚9:30 3d发布喵 2022年8月20日21点场',
+            'upload_date': '20220820',
+            'description': '',
+            'timestamp': 1661016330,
+            'uploader_id': '1958703906',
+            'uploader': '靡烟miya',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'duration': 9552.903,
+            'tags': list,
+            'comment_count': int,
+            'view_count': int,
+            'like_count': int,
+            '_old_archive_ids': ['bilibili 687146339_part1'],
+        },
+        'params': {'noplaylist': True},
     }, {
         'url': 'https://www.bilibili.com/medialist/play/1958703906?business=space_series&business_id=547718&desc=1',
         'info_dict': {
@@ -1356,6 +1376,11 @@ def _extract_medialist(self, query, list_id):
 
     def _real_extract(self, url):
         list_id = self._match_id(url)
+
+        bvid = traverse_obj(parse_qs(url), ('bvid', 0))
+        if not self._yes_playlist(list_id, bvid):
+            return self.url_result(f'https://www.bilibili.com/video/{bvid}', BiliBiliIE)
+
         webpage = self._download_webpage(url, list_id)
         initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', list_id)
         if traverse_obj(initial_state, ('error', 'code', {int_or_none})) != 200:

From 07256b9fee23960799024b95d5972abc7174aa81 Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Mon, 5 Feb 2024 00:35:52 +0000
Subject: [PATCH 845/871] [ie/nytimes] Overhaul extractors (#9075)

Closes #2899, Closes #8605
Authored by: SirElderling
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/nytimes.py     | 448 +++++++++++++++++++++-----------
 2 files changed, 302 insertions(+), 147 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 04318a716..36335286c 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1352,6 +1352,7 @@
     NYTimesIE,
     NYTimesArticleIE,
     NYTimesCookingIE,
+    NYTimesCookingRecipeIE,
 )
 from .nuvid import NuvidIE
 from .nzherald import NZHeraldIE
diff --git a/yt_dlp/extractor/nytimes.py b/yt_dlp/extractor/nytimes.py
index 2e21edbb4..354eb02c3 100644
--- a/yt_dlp/extractor/nytimes.py
+++ b/yt_dlp/extractor/nytimes.py
@@ -1,50 +1,92 @@
-import hmac
-import hashlib
-import base64
+import json
+import uuid
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
+    clean_html,
     determine_ext,
+    extract_attributes,
     float_or_none,
+    get_elements_html_by_class,
     int_or_none,
-    js_to_json,
+    merge_dicts,
     mimetype2ext,
     parse_iso8601,
+    remove_end,
     remove_start,
+    str_or_none,
+    traverse_obj,
+    url_or_none,
 )
 
 
 class NYTimesBaseIE(InfoExtractor):
-    _SECRET = b'pX(2MbU2);4N{7J8)>YwKRJ+/pQ3JkiU2Q^V>mFYv6g6gYvt6v'
+    _DNS_NAMESPACE = uuid.UUID('36dd619a-56dc-595b-9e09-37f4152c7b5d')
+    _TOKEN = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuNIzKBOFB77aT/jN/FQ+/QVKWq5V1ka1AYmCR9hstz1pGNPH5ajOU9gAqta0T89iPnhjwla+3oec/Z3kGjxbpv6miQXufHFq3u2RC6HyU458cLat5kVPSOQCe3VVB5NRpOlRuwKHqn0txfxnwSSj8mqzstR997d3gKB//RO9zE16y3PoWlDQXkASngNJEWvL19iob/xwAkfEWCjyRILWFY0JYX3AvLMSbq7wsqOCE5srJpo7rRU32zsByhsp1D5W9OYqqwDmflsgCEQy2vqTsJjrJohuNg+urMXNNZ7Y3naMoqttsGDrWVxtPBafKMI8pM2ReNZBbGQsQXRzQNo7+QIDAQAB'
+    _GRAPHQL_API = 'https://samizdat-graphql.nytimes.com/graphql/v2'
+    _GRAPHQL_QUERY = '''query VideoQuery($id: String!) {
+  video(id: $id) {
+    ... on Video {
+      bylines {
+        renderedRepresentation
+      }
+      duration
+      promotionalHeadline
+      promotionalMedia {
+        ... on Image {
+          crops {
+            name
+            renditions {
+              name
+              width
+              height
+              url
+            }
+          }
+        }
+      }
+      renditions {
+        type
+        width
+        height
+        url
+        bitrate
+      }
+      summary
+    }
+  }
+}'''
 
-    def _extract_video_from_id(self, video_id):
-        # Authorization generation algorithm is reverse engineered from `signer` in
-        # http://graphics8.nytimes.com/video/vhs/vhs-2.x.min.js
-        path = '/svc/video/api/v3/video/' + video_id
-        hm = hmac.new(self._SECRET, (path + ':vhs').encode(), hashlib.sha512).hexdigest()
-        video_data = self._download_json('http://www.nytimes.com' + path, video_id, 'Downloading video JSON', headers={
-            'Authorization': 'NYTV ' + base64.b64encode(hm.encode()).decode(),
-            'X-NYTV': 'vhs',
-        }, fatal=False)
-        if not video_data:
-            video_data = self._download_json(
-                'http://www.nytimes.com/svc/video/api/v2/video/' + video_id,
-                video_id, 'Downloading video JSON')
+    def _call_api(self, media_id):
+        # reference: `id-to-uri.js`
+        video_uuid = uuid.uuid5(self._DNS_NAMESPACE, 'video')
+        media_uuid = uuid.uuid5(video_uuid, media_id)
 
-        title = video_data['headline']
+        return traverse_obj(self._download_json(
+            self._GRAPHQL_API, media_id, 'Downloading JSON from GraphQL API', data=json.dumps({
+                'query': self._GRAPHQL_QUERY,
+                'variables': {'id': f'nyt://video/{media_uuid}'},
+            }, separators=(',', ':')).encode(), headers={
+                'Content-Type': 'application/json',
+                'Nyt-App-Type': 'vhs',
+                'Nyt-App-Version': 'v3.52.21',
+                'Nyt-Token': self._TOKEN,
+                'Origin': 'https://nytimes.com',
+            }, fatal=False), ('data', 'video', {dict})) or {}
 
-        def get_file_size(file_size):
-            if isinstance(file_size, int):
-                return file_size
-            elif isinstance(file_size, dict):
-                return int(file_size.get('value', 0))
-            else:
-                return None
+    def _extract_thumbnails(self, thumbs):
+        return traverse_obj(thumbs, (lambda _, v: url_or_none(v['url']), {
+            'url': 'url',
+            'width': ('width', {int_or_none}),
+            'height': ('height', {int_or_none}),
+        }), default=None)
 
+    def _extract_formats_and_subtitles(self, video_id, content_media_json):
         urls = []
         formats = []
         subtitles = {}
-        for video in video_data.get('renditions', []):
+        for video in traverse_obj(content_media_json, ('renditions', ..., {dict})):
             video_url = video.get('url')
             format_id = video.get('type')
             if not video_url or format_id == 'thumbs' or video_url in urls:
@@ -56,11 +98,9 @@ def get_file_size(file_size):
                     video_url, video_id, 'mp4', 'm3u8_native',
                     m3u8_id=format_id or 'hls', fatal=False)
                 formats.extend(m3u8_fmts)
-                subtitles = self._merge_subtitles(subtitles, m3u8_subs)
+                self._merge_subtitles(m3u8_subs, target=subtitles)
             elif ext == 'mpd':
-                continue
-            #     formats.extend(self._extract_mpd_formats(
-            #         video_url, video_id, format_id or 'dash', fatal=False))
+                continue  # all mpd urls give 404 errors
             else:
                 formats.append({
                     'url': video_url,
@@ -68,55 +108,49 @@ def get_file_size(file_size):
                     'vcodec': video.get('videoencoding') or video.get('video_codec'),
                     'width': int_or_none(video.get('width')),
                     'height': int_or_none(video.get('height')),
-                    'filesize': get_file_size(video.get('file_size') or video.get('fileSize')),
+                    'filesize': traverse_obj(video, (
+                        ('file_size', 'fileSize'), (None, ('value')), {int_or_none}), get_all=False),
                     'tbr': int_or_none(video.get('bitrate'), 1000) or None,
                     'ext': ext,
                 })
 
-        thumbnails = []
-        for image in video_data.get('images', []):
-            image_url = image.get('url')
-            if not image_url:
-                continue
-            thumbnails.append({
-                'url': 'http://www.nytimes.com/' + image_url,
-                'width': int_or_none(image.get('width')),
-                'height': int_or_none(image.get('height')),
-            })
+        return formats, subtitles
 
-        publication_date = video_data.get('publication_date')
-        timestamp = parse_iso8601(publication_date[:-8]) if publication_date else None
+    def _extract_video(self, media_id):
+        data = self._call_api(media_id)
+        formats, subtitles = self._extract_formats_and_subtitles(media_id, data)
 
         return {
-            'id': video_id,
-            'title': title,
-            'description': video_data.get('summary'),
-            'timestamp': timestamp,
-            'uploader': video_data.get('byline'),
-            'duration': float_or_none(video_data.get('duration'), 1000),
+            'id': media_id,
+            'title': data.get('promotionalHeadline'),
+            'description': data.get('summary'),
+            'duration': float_or_none(data.get('duration'), scale=1000),
+            'creator': ', '.join(traverse_obj(data, (  # TODO: change to 'creators'
+                'bylines', ..., 'renderedRepresentation', {lambda x: remove_start(x, 'By ')}))),
             'formats': formats,
             'subtitles': subtitles,
-            'thumbnails': thumbnails,
+            'thumbnails': self._extract_thumbnails(
+                traverse_obj(data, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
         }
 
 
 class NYTimesIE(NYTimesBaseIE):
     _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
     _EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>']
-
     _TESTS = [{
         'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
-        'md5': 'd665342765db043f7e225cff19df0f2d',
+        'md5': 'a553aa344014e3723d33893d89d4defc',
         'info_dict': {
             'id': '100000002847155',
-            'ext': 'mov',
+            'ext': 'mp4',
             'title': 'Verbatim: What Is a Photocopier?',
             'description': 'md5:93603dada88ddbda9395632fdc5da260',
-            'timestamp': 1398631707,
-            'upload_date': '20140427',
-            'uploader': 'Brett Weiner',
+            'timestamp': 1398631707,  # FIXME
+            'upload_date': '20140427',  # FIXME
+            'creator': 'Brett Weiner',
+            'thumbnail': r're:https?://\w+\.nyt.com/images/.+\.jpg',
             'duration': 419,
-        }
+        },
     }, {
         'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html',
         'only_matching': True,
@@ -125,138 +159,258 @@ class NYTimesIE(NYTimesBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        return self._extract_video_from_id(video_id)
+        return self._extract_video(video_id)
 
 
 class NYTimesArticleIE(NYTimesBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
+    _VALID_URL = r'https?://(?:www\.)?nytimes\.com/\d{4}/\d{2}/\d{2}/(?!books|podcasts)[^/?#]+/(?:\w+/)?(?P<id>[^./?#]+)(?:\.html)?'
     _TESTS = [{
         'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0',
-        'md5': 'e2076d58b4da18e6a001d53fd56db3c9',
+        'md5': '3eb5ddb1d6f86254fe4f233826778737',
         'info_dict': {
             'id': '100000003628438',
-            'ext': 'mov',
-            'title': 'New Minimum Wage: $70,000 a Year',
-            'description': 'Dan Price, C.E.O. of Gravity Payments, surprised his 120-person staff by announcing that he planned over the next three years to raise the salary of every employee to $70,000 a year.',
-            'timestamp': 1429033037,
+            'ext': 'mp4',
+            'title': 'One Company’s New Minimum Wage: $70,000 a Year',
+            'description': 'md5:89ba9ab67ca767bb92bf823d1f138433',
+            'timestamp': 1429047468,
             'upload_date': '20150414',
             'uploader': 'Matthew Williams',
-        }
+            'creator': 'Patricia Cohen',
+            'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
+            'duration': 119.0,
+        },
     }, {
-        'url': 'http://www.nytimes.com/2016/10/14/podcasts/revelations-from-the-final-weeks.html',
-        'md5': 'e0d52040cafb07662acf3c9132db3575',
+        # article with audio and no video
+        'url': 'https://www.nytimes.com/2023/09/29/health/mosquitoes-genetic-engineering.html',
+        'md5': '2365b3555c8aa7f4dd34ca735ad02e6a',
         'info_dict': {
-            'id': '100000004709062',
-            'title': 'The Run-Up: ‘He Was Like an Octopus’',
+            'id': '100000009110381',
             'ext': 'mp3',
-            'description': 'md5:fb5c6b93b12efc51649b4847fe066ee4',
-            'series': 'The Run-Up',
-            'episode': '‘He Was Like an Octopus’',
-            'episode_number': 20,
-            'duration': 2130,
-        }
+            'title': 'The Gamble: Can Genetically Modified Mosquitoes End Disease?',
+            'description': 'md5:9ff8b47acbaf7f3ca8c732f5c815be2e',
+            'timestamp': 1695960700,
+            'upload_date': '20230929',
+            'creator': 'Stephanie Nolen, Natalija Gormalova',
+            'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
+            'duration': 1322,
+        },
     }, {
-        'url': 'http://www.nytimes.com/2016/10/16/books/review/inside-the-new-york-times-book-review-the-rise-of-hitler.html',
+        'url': 'https://www.nytimes.com/2023/11/29/business/dealbook/kamala-harris-biden-voters.html',
+        'md5': '3eb5ddb1d6f86254fe4f233826778737',
         'info_dict': {
-            'id': '100000004709479',
-            'title': 'The Rise of Hitler',
-            'ext': 'mp3',
-            'description': 'md5:bce877fd9e3444990cb141875fab0028',
-            'creator': 'Pamela Paul',
-            'duration': 3475,
+            'id': '100000009202270',
+            'ext': 'mp4',
+            'title': 'Kamala Harris Defends Biden Policies, but Says ‘More Work’ Needed to Reach Voters',
+            'description': 'md5:de4212a7e19bb89e4fb14210ca915f1f',
+            'timestamp': 1701290997,
+            'upload_date': '20231129',
+            'uploader': 'By The New York Times',
+            'creator': 'Katie Rogers',
+            'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
+            'duration': 97.631,
         },
         'params': {
-            'skip_download': True,
+            'skip_download': 'm3u8',
         },
     }, {
-        'url': 'http://www.nytimes.com/news/minute/2014/03/17/times-minute-whats-next-in-crimea/?_php=true&_type=blogs&_php=true&_type=blogs&_r=1',
+        # multiple videos in the same article
+        'url': 'https://www.nytimes.com/2023/12/02/business/air-traffic-controllers-safety.html',
+        'info_dict': {
+            'id': 'air-traffic-controllers-safety',
+            'title': 'Drunk and Asleep on the Job: Air Traffic Controllers Pushed to the Brink',
+            'description': 'md5:549e5a5e935bf7d048be53ba3d2c863d',
+            'upload_date': '20231202',
+            'creator': 'Emily Steel, Sydney Ember',
+            'timestamp': 1701511264,
+        },
+        'playlist_count': 3,
+    }, {
+        'url': 'https://www.nytimes.com/2023/12/02/business/media/netflix-squid-game-challenge.html',
         'only_matching': True,
     }]
 
-    def _extract_podcast_from_json(self, json, page_id, webpage):
-        podcast_audio = self._parse_json(
-            json, page_id, transform_source=js_to_json)
+    def _extract_content_from_block(self, block):
+        details = traverse_obj(block, {
+            'id': ('sourceId', {str}),
+            'uploader': ('bylines', ..., 'renderedRepresentation', {str}),
+            'duration': (None, (('duration', {lambda x: float_or_none(x, scale=1000)}), ('length', {int_or_none}))),
+            'timestamp': ('firstPublished', {parse_iso8601}),
+            'series': ('podcastSeries', {str}),
+        }, get_all=False)
 
-        audio_data = podcast_audio['data']
-        track = audio_data['track']
-
-        episode_title = track['title']
-        video_url = track['source']
-
-        description = track.get('description') or self._html_search_meta(
-            ['og:description', 'twitter:description'], webpage)
-
-        podcast_title = audio_data.get('podcast', {}).get('title')
-        title = ('%s: %s' % (podcast_title, episode_title)
-                 if podcast_title else episode_title)
-
-        episode = audio_data.get('podcast', {}).get('episode') or ''
-        episode_number = int_or_none(self._search_regex(
-            r'[Ee]pisode\s+(\d+)', episode, 'episode number', default=None))
+        formats, subtitles = self._extract_formats_and_subtitles(details.get('id'), block)
+        # audio articles will have an url and no formats
+        url = traverse_obj(block, ('fileUrl', {url_or_none}))
+        if not formats and url:
+            formats.append({'url': url, 'vcodec': 'none'})
 
         return {
-            'id': remove_start(podcast_audio.get('target'), 'FT') or page_id,
-            'url': video_url,
-            'title': title,
-            'description': description,
-            'creator': track.get('credit'),
-            'series': podcast_title,
-            'episode': episode_title,
-            'episode_number': episode_number,
-            'duration': int_or_none(track.get('duration')),
+            **details,
+            'thumbnails': self._extract_thumbnails(traverse_obj(
+                block, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
+            'formats': formats,
+            'subtitles': subtitles
         }
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
-
         webpage = self._download_webpage(url, page_id)
+        art_json = self._search_json(
+            r'window\.__preloadedData\s*=', webpage, 'media details', page_id,
+            transform_source=lambda x: x.replace('undefined', 'null'))['initialData']['data']['article']
 
-        video_id = self._search_regex(
-            r'data-videoid=["\'](\d+)', webpage, 'video id',
-            default=None, fatal=False)
-        if video_id is not None:
-            return self._extract_video_from_id(video_id)
+        blocks = traverse_obj(art_json, (
+            'sprinkledBody', 'content', ..., ('ledeMedia', None),
+            lambda _, v: v['__typename'] in ('Video', 'Audio')))
+        if not blocks:
+            raise ExtractorError('Unable to extract any media blocks from webpage')
 
-        podcast_data = self._search_regex(
-            (r'NYTD\.FlexTypes\.push\s*\(\s*({.+?})\s*\)\s*;\s*</script',
-             r'NYTD\.FlexTypes\.push\s*\(\s*({.+})\s*\)\s*;'),
-            webpage, 'podcast data')
-        return self._extract_podcast_from_json(podcast_data, page_id, webpage)
+        common_info = {
+            'title': remove_end(self._html_extract_title(webpage), ' - The New York Times'),
+            'description': traverse_obj(art_json, (
+                'sprinkledBody', 'content', ..., 'summary', 'content', ..., 'text', {str}),
+                get_all=False) or self._html_search_meta(['og:description', 'twitter:description'], webpage),
+            'timestamp': traverse_obj(art_json, ('firstPublished', {parse_iso8601})),
+            'creator': ', '.join(
+                traverse_obj(art_json, ('bylines', ..., 'creators', ..., 'displayName'))),  # TODO: change to 'creators' (list)
+            'thumbnails': self._extract_thumbnails(traverse_obj(
+                art_json, ('promotionalMedia', 'assetCrops', ..., 'renditions', ...))),
+        }
+
+        entries = []
+        for block in blocks:
+            entries.append(merge_dicts(self._extract_content_from_block(block), common_info))
+
+        if len(entries) > 1:
+            return self.playlist_result(entries, page_id, **common_info)
+
+        return {
+            'id': page_id,
+            **entries[0],
+        }
 
 
 class NYTimesCookingIE(NYTimesBaseIE):
-    _VALID_URL = r'https?://cooking\.nytimes\.com/(?:guid|recip)es/(?P<id>\d+)'
+    IE_NAME = 'NYTimesCookingGuide'
+    _VALID_URL = r'https?://cooking\.nytimes\.com/guides/(?P<id>[\w-]+)'
     _TESTS = [{
-        'url': 'https://cooking.nytimes.com/recipes/1017817-cranberry-curd-tart',
-        'md5': 'dab81fa2eaeb3f9ed47498bdcfcdc1d3',
+        'url': 'https://cooking.nytimes.com/guides/13-how-to-cook-a-turkey',
         'info_dict': {
-            'id': '100000004756089',
-            'ext': 'mov',
-            'timestamp': 1479383008,
-            'uploader': 'By SHAW LASH, ADAM SAEWITZ and JAMES HERRON',
-            'title': 'Cranberry Tart',
-            'upload_date': '20161117',
-            'description': 'If you are a fan of lemon curd or the classic French tarte au citron, you will love this cranberry version.',
+            'id': '13-how-to-cook-a-turkey',
+            'title': 'How to Cook a Turkey',
+            'description': 'md5:726cfd3f9b161bdf5c279879e8050ca0',
+        },
+        'playlist_count': 2,
+    }, {
+        # single video example
+        'url': 'https://cooking.nytimes.com/guides/50-how-to-make-mac-and-cheese',
+        'md5': '64415805fe0b8640fce6b0b9def5989a',
+        'info_dict': {
+            'id': '100000005835845',
+            'ext': 'mp4',
+            'title': 'How to Make Mac and Cheese',
+            'description': 'md5:b8f2f33ec1fb7523b21367147c9594f1',
+            'duration': 9.51,
+            'creator': 'Alison Roman',
+            'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
         },
     }, {
-        'url': 'https://cooking.nytimes.com/guides/13-how-to-cook-a-turkey',
-        'md5': '4b2e8c70530a89b8d905a2b572316eb8',
+        'url': 'https://cooking.nytimes.com/guides/20-how-to-frost-a-cake',
+        'md5': '64415805fe0b8640fce6b0b9def5989a',
         'info_dict': {
-            'id': '100000003951728',
-            'ext': 'mov',
-            'timestamp': 1445509539,
-            'description': 'Turkey guide',
-            'upload_date': '20151022',
-            'title': 'Turkey',
-        }
+            'id': '20-how-to-frost-a-cake',
+            'title': 'How to Frost a Cake',
+            'description': 'md5:a31fe3b98a8ce7b98aae097730c269cd',
+        },
+        'playlist_count': 8,
     }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
-
         webpage = self._download_webpage(url, page_id)
+        title = self._html_search_meta(['og:title', 'twitter:title'], webpage)
+        description = self._html_search_meta(['og:description', 'twitter:description'], webpage)
 
-        video_id = self._search_regex(
-            r'data-video-id=["\'](\d+)', webpage, 'video id')
+        lead_video_id = self._search_regex(
+            r'data-video-player-id="(\d+)"></div>', webpage, 'lead video')
+        media_ids = traverse_obj(
+            get_elements_html_by_class('video-item', webpage), (..., {extract_attributes}, 'data-video-id'))
 
-        return self._extract_video_from_id(video_id)
+        if media_ids:
+            media_ids.append(lead_video_id)
+            return self.playlist_result(
+                [self._extract_video(media_id) for media_id in media_ids], page_id, title, description)
+
+        return {
+            **self._extract_video(lead_video_id),
+            'title': title,
+            'description': description,
+            'creator': self._search_regex(  # TODO: change to 'creators'
+                r'<span itemprop="author">([^<]+)</span></p>', webpage, 'author', default=None),
+        }
+
+
+class NYTimesCookingRecipeIE(InfoExtractor):
+    _VALID_URL = r'https?://cooking\.nytimes\.com/recipes/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://cooking.nytimes.com/recipes/1017817-cranberry-curd-tart',
+        'md5': '579e83bbe8e61e9de67f80edba8a78a8',
+        'info_dict': {
+            'id': '1017817',
+            'ext': 'mp4',
+            'title': 'Cranberry Curd Tart',
+            'description': 'md5:ad77a3fc321db636256d4343c5742152',
+            'timestamp': 1447804800,
+            'upload_date': '20151118',
+            'creator': 'David Tanis',
+            'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
+        },
+    }, {
+        'url': 'https://cooking.nytimes.com/recipes/1024781-neapolitan-checkerboard-cookies',
+        'md5': '58df35998241dcf0620e99e646331b42',
+        'info_dict': {
+            'id': '1024781',
+            'ext': 'mp4',
+            'title': 'Neapolitan Checkerboard Cookies',
+            'description': 'md5:ba12394c585ababea951cb6d2fcc6631',
+            'timestamp': 1701302400,
+            'upload_date': '20231130',
+            'creator': 'Sue Li',
+            'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
+        },
+    }, {
+        'url': 'https://cooking.nytimes.com/recipes/1019516-overnight-oats',
+        'md5': '2fe7965a3adc899913b8e25ada360823',
+        'info_dict': {
+            'id': '1019516',
+            'ext': 'mp4',
+            'timestamp': 1546387200,
+            'description': 'md5:8856ce10239161bd2596ac335b9f9bfb',
+            'upload_date': '20190102',
+            'title': 'Overnight Oats',
+            'creator': 'Genevieve Ko',
+            'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
+        },
+    }]
+
+    def _real_extract(self, url):
+        page_id = self._match_id(url)
+        webpage = self._download_webpage(url, page_id)
+        recipe_data = self._search_nextjs_data(webpage, page_id)['props']['pageProps']['recipe']
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            recipe_data['videoSrc'], page_id, 'mp4', m3u8_id='hls')
+
+        return {
+            **traverse_obj(recipe_data, {
+                'id': ('id', {str_or_none}),
+                'title': ('title', {str}),
+                'description': ('topnote', {clean_html}),
+                'timestamp': ('publishedAt', {int_or_none}),
+                'creator': ('contentAttribution', 'cardByline', {str}),
+            }),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnails': [{'url': thumb_url} for thumb_url in traverse_obj(
+                recipe_data, ('image', 'crops', 'recipe', ..., {url_or_none}))],
+        }

From acaf806c15f0a802ba286c23af02a10cf4bd4731 Mon Sep 17 00:00:00 2001
From: DmitryScaletta <DmitryScaletta@users.noreply.github.com>
Date: Mon, 5 Feb 2024 05:17:39 +0300
Subject: [PATCH 846/871] [ie/nuum] Add extractors (#8868)

Authored by: DmitryScaletta, seproDev

Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 yt_dlp/extractor/_extractors.py |  10 +-
 yt_dlp/extractor/nuum.py        | 199 ++++++++++++++++++++++++++++++++
 yt_dlp/extractor/wasdtv.py      | 159 -------------------------
 3 files changed, 204 insertions(+), 164 deletions(-)
 create mode 100644 yt_dlp/extractor/nuum.py
 delete mode 100644 yt_dlp/extractor/wasdtv.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 36335286c..e7dd34c77 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1354,6 +1354,11 @@
     NYTimesCookingIE,
     NYTimesCookingRecipeIE,
 )
+from .nuum import (
+    NuumLiveIE,
+    NuumTabIE,
+    NuumMediaIE,
+)
 from .nuvid import NuvidIE
 from .nzherald import NZHeraldIE
 from .nzonscreen import NZOnScreenIE
@@ -2315,11 +2320,6 @@
     WashingtonPostIE,
     WashingtonPostArticleIE,
 )
-from .wasdtv import (
-    WASDTVStreamIE,
-    WASDTVRecordIE,
-    WASDTVClipIE,
-)
 from .wat import WatIE
 from .wdr import (
     WDRIE,
diff --git a/yt_dlp/extractor/nuum.py b/yt_dlp/extractor/nuum.py
new file mode 100644
index 000000000..3db663ded
--- /dev/null
+++ b/yt_dlp/extractor/nuum.py
@@ -0,0 +1,199 @@
+import functools
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    OnDemandPagedList,
+    UserNotLive,
+    filter_dict,
+    int_or_none,
+    parse_iso8601,
+    str_or_none,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class NuumBaseIE(InfoExtractor):
+    def _call_api(self, path, video_id, description, query={}):
+        response = self._download_json(
+            f'https://nuum.ru/api/v2/{path}', video_id, query=query,
+            note=f'Downloading {description} metadata',
+            errnote=f'Unable to download {description} metadata')
+        if error := response.get('error'):
+            raise ExtractorError(f'API returned error: {error!r}')
+        return response['result']
+
+    def _get_channel_info(self, channel_name):
+        return self._call_api(
+            'broadcasts/public', video_id=channel_name, description='channel',
+            query={
+                'with_extra': 'true',
+                'channel_name': channel_name,
+                'with_deleted': 'true',
+            })
+
+    def _parse_video_data(self, container, extract_formats=True):
+        stream = traverse_obj(container, ('media_container_streams', 0, {dict})) or {}
+        media = traverse_obj(stream, ('stream_media', 0, {dict})) or {}
+        media_url = traverse_obj(media, (
+            'media_meta', ('media_archive_url', 'media_url'), {url_or_none}), get_all=False)
+
+        video_id = str(container['media_container_id'])
+        is_live = media.get('media_status') == 'RUNNING'
+
+        formats, subtitles = None, None
+        if extract_formats:
+            formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+                media_url, video_id, 'mp4', live=is_live)
+
+        return filter_dict({
+            'id': video_id,
+            'is_live': is_live,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(container, {
+                'title': ('media_container_name', {str}),
+                'description': ('media_container_description', {str}),
+                'timestamp': ('created_at', {parse_iso8601}),
+                'channel': ('media_container_channel', 'channel_name', {str}),
+                'channel_id': ('media_container_channel', 'channel_id', {str_or_none}),
+            }),
+            **traverse_obj(stream, {
+                'view_count': ('stream_total_viewers', {int_or_none}),
+                'concurrent_view_count': ('stream_current_viewers', {int_or_none}),
+            }),
+            **traverse_obj(media, {
+                'duration': ('media_duration', {int_or_none}),
+                'thumbnail': ('media_meta', ('media_preview_archive_url', 'media_preview_url'), {url_or_none}),
+            }, get_all=False),
+        })
+
+
+class NuumMediaIE(NuumBaseIE):
+    IE_NAME = 'nuum:media'
+    _VALID_URL = r'https?://nuum\.ru/(?:streams|videos|clips)/(?P<id>[\d]+)'
+    _TESTS = [{
+        'url': 'https://nuum.ru/streams/1592713-7-days-to-die',
+        'only_matching': True,
+    }, {
+        'url': 'https://nuum.ru/videos/1567547-toxi-hurtz',
+        'md5': 'f1d9118a30403e32b702a204eb03aca3',
+        'info_dict': {
+            'id': '1567547',
+            'ext': 'mp4',
+            'title': 'Toxi$ - Hurtz',
+            'description': '',
+            'timestamp': 1702631651,
+            'upload_date': '20231215',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'view_count': int,
+            'concurrent_view_count': int,
+            'channel_id': '6911',
+            'channel': 'toxis',
+            'duration': 116,
+        },
+    }, {
+        'url': 'https://nuum.ru/clips/1552564-pro-misu',
+        'md5': 'b248ae1565b1e55433188f11beeb0ca1',
+        'info_dict': {
+            'id': '1552564',
+            'ext': 'mp4',
+            'title': 'Про Мису 🙃',
+            'timestamp': 1701971828,
+            'upload_date': '20231207',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'view_count': int,
+            'concurrent_view_count': int,
+            'channel_id': '3320',
+            'channel': 'Misalelik',
+            'duration': 41,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._call_api(f'media-containers/{video_id}', video_id, 'media')
+
+        return self._parse_video_data(video_data)
+
+
+class NuumLiveIE(NuumBaseIE):
+    IE_NAME = 'nuum:live'
+    _VALID_URL = r'https?://nuum\.ru/channel/(?P<id>[^/#?]+)/?(?:$|[#?])'
+    _TESTS = [{
+        'url': 'https://nuum.ru/channel/mts_live',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        channel = self._match_id(url)
+        channel_info = self._get_channel_info(channel)
+        if traverse_obj(channel_info, ('channel', 'channel_is_live')) is False:
+            raise UserNotLive(video_id=channel)
+
+        info = self._parse_video_data(channel_info['media_container'])
+        return {
+            'webpage_url': f'https://nuum.ru/streams/{info["id"]}',
+            'extractor_key': NuumMediaIE.ie_key(),
+            'extractor': NuumMediaIE.IE_NAME,
+            **info,
+        }
+
+
+class NuumTabIE(NuumBaseIE):
+    IE_NAME = 'nuum:tab'
+    _VALID_URL = r'https?://nuum\.ru/channel/(?P<id>[^/#?]+)/(?P<type>streams|videos|clips)'
+    _TESTS = [{
+        'url': 'https://nuum.ru/channel/dankon_/clips',
+        'info_dict': {
+            'id': 'dankon__clips',
+            'title': 'Dankon_',
+        },
+        'playlist_mincount': 29,
+    }, {
+        'url': 'https://nuum.ru/channel/dankon_/videos',
+        'info_dict': {
+            'id': 'dankon__videos',
+            'title': 'Dankon_',
+        },
+        'playlist_mincount': 2,
+    }, {
+        'url': 'https://nuum.ru/channel/dankon_/streams',
+        'info_dict': {
+            'id': 'dankon__streams',
+            'title': 'Dankon_',
+        },
+        'playlist_mincount': 1,
+    }]
+
+    _PAGE_SIZE = 50
+
+    def _fetch_page(self, channel_id, tab_type, tab_id, page):
+        CONTAINER_TYPES = {
+            'clips': ['SHORT_VIDEO', 'REVIEW_VIDEO'],
+            'videos': ['LONG_VIDEO'],
+            'streams': ['SINGLE'],
+        }
+
+        media_containers = self._call_api(
+            'media-containers', video_id=tab_id, description=f'{tab_type} tab page {page + 1}',
+            query={
+                'limit': self._PAGE_SIZE,
+                'offset': page * self._PAGE_SIZE,
+                'channel_id': channel_id,
+                'media_container_status': 'STOPPED',
+                'media_container_type': CONTAINER_TYPES[tab_type],
+            })
+        for container in traverse_obj(media_containers, (..., {dict})):
+            metadata = self._parse_video_data(container, extract_formats=False)
+            yield self.url_result(f'https://nuum.ru/videos/{metadata["id"]}', NuumMediaIE, **metadata)
+
+    def _real_extract(self, url):
+        channel_name, tab_type = self._match_valid_url(url).group('id', 'type')
+        tab_id = f'{channel_name}_{tab_type}'
+        channel_data = self._get_channel_info(channel_name)['channel']
+
+        return self.playlist_result(OnDemandPagedList(functools.partial(
+            self._fetch_page, channel_data['channel_id'], tab_type, tab_id), self._PAGE_SIZE),
+            playlist_id=tab_id, playlist_title=channel_data.get('channel_name'))
diff --git a/yt_dlp/extractor/wasdtv.py b/yt_dlp/extractor/wasdtv.py
deleted file mode 100644
index f57c619b5..000000000
--- a/yt_dlp/extractor/wasdtv.py
+++ /dev/null
@@ -1,159 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    parse_iso8601,
-    traverse_obj,
-    try_get,
-)
-
-
-class WASDTVBaseIE(InfoExtractor):
-
-    def _fetch(self, path, video_id, description, query={}):
-        response = self._download_json(
-            f'https://wasd.tv/api/{path}', video_id, query=query,
-            note=f'Downloading {description} metadata',
-            errnote=f'Unable to download {description} metadata')
-        error = response.get('error')
-        if error:
-            raise ExtractorError(f'{self.IE_NAME} returned error: {error}', expected=True)
-        return response.get('result')
-
-    def _extract_thumbnails(self, thumbnails_dict):
-        return [{
-            'url': url,
-            'preference': index,
-        } for index, url in enumerate(
-            traverse_obj(thumbnails_dict, (('small', 'medium', 'large'),))) if url]
-
-    def _real_extract(self, url):
-        container = self._get_container(url)
-        stream = traverse_obj(container, ('media_container_streams', 0))
-        media = try_get(stream, lambda x: x['stream_media'][0])
-        if not media:
-            raise ExtractorError('Can not extract media data.', expected=True)
-        media_meta = media.get('media_meta')
-        media_url, is_live = self._get_media_url(media_meta)
-        video_id = media.get('media_id') or container.get('media_container_id')
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(media_url, video_id, 'mp4')
-        return {
-            'id': str(video_id),
-            'title': container.get('media_container_name') or self._og_search_title(self._download_webpage(url, video_id)),
-            'description': container.get('media_container_description'),
-            'thumbnails': self._extract_thumbnails(media_meta.get('media_preview_images')),
-            'timestamp': parse_iso8601(container.get('created_at')),
-            'view_count': int_or_none(stream.get('stream_current_viewers' if is_live else 'stream_total_viewers')),
-            'is_live': is_live,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
-
-    def _get_container(self, url):
-        raise NotImplementedError('Subclass for get media container')
-
-    def _get_media_url(self, media_meta):
-        raise NotImplementedError('Subclass for get media url')
-
-
-class WASDTVStreamIE(WASDTVBaseIE):
-    IE_NAME = 'wasdtv:stream'
-    _VALID_URL = r'https?://wasd\.tv/(?P<id>[^/#?]+)$'
-    _TESTS = [{
-        'url': 'https://wasd.tv/24_7',
-        'info_dict': {
-            'id': '559738',
-            'ext': 'mp4',
-            'title': 'Live 24/7 Music',
-            'description': '24&#x2F;7 Music',
-            'timestamp': int,
-            'upload_date': r're:^\d{8}$',
-            'is_live': True,
-            'view_count': int,
-        },
-    }]
-
-    def _get_container(self, url):
-        nickname = self._match_id(url)
-        channel = self._fetch(f'channels/nicknames/{nickname}', video_id=nickname, description='channel')
-        channel_id = channel.get('channel_id')
-        containers = self._fetch(
-            'v2/media-containers', channel_id, 'running media containers',
-            query={
-                'channel_id': channel_id,
-                'media_container_type': 'SINGLE',
-                'media_container_status': 'RUNNING',
-            })
-        if not containers:
-            raise ExtractorError(f'{nickname} is offline', expected=True)
-        return containers[0]
-
-    def _get_media_url(self, media_meta):
-        return media_meta['media_url'], True
-
-
-class WASDTVRecordIE(WASDTVBaseIE):
-    IE_NAME = 'wasdtv:record'
-    _VALID_URL = r'https?://wasd\.tv/[^/#?]+(?:/videos)?\?record=(?P<id>\d+)$'
-    _TESTS = [{
-        'url': 'https://wasd.tv/spacemita/videos?record=907755',
-        'md5': 'c9899dd85be4cc997816ff9f9ca516ce',
-        'info_dict': {
-            'id': '906825',
-            'ext': 'mp4',
-            'title': 'Музыкальный',
-            'description': 'md5:f510388d929ff60ae61d4c3cab3137cc',
-            'timestamp': 1645812079,
-            'upload_date': '20220225',
-            'thumbnail': r're:^https?://.+\.jpg',
-            'is_live': False,
-            'view_count': int,
-        },
-    }, {
-        'url': 'https://wasd.tv/spacemita?record=907755',
-        'only_matching': True,
-    }]
-
-    def _get_container(self, url):
-        container_id = self._match_id(url)
-        return self._fetch(
-            f'v2/media-containers/{container_id}', container_id, 'media container')
-
-    def _get_media_url(self, media_meta):
-        media_archive_url = media_meta.get('media_archive_url')
-        if media_archive_url:
-            return media_archive_url, False
-        return media_meta['media_url'], True
-
-
-class WASDTVClipIE(WASDTVBaseIE):
-    IE_NAME = 'wasdtv:clip'
-    _VALID_URL = r'https?://wasd\.tv/[^/#?]+/clips\?clip=(?P<id>\d+)$'
-    _TESTS = [{
-        'url': 'https://wasd.tv/spacemita/clips?clip=26804',
-        'md5': '818885e720143d7a4e776ff66fcff148',
-        'info_dict': {
-            'id': '26804',
-            'ext': 'mp4',
-            'title': 'Пуш флексит на голове стримера',
-            'timestamp': 1646682908,
-            'upload_date': '20220307',
-            'thumbnail': r're:^https?://.+\.jpg',
-            'view_count': int,
-        },
-    }]
-
-    def _real_extract(self, url):
-        clip_id = self._match_id(url)
-        clip = self._fetch(f'v2/clips/{clip_id}', video_id=clip_id, description='clip')
-        clip_data = clip.get('clip_data')
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(clip_data.get('url'), video_id=clip_id, ext='mp4')
-        return {
-            'id': clip_id,
-            'title': clip.get('clip_title') or self._og_search_title(self._download_webpage(url, clip_id, fatal=False)),
-            'thumbnails': self._extract_thumbnails(clip_data.get('preview')),
-            'timestamp': parse_iso8601(clip.get('created_at')),
-            'view_count': int_or_none(clip.get('clip_views_count')),
-            'formats': formats,
-            'subtitles': subtitles,
-        }

From 35d96982f1033e36215d323317981ee17e8ab0d5 Mon Sep 17 00:00:00 2001
From: Chocobozzz <chocobozzz@cpy.re>
Date: Mon, 5 Feb 2024 20:58:32 +0100
Subject: [PATCH 847/871] [ie/peertube] Update instances (#9070)

Authored by: Chocobozzz
---
 yt_dlp/extractor/peertube.py | 972 ++++++++++++++++++++++-------------
 1 file changed, 610 insertions(+), 362 deletions(-)

diff --git a/yt_dlp/extractor/peertube.py b/yt_dlp/extractor/peertube.py
index 68e15737b..730b2393e 100644
--- a/yt_dlp/extractor/peertube.py
+++ b/yt_dlp/extractor/peertube.py
@@ -19,636 +19,902 @@
 class PeerTubeIE(InfoExtractor):
     _INSTANCES_RE = r'''(?:
                             # Taken from https://instances.joinpeertube.org/instances
-                            40two\.tube|
-                            a\.metube\.ch|
-                            advtv\.ml|
-                            algorithmic\.tv|
-                            alimulama\.com|
-                            arcana\.fun|
-                            archive\.vidicon\.org|
-                            artefac-paris\.tv|
-                            auf1\.eu|
+                            0ch\.tv|
+                            3dctube\.3dcandy\.social|
+                            all\.electric\.kitchen|
+                            alterscope\.fr|
+                            anarchy\.tube|
+                            apathy\.tv|
+                            apertatube\.net|
+                            archive\.nocopyrightintended\.tv|
+                            archive\.reclaim\.tv|
+                            area51\.media|
+                            astrotube-ufe\.obspm\.fr|
+                            astrotube\.obspm\.fr|
+                            audio\.freediverse\.com|
+                            azxtube\.youssefc\.tn|
+                            bark\.video|
                             battlepenguin\.video|
-                            beertube\.epgn\.ch|
-                            befree\.nohost\.me|
+                            bava\.tv|
+                            bee-tube\.fr|
+                            beetoons\.tv|
+                            biblion\.refchat\.net|
+                            biblioteca\.theowlclub\.net|
                             bideoak\.argia\.eus|
-                            birkeundnymphe\.de|
+                            bideoteka\.eus|
+                            birdtu\.be|
                             bitcointv\.com|
-                            cattube\.org|
-                            clap\.nerv-project\.eu|
-                            climatejustice\.video|
+                            bonn\.video|
+                            breeze\.tube|
+                            brioco\.live|
+                            brocosoup\.fr|
+                            canal\.facil\.services|
+                            canard\.tube|
+                            cdn01\.tilvids\.com|
+                            celluloid-media\.huma-num\.fr|
+                            chicago1\.peertube\.support|
+                            cliptube\.org|
+                            cloudtube\.ise\.fraunhofer\.de|
                             comf\.tube|
+                            comics\.peertube\.biz|
+                            commons\.tube|
+                            communitymedia\.video|
                             conspiracydistillery\.com|
+                            crank\.recoil\.org|
+                            dalek\.zone|
+                            dalliance\.network|
+                            dangly\.parts|
                             darkvapor\.nohost\.me|
                             daschauher\.aksel\.rocks|
                             digitalcourage\.video|
-                            dreiecksnebel\.alex-detsch\.de|
-                            eduvid\.org|
+                            displayeurope\.video|
+                            ds106\.tv|
+                            dud-video\.inf\.tu-dresden\.de|
+                            dud175\.inf\.tu-dresden\.de|
+                            dytube\.com|
+                            ebildungslabor\.video|
                             evangelisch\.video|
-                            exo\.tube|
                             fair\.tube|
+                            fedi\.video|
+                            fedimovie\.com|
                             fediverse\.tv|
                             film\.k-prod\.fr|
-                            flim\.txmn\.tk|
+                            flipboard\.video|
+                            foss\.video|
+                            fossfarmers\.company|
                             fotogramas\.politicaconciencia\.org|
-                            ftsi\.ru|
-                            gary\.vger\.cloud|
-                            graeber\.video|
+                            freediverse\.com|
+                            freesoto-u2151\.vm\.elestio\.app|
+                            freesoto\.tv|
+                            garr\.tv|
                             greatview\.video|
                             grypstube\.uni-greifswald\.de|
-                            highvoltage\.tv|
-                            hpstube\.fr|
-                            htp\.live|
-                            hyperreal\.tube|
+                            habratube\.site|
+                            ilbjach\.ru|
+                            infothema\.net|
+                            itvplus\.iiens\.net|
+                            johnydeep\.net|
                             juggling\.digital|
+                            jupiter\.tube|
+                            kadras\.live|
                             kino\.kompot\.si|
                             kino\.schuerz\.at|
                             kinowolnosc\.pl|
                             kirche\.peertube-host\.de|
+                            kiwi\.froggirl\.club|
                             kodcast\.com|
                             kolektiva\.media|
-                            kraut\.zone|
+                            kpop\.22x22\.ru|
                             kumi\.tube|
+                            la2\.peertube\.support|
+                            la3\.peertube\.support|
+                            la4\.peertube\.support|
                             lastbreach\.tv|
-                            lepetitmayennais\.fr\.nf|
-                            lexx\.impa\.me|
-                            libertynode\.tv|
-                            libra\.syntazia\.org|
-                            libremedia\.video|
+                            lawsplaining\.peertube\.biz|
+                            leopard\.tube|
+                            live\.codinglab\.ch|
                             live\.libratoi\.org|
-                            live\.nanao\.moe|
-                            live\.toobnix\.org|
-                            livegram\.net|
-                            lolitube\.freedomchan\.moe|
+                            live\.oldskool\.fi|
+                            live\.solari\.com|
                             lucarne\.balsamine\.be|
-                            maindreieck-tv\.de|
-                            mani\.tube|
-                            manicphase\.me|
+                            luxtube\.lu|
+                            makertube\.net|
+                            media\.econoalchemist\.com|
+                            media\.exo\.cat|
                             media\.fsfe\.org|
                             media\.gzevd\.de|
-                            media\.inno3\.cricket|
-                            media\.kaitaia\.life|
+                            media\.interior\.edu\.uy|
                             media\.krashboyz\.org|
-                            media\.over-world\.org|
-                            media\.skewed\.de|
+                            media\.mzhd\.de|
+                            media\.smz-ma\.de|
+                            media\.theplattform\.net|
                             media\.undeadnetwork\.de|
+                            medias\.debrouillonet\.org|
                             medias\.pingbase\.net|
+                            mediatube\.fermalo\.fr|
                             melsungen\.peertube-host\.de|
-                            mirametube\.fr|
-                            mojotube\.net|
-                            monplaisirtube\.ddns\.net|
+                            merci-la-police\.fr|
+                            mindlyvideos\.com|
+                            mirror\.peertube\.metalbanana\.net|
+                            mirrored\.rocks|
+                            mix\.video|
                             mountaintown\.video|
-                            my\.bunny\.cafe|
-                            myfreetube\.de|
+                            movies\.metricsmaster\.eu|
+                            mtube\.mooo\.com|
                             mytube\.kn-cloud\.de|
+                            mytube\.le5emeaxe\.fr|
                             mytube\.madzel\.de|
-                            myworkoutarenapeertube\.cf|
+                            nadajemy\.com|
                             nanawel-peertube\.dyndns\.org|
-                            nastub\.cz|
-                            offenes\.tv|
-                            orgdup\.media|
-                            ovaltube\.codinglab\.ch|
+                            neat\.tube|
+                            nethack\.tv|
+                            nicecrew\.tv|
+                            nightshift\.minnix\.dev|
+                            nolog\.media|
+                            nyltube\.nylarea\.com|
+                            ocfedtest\.hosted\.spacebear\.ee|
+                            openmedia\.edunova\.it|
                             p2ptv\.ru|
                             p\.eertu\.be|
                             p\.lu|
+                            pastafriday\.club|
+                            patriottube\.sonsofliberty\.red|
+                            pcbu\.nl|
                             peer\.azurs\.fr|
-                            peertube1\.zeteo\.me|
+                            peer\.d0g4\.me|
+                            peer\.lukeog\.com|
+                            peer\.madiator\.cloud|
+                            peer\.raise-uav\.com|
+                            peershare\.togart\.de|
+                            peertube-blablalinux\.be|
+                            peertube-demo\.learning-hub\.fr|
+                            peertube-docker\.cpy\.re|
+                            peertube-eu\.howlround\.com|
+                            peertube-u5014\.vm\.elestio\.app|
+                            peertube-us\.howlround\.com|
                             peertube\.020\.pl|
                             peertube\.0x5e\.eu|
+                            peertube\.1984\.cz|
+                            peertube\.2i2l\.net|
+                            peertube\.adjutor\.xyz|
+                            peertube\.adresse\.data\.gouv\.fr|
                             peertube\.alpharius\.io|
                             peertube\.am-networks\.fr|
                             peertube\.anduin\.net|
-                            peertube\.anzui\.dev|
-                            peertube\.arbleizez\.bzh|
+                            peertube\.anti-logic\.com|
+                            peertube\.arch-linux\.cz|
                             peertube\.art3mis\.de|
-                            peertube\.atilla\.org|
+                            peertube\.artsrn\.ualberta\.ca|
+                            peertube\.askan\.info|
+                            peertube\.astral0pitek\.synology\.me|
                             peertube\.atsuchan\.page|
-                            peertube\.aukfood\.net|
-                            peertube\.aventer\.biz|
+                            peertube\.automat\.click|
                             peertube\.b38\.rural-it\.org|
-                            peertube\.beeldengeluid\.nl|
                             peertube\.be|
+                            peertube\.beeldengeluid\.nl|
                             peertube\.bgzashtita\.es|
-                            peertube\.bitsandlinux\.com|
+                            peertube\.bike|
+                            peertube\.bildung-ekhn\.de|
                             peertube\.biz|
-                            peertube\.boba\.best|
                             peertube\.br0\.fr|
                             peertube\.bridaahost\.ynh\.fr|
                             peertube\.bubbletea\.dev|
                             peertube\.bubuit\.net|
                             peertube\.cabaal\.net|
-                            peertube\.cats-home\.net|
-                            peertube\.chemnitz\.freifunk\.net|
-                            peertube\.chevro\.fr|
-                            peertube\.chrisspiegl\.com|
+                            peertube\.chatinbit\.com|
+                            peertube\.chaunchy\.com|
+                            peertube\.chir\.rs|
+                            peertube\.christianpacaud\.com|
                             peertube\.chtisurel\.net|
+                            peertube\.chuggybumba\.com|
                             peertube\.cipherbliss\.com|
+                            peertube\.cirkau\.art|
+                            peertube\.cloud\.nerdraum\.de|
                             peertube\.cloud\.sans\.pub|
+                            peertube\.coko\.foundation|
+                            peertube\.communecter\.org|
+                            peertube\.concordia\.social|
+                            peertube\.corrigan\.xyz|
                             peertube\.cpge-brizeux\.fr|
                             peertube\.ctseuro\.com|
                             peertube\.cuatrolibertades\.org|
-                            peertube\.cybercirujas\.club|
-                            peertube\.cythin\.com|
+                            peertube\.cube4fun\.net|
+                            peertube\.dair-institute\.org|
                             peertube\.davigge\.com|
                             peertube\.dc\.pini\.fr|
+                            peertube\.deadtom\.me|
                             peertube\.debian\.social|
+                            peertube\.delta0189\.xyz|
                             peertube\.demonix\.fr|
                             peertube\.designersethiques\.org|
                             peertube\.desmu\.fr|
-                            peertube\.devloprog\.org|
                             peertube\.devol\.it|
-                            peertube\.dtmf\.ca|
-                            peertube\.ecologie\.bzh|
+                            peertube\.dk|
+                            peertube\.doesstuff\.social|
+                            peertube\.eb8\.org|
+                            peertube\.education-forum\.com|
+                            peertube\.elforcer\.ru|
+                            peertube\.em\.id\.lv|
+                            peertube\.ethibox\.fr|
                             peertube\.eu\.org|
                             peertube\.european-pirates\.eu|
+                            peertube\.eus|
                             peertube\.euskarabildua\.eus|
+                            peertube\.expi\.studio|
+                            peertube\.familie-berner\.de|
+                            peertube\.familleboisteau\.fr|
+                            peertube\.fedihost\.website|
                             peertube\.fenarinarsa\.com|
-                            peertube\.fomin\.site|
-                            peertube\.forsud\.be|
-                            peertube\.francoispelletier\.org|
-                            peertube\.freenet\.ru|
-                            peertube\.freetalklive\.com|
+                            peertube\.festnoz\.de|
+                            peertube\.forteza\.fr|
+                            peertube\.freestorm\.online|
                             peertube\.functional\.cafe|
-                            peertube\.gardeludwig\.fr|
+                            peertube\.gaminglinux\.fr|
                             peertube\.gargantia\.fr|
-                            peertube\.gcfamily\.fr|
+                            peertube\.geekgalaxy\.fr|
+                            peertube\.gemlog\.ca|
                             peertube\.genma\.fr|
                             peertube\.get-racing\.de|
+                            peertube\.ghis94\.ovh|
                             peertube\.gidikroon\.eu|
-                            peertube\.gruezishop\.ch|
-                            peertube\.habets\.house|
-                            peertube\.hackerfraternity\.org|
+                            peertube\.giftedmc\.com|
+                            peertube\.grosist\.fr|
+                            peertube\.gruntwerk\.org|
+                            peertube\.gsugambit\.com|
+                            peertube\.hackerfoo\.com|
+                            peertube\.hellsite\.net|
+                            peertube\.helvetet\.eu|
+                            peertube\.histoirescrepues\.fr|
+                            peertube\.home\.x0r\.fr|
+                            peertube\.hyperfreedom\.org|
                             peertube\.ichigo\.everydayimshuflin\.com|
-                            peertube\.ignifi\.me|
+                            peertube\.ifwo\.eu|
+                            peertube\.in\.ua|
                             peertube\.inapurna\.org|
                             peertube\.informaction\.info|
                             peertube\.interhop\.org|
-                            peertube\.iselfhost\.com|
                             peertube\.it|
+                            peertube\.it-arts\.net|
                             peertube\.jensdiemer\.de|
-                            peertube\.joffreyverd\.fr|
+                            peertube\.johntheserg\.al|
+                            peertube\.kaleidos\.net|
                             peertube\.kalua\.im|
-                            peertube\.kathryl\.fr|
+                            peertube\.kcore\.org|
                             peertube\.keazilla\.net|
                             peertube\.klaewyss\.fr|
-                            peertube\.kodcast\.com|
+                            peertube\.kleph\.eu|
+                            peertube\.kodein\.be|
+                            peertube\.kooperatywa\.tech|
+                            peertube\.kriom\.net|
                             peertube\.kx\.studio|
+                            peertube\.kyriog\.eu|
+                            peertube\.la-famille-muller\.fr|
+                            peertube\.labeuropereunion\.eu|
                             peertube\.lagvoid\.com|
-                            peertube\.lavallee\.tech|
-                            peertube\.le5emeaxe\.fr|
-                            peertube\.lestutosdeprocessus\.fr|
-                            peertube\.librenet\.co\.za|
+                            peertube\.lhc\.net\.br|
+                            peertube\.libresolutions\.network|
+                            peertube\.libretic\.fr|
+                            peertube\.librosphere\.fr|
                             peertube\.logilab\.fr|
+                            peertube\.lon\.tv|
                             peertube\.louisematic\.site|
                             peertube\.luckow\.org|
                             peertube\.luga\.at|
                             peertube\.lyceeconnecte\.fr|
-                            peertube\.manalejandro\.com|
+                            peertube\.madixam\.xyz|
+                            peertube\.magicstone\.dev|
+                            peertube\.marienschule\.de|
                             peertube\.marud\.fr|
-                            peertube\.mattone\.net|
                             peertube\.maxweiss\.io|
+                            peertube\.miguelcr\.me|
+                            peertube\.mikemestnik\.net|
+                            peertube\.mobilsicher\.de|
                             peertube\.monlycee\.net|
                             peertube\.mxinfo\.fr|
-                            peertube\.myrasp\.eu|
-                            peertube\.nebelcloud\.de|
+                            peertube\.naln1\.ca|
                             peertube\.netzbegruenung\.de|
-                            peertube\.newsocial\.tech|
                             peertube\.nicolastissot\.fr|
+                            peertube\.nogafam\.fr|
+                            peertube\.normalgamingcommunity\.cz|
                             peertube\.nz|
                             peertube\.offerman\.com|
+                            peertube\.ohioskates\.com|
+                            peertube\.onionstorm\.net|
                             peertube\.opencloud\.lu|
-                            peertube\.orthus\.link|
-                            peertube\.patapouf\.xyz|
-                            peertube\.pi2\.dev|
-                            peertube\.plataformess\.org|
-                            peertube\.pl|
-                            peertube\.portaesgnos\.org|
+                            peertube\.otakufarms\.com|
+                            peertube\.paladyn\.org|
+                            peertube\.pix-n-chill\.fr|
                             peertube\.r2\.enst\.fr|
                             peertube\.r5c3\.fr|
-                            peertube\.radres\.xyz|
-                            peertube\.red|
-                            peertube\.robonomics\.network|
-                            peertube\.rtnkv\.cloud|
-                            peertube\.runfox\.tk|
+                            peertube\.redpill-insight\.com|
+                            peertube\.researchinstitute\.at|
+                            peertube\.revelin\.fr|
+                            peertube\.rlp\.schule|
+                            peertube\.rokugan\.fr|
+                            peertube\.rougevertbleu\.tv|
+                            peertube\.roundpond\.net|
+                            peertube\.rural-it\.org|
                             peertube\.satoshishop\.de|
-                            peertube\.scic-tetris\.org|
+                            peertube\.scyldings\.com|
                             peertube\.securitymadein\.lu|
+                            peertube\.semperpax\.com|
                             peertube\.semweb\.pro|
-                            peertube\.social\.my-wan\.de|
-                            peertube\.soykaf\.org|
-                            peertube\.stefofficiel\.me|
+                            peertube\.sensin\.eu|
+                            peertube\.sidh\.bzh|
+                            peertube\.skorpil\.cz|
+                            peertube\.smertrios\.com|
+                            peertube\.sqweeb\.net|
+                            peertube\.stattzeitung\.org|
                             peertube\.stream|
                             peertube\.su|
                             peertube\.swrs\.net|
                             peertube\.takeko\.cyou|
-                            peertube\.tangentfox\.com|
                             peertube\.taxinachtegel\.de|
-                            peertube\.thenewoil\.xyz|
+                            peertube\.teftera\.com|
+                            peertube\.teutronic-services\.de|
                             peertube\.ti-fr\.com|
                             peertube\.tiennot\.net|
-                            peertube\.troback\.com|
+                            peertube\.tmp\.rcp\.tf|
                             peertube\.tspu\.edu\.ru|
-                            peertube\.tux\.ovh|
                             peertube\.tv|
                             peertube\.tweb\.tv|
-                            peertube\.ucy\.de|
                             peertube\.underworld\.fr|
-                            peertube\.us\.to|
-                            peertube\.ventresmous\.fr|
+                            peertube\.vapronva\.pw|
+                            peertube\.veen\.world|
+                            peertube\.vesdia\.eu|
+                            peertube\.virtual-assembly\.org|
+                            peertube\.viviers-fibre\.net|
                             peertube\.vlaki\.cz|
-                            peertube\.w\.utnw\.de|
-                            peertube\.westring\.digital|
+                            peertube\.wiesbaden\.social|
+                            peertube\.wivodaim\.net|
+                            peertube\.wtf|
+                            peertube\.wtfayla\.net|
+                            peertube\.xrcb\.cat|
                             peertube\.xwiki\.com|
+                            peertube\.zd\.do|
+                            peertube\.zetamc\.net|
+                            peertube\.zmuuf\.org|
                             peertube\.zoz-serv\.org|
+                            peertube\.zwindler\.fr|
                             peervideo\.ru|
                             periscope\.numenaute\.org|
-                            perron-tube\.de|
+                            pete\.warpnine\.de|
                             petitlutinartube\.fr|
                             phijkchu\.com|
-                            pierre\.tube|
+                            phoenixproject\.group|
                             piraten\.space|
-                            play\.rosano\.ca|
+                            pirtube\.calut\.fr|
+                            pityu\.flaki\.hu|
+                            play\.mittdata\.se|
                             player\.ojamajo\.moe|
-                            plextube\.nl|
-                            pocketnetpeertube1\.nohost\.me|
-                            pocketnetpeertube3\.nohost\.me|
-                            pocketnetpeertube4\.nohost\.me|
-                            pocketnetpeertube5\.nohost\.me|
-                            pocketnetpeertube6\.nohost\.me|
-                            pt\.24-7\.ro|
-                            pt\.apathy\.top|
+                            podlibre\.video|
+                            portal\.digilab\.nfa\.cz|
+                            private\.fedimovie\.com|
+                            pt01\.lehrerfortbildung-bw\.de|
                             pt\.diaspodon\.fr|
-                            pt\.fedi\.tech|
-                            pt\.maciej\.website|
+                            pt\.freedomwolf\.cc|
+                            pt\.gordons\.gen\.nz|
+                            pt\.ilyamikcoder\.com|
+                            pt\.irnok\.net|
+                            pt\.mezzo\.moe|
+                            pt\.na4\.eu|
+                            pt\.netcraft\.ch|
+                            pt\.rwx\.ch|
+                            pt\.sfunk1x\.com|
+                            pt\.thishorsie\.rocks|
+                            pt\.vern\.cc|
                             ptb\.lunarviews\.net|
-                            ptmir1\.inter21\.net|
-                            ptmir2\.inter21\.net|
-                            ptmir3\.inter21\.net|
-                            ptmir4\.inter21\.net|
-                            ptmir5\.inter21\.net|
-                            ptube\.horsentiers\.fr|
-                            ptube\.xmanifesto\.club|
-                            queermotion\.org|
-                            re-wizja\.re-medium\.com|
-                            regarder\.sans\.pub|
-                            ruraletv\.ovh|
-                            s1\.gegenstimme\.tv|
-                            s2\.veezee\.tube|
+                            ptube\.de|
+                            ptube\.ranranhome\.info|
+                            puffy\.tube|
+                            puppet\.zone|
+                            qtube\.qlyoung\.net|
+                            quantube\.win|
+                            rankett\.net|
+                            replay\.jres\.org|
+                            review\.peertube\.biz|
                             sdmtube\.fr|
-                            sender-fm\.veezee\.tube|
-                            serv1\.wiki-tube\.de|
+                            secure\.direct-live\.net|
+                            secure\.scanovid\.com|
+                            seka\.pona\.la|
                             serv3\.wiki-tube\.de|
-                            sickstream\.net|
-                            sleepy\.tube|
+                            skeptube\.fr|
+                            social\.fedimovie\.com|
+                            socpeertube\.ru|
                             sovran\.video|
+                            special\.videovortex\.tv|
                             spectra\.video|
+                            stl1988\.peertube-host\.de|
+                            stream\.biovisata\.lt|
+                            stream\.conesphere\.cloud|
                             stream\.elven\.pw|
+                            stream\.jurnalfm\.md|
                             stream\.k-prod\.fr|
-                            stream\.shahab\.nohost\.me|
-                            streamsource\.video|
+                            stream\.litera\.tools|
+                            stream\.nuemedia\.se|
+                            stream\.rlp-media\.de|
+                            stream\.vrse\.be|
                             studios\.racer159\.com|
-                            testtube\.florimond\.eu|
+                            styxhexenhammer666\.com|
+                            syrteplay\.obspm\.fr|
+                            t\.0x0\.st|
+                            tbh\.co-shaoghal\.net|
+                            test-fab\.ynh\.fr|
+                            testube\.distrilab\.fr|
                             tgi\.hosted\.spacebear\.ee|
-                            thaitube\.in\.th|
-                            the\.jokertv\.eu|
                             theater\.ethernia\.net|
                             thecool\.tube|
+                            thevideoverse\.com|
                             tilvids\.com|
-                            toob\.bub\.org|
-                            tpaw\.video|
-                            truetube\.media|
-                            tuba\.lhub\.pl|
-                            tube-aix-marseille\.beta\.education\.fr|
-                            tube-amiens\.beta\.education\.fr|
-                            tube-besancon\.beta\.education\.fr|
-                            tube-bordeaux\.beta\.education\.fr|
-                            tube-clermont-ferrand\.beta\.education\.fr|
-                            tube-corse\.beta\.education\.fr|
-                            tube-creteil\.beta\.education\.fr|
-                            tube-dijon\.beta\.education\.fr|
-                            tube-education\.beta\.education\.fr|
-                            tube-grenoble\.beta\.education\.fr|
-                            tube-lille\.beta\.education\.fr|
-                            tube-limoges\.beta\.education\.fr|
-                            tube-montpellier\.beta\.education\.fr|
-                            tube-nancy\.beta\.education\.fr|
-                            tube-nantes\.beta\.education\.fr|
-                            tube-nice\.beta\.education\.fr|
-                            tube-normandie\.beta\.education\.fr|
-                            tube-orleans-tours\.beta\.education\.fr|
-                            tube-outremer\.beta\.education\.fr|
-                            tube-paris\.beta\.education\.fr|
-                            tube-poitiers\.beta\.education\.fr|
-                            tube-reims\.beta\.education\.fr|
-                            tube-rennes\.beta\.education\.fr|
-                            tube-strasbourg\.beta\.education\.fr|
-                            tube-toulouse\.beta\.education\.fr|
-                            tube-versailles\.beta\.education\.fr|
-                            tube1\.it\.tuwien\.ac\.at|
+                            tinkerbetter\.tube|
+                            tinsley\.video|
+                            trailers\.ddigest\.com|
+                            tube-action-educative\.apps\.education\.fr|
+                            tube-arts-lettres-sciences-humaines\.apps\.education\.fr|
+                            tube-cycle-2\.apps\.education\.fr|
+                            tube-cycle-3\.apps\.education\.fr|
+                            tube-education-physique-et-sportive\.apps\.education\.fr|
+                            tube-enseignement-professionnel\.apps\.education\.fr|
+                            tube-institutionnel\.apps\.education\.fr|
+                            tube-langues-vivantes\.apps\.education\.fr|
+                            tube-maternelle\.apps\.education\.fr|
+                            tube-numerique-educatif\.apps\.education\.fr|
+                            tube-sciences-technologies\.apps\.education\.fr|
+                            tube-test\.apps\.education\.fr|
+                            tube1\.perron-service\.de|
+                            tube\.9minuti\.it|
                             tube\.abolivier\.bzh|
-                            tube\.ac-amiens\.fr|
-                            tube\.aerztefueraufklaerung\.de|
-                            tube\.alexx\.ml|
+                            tube\.alado\.space|
                             tube\.amic37\.fr|
-                            tube\.anufrij\.de|
-                            tube\.apolut\.net|
-                            tube\.arkhalabs\.io|
+                            tube\.area404\.cloud|
                             tube\.arthack\.nz|
-                            tube\.as211696\.net|
-                            tube\.avensio\.de|
+                            tube\.asulia\.fr|
+                            tube\.awkward\.company|
                             tube\.azbyka\.ru|
                             tube\.azkware\.net|
-                            tube\.bachaner\.fr|
-                            tube\.bmesh\.org|
-                            tube\.borked\.host|
+                            tube\.bartrip\.me\.uk|
+                            tube\.belowtoxic\.media|
+                            tube\.bingle\.plus|
+                            tube\.bit-friends\.de|
                             tube\.bstly\.de|
-                            tube\.chaoszone\.tv|
-                            tube\.chatelet\.ovh|
-                            tube\.cloud-libre\.eu|
+                            tube\.chosto\.me|
                             tube\.cms\.garden|
-                            tube\.cowfee\.moe|
-                            tube\.cryptography\.dog|
-                            tube\.darknight-coffee\.org|
-                            tube\.dev\.lhub\.pl|
+                            tube\.communia\.org|
+                            tube\.cyberia\.club|
+                            tube\.cybershock\.life|
+                            tube\.dembased\.xyz|
+                            tube\.dev\.displ\.eu|
+                            tube\.digitalesozialearbeit\.de|
                             tube\.distrilab\.fr|
+                            tube\.doortofreedom\.org|
                             tube\.dsocialize\.net|
+                            tube\.e-jeremy\.com|
                             tube\.ebin\.club|
+                            tube\.elemac\.fr|
+                            tube\.erzbistum-hamburg\.de|
+                            tube\.exozy\.me|
                             tube\.fdn\.fr|
-                            tube\.florimond\.eu|
-                            tube\.foxarmy\.ml|
-                            tube\.foxden\.party|
-                            tube\.frischesicht\.de|
+                            tube\.fedi\.quebec|
+                            tube\.fediverse\.at|
+                            tube\.felinn\.org|
+                            tube\.flokinet\.is|
+                            tube\.foad\.me\.uk|
+                            tube\.freepeople\.fr|
+                            tube\.friloux\.me|
+                            tube\.froth\.zone|
+                            tube\.fulda\.social|
                             tube\.futuretic\.fr|
-                            tube\.gnous\.eu|
+                            tube\.g1zm0\.de|
+                            tube\.g4rf\.net|
+                            tube\.gaiac\.io|
+                            tube\.geekyboo\.net|
+                            tube\.genb\.de|
+                            tube\.ghk-academy\.info|
+                            tube\.gi-it\.de|
                             tube\.grap\.coop|
                             tube\.graz\.social|
                             tube\.grin\.hu|
-                            tube\.hackerscop\.org|
-                            tube\.hordearii\.fr|
+                            tube\.hokai\.lol|
+                            tube\.int5\.net|
+                            tube\.interhacker\.space|
+                            tube\.invisible\.ch|
+                            tube\.io18\.top|
+                            tube\.itsg\.host|
                             tube\.jeena\.net|
-                            tube\.kai-stuht\.com|
+                            tube\.kh-berlin\.de|
                             tube\.kockatoo\.org|
                             tube\.kotur\.org|
+                            tube\.koweb\.fr|
+                            tube\.la-dina\.net|
+                            tube\.lab\.nrw|
                             tube\.lacaveatonton\.ovh|
+                            tube\.laurent-malys\.fr|
+                            tube\.leetdreams\.ch|
                             tube\.linkse\.media|
                             tube\.lokad\.com|
                             tube\.lucie-philou\.com|
-                            tube\.melonbread\.xyz|
-                            tube\.mfraters\.net|
-                            tube\.motuhake\.xyz|
-                            tube\.mrbesen\.de|
-                            tube\.nah\.re|
-                            tube\.nchoco\.net|
+                            tube\.media-techport\.de|
+                            tube\.morozoff\.pro|
+                            tube\.neshweb\.net|
+                            tube\.nestor\.coop|
+                            tube\.network\.europa\.eu|
+                            tube\.nicfab\.eu|
+                            tube\.nieuwwestbrabant\.nl|
+                            tube\.nogafa\.org|
                             tube\.novg\.net|
                             tube\.nox-rhea\.org|
                             tube\.nuagelibre\.fr|
+                            tube\.numerique\.gouv\.fr|
+                            tube\.nuxnik\.com|
                             tube\.nx12\.net|
                             tube\.octaplex\.net|
-                            tube\.odat\.xyz|
                             tube\.oisux\.org|
+                            tube\.okcinfo\.news|
+                            tube\.onlinekirche\.net|
                             tube\.opportunis\.me|
+                            tube\.oraclefilms\.com|
                             tube\.org\.il|
-                            tube\.ortion\.xyz|
-                            tube\.others\.social|
+                            tube\.pacapime\.ovh|
+                            tube\.parinux\.org|
+                            tube\.pastwind\.top|
                             tube\.picasoft\.net|
-                            tube\.plomlompom\.com|
+                            tube\.pilgerweg-21\.de|
                             tube\.pmj\.rocks|
+                            tube\.pol\.social|
+                            tube\.ponsonaille\.fr|
                             tube\.portes-imaginaire\.org|
+                            tube\.public\.apolut\.net|
+                            tube\.pustule\.org|
                             tube\.pyngu\.com|
+                            tube\.querdenken-711\.de|
                             tube\.rebellion\.global|
+                            tube\.reseau-canope\.fr|
                             tube\.rhythms-of-resistance\.org|
-                            tube\.rita\.moe|
+                            tube\.risedsky\.ovh|
+                            tube\.rooty\.fr|
                             tube\.rsi\.cnr\.it|
-                            tube\.s1gm4\.eu|
-                            tube\.saumon\.io|
+                            tube\.ryne\.moe|
                             tube\.schleuss\.online|
                             tube\.schule\.social|
-                            tube\.seditio\.fr|
+                            tube\.sekretaerbaer\.net|
                             tube\.shanti\.cafe|
                             tube\.shela\.nu|
                             tube\.skrep\.in|
+                            tube\.sleeping\.town|
                             tube\.sp-codes\.de|
-                            tube\.sp4ke\.com|
-                            tube\.superseriousbusiness\.org|
+                            tube\.spdns\.org|
+                            tube\.systerserver\.net|
                             tube\.systest\.eu|
                             tube\.tappret\.fr|
-                            tube\.tardis\.world|
-                            tube\.toontoet\.nl|
+                            tube\.techeasy\.org|
+                            tube\.thierrytalbert\.fr|
+                            tube\.tinfoil-hat\.net|
+                            tube\.toldi\.eu|
                             tube\.tpshd\.de|
+                            tube\.trax\.im|
                             tube\.troopers\.agency|
+                            tube\.ttk\.is|
+                            tube\.tuxfriend\.fr|
                             tube\.tylerdavis\.xyz|
+                            tube\.ullihome\.de|
+                            tube\.ulne\.be|
                             tube\.undernet\.uy|
-                            tube\.vigilian-consulting\.nl|
-                            tube\.vraphim\.com|
-                            tube\.wehost\.lgbt|
-                            tube\.wien\.rocks|
+                            tube\.vrpnet\.org|
                             tube\.wolfe\.casa|
                             tube\.xd0\.de|
+                            tube\.xn--baw-joa\.social|
                             tube\.xy-space\.de|
                             tube\.yapbreak\.fr|
                             tubedu\.org|
-                            tubes\.jodh\.us|
-                            tuktube\.com|
-                            turkum\.me|
+                            tubulus\.openlatin\.org|
+                            turtleisland\.video|
                             tututu\.tube|
-                            tuvideo\.encanarias\.info|
-                            tv1\.cocu\.cc|
-                            tv1\.gomntu\.space|
-                            tv2\.cocu\.cc|
+                            tv\.adast\.dk|
                             tv\.adn\.life|
+                            tv\.arns\.lt|
                             tv\.atmx\.ca|
-                            tv\.bitma\.st|
-                            tv\.generallyrubbish\.net\.au|
+                            tv\.based\.quest|
+                            tv\.farewellutopia\.com|
+                            tv\.filmfreedom\.net|
+                            tv\.gravitons\.org|
+                            tv\.io\.seg\.br|
                             tv\.lumbung\.space|
-                            tv\.mattchristiansenmedia\.com|
-                            tv\.netwhood\.online|
-                            tv\.neue\.city|
-                            tv\.piejacker\.net|
                             tv\.pirateradio\.social|
+                            tv\.pirati\.cz|
+                            tv\.santic-zombie\.ru|
                             tv\.undersco\.re|
+                            tv\.zonepl\.net|
                             tvox\.ru|
                             twctube\.twc-zone\.eu|
-                            unfilter\.tube|
+                            twobeek\.com|
+                            urbanists\.video|
+                            v\.9tail\.net|
                             v\.basspistol\.org|
+                            v\.j4\.lc|
                             v\.kisombrella\.top|
-                            v\.lastorder\.xyz|
+                            v\.koa\.im|
+                            v\.kyaru\.xyz|
                             v\.lor\.sh|
-                            v\.phreedom\.club|
-                            v\.sil\.sh|
-                            v\.szy\.io|
-                            v\.xxxapex\.com|
-                            veezee\.tube|
-                            vid\.dascoyote\.xyz|
-                            vid\.garwood\.io|
-                            vid\.ncrypt\.at|
-                            vid\.pravdastalina\.info|
-                            vid\.qorg11\.net|
-                            vid\.rajeshtaylor\.com|
-                            vid\.samtripoli\.com|
-                            vid\.werefox\.dev|
+                            v\.mkp\.ca|
+                            v\.posm\.gay|
+                            v\.slaycer\.top|
+                            veedeo\.org|
+                            vhs\.absturztau\.be|
+                            vid\.cthos\.dev|
+                            vid\.kinuseka\.us|
+                            vid\.mkp\.ca|
+                            vid\.nocogabriel\.fr|
+                            vid\.norbipeti\.eu|
+                            vid\.northbound\.online|
+                            vid\.ohboii\.de|
+                            vid\.plantplotting\.co\.uk|
+                            vid\.pretok\.tv|
+                            vid\.prometheus\.systems|
+                            vid\.soafen\.love|
+                            vid\.twhtv\.club|
                             vid\.wildeboer\.net|
                             video-cave-v2\.de|
+                            video-liberty\.com|
                             video\.076\.ne\.jp|
                             video\.1146\.nohost\.me|
-                            video\.altertek\.org|
+                            video\.9wd\.eu|
+                            video\.abraum\.de|
+                            video\.ados\.accoord\.fr|
+                            video\.amiga-ng\.org|
                             video\.anartist\.org|
-                            video\.apps\.thedoodleproject\.net|
-                            video\.artist\.cx|
                             video\.asgardius\.company|
-                            video\.balsillie\.net|
+                            video\.audiovisuel-participatif\.org|
                             video\.bards\.online|
-                            video\.binarydad\.com|
+                            video\.barkoczy\.social|
+                            video\.benetou\.fr|
+                            video\.beyondwatts\.social|
+                            video\.bgeneric\.net|
+                            video\.bilecik\.edu\.tr|
                             video\.blast-info\.fr|
+                            video\.bmu\.cloud|
                             video\.catgirl\.biz|
+                            video\.causa-arcana\.com|
+                            video\.chasmcity\.net|
+                            video\.chbmeyer\.de|
                             video\.cigliola\.com|
-                            video\.cm-en-transition\.fr|
+                            video\.citizen4\.eu|
+                            video\.clumsy\.computer|
+                            video\.cnnumerique\.fr|
+                            video\.cnr\.it|
                             video\.cnt\.social|
                             video\.coales\.co|
-                            video\.codingfield\.com|
-                            video\.comptoir\.net|
                             video\.comune\.trento\.it|
-                            video\.cpn\.so|
+                            video\.coyp\.us|
                             video\.csc49\.fr|
-                            video\.cybre\.town|
-                            video\.demokratischer-sommer\.de|
-                            video\.discord-insoumis\.fr|
-                            video\.dolphincastle\.com|
+                            video\.davduf\.net|
+                            video\.davejansen\.com|
+                            video\.dlearning\.nl|
+                            video\.dnfi\.no|
                             video\.dresden\.network|
-                            video\.ecole-89\.com|
-                            video\.elgrillolibertario\.org|
+                            video\.drgnz\.club|
+                            video\.dudenas\.lt|
+                            video\.eientei\.org|
+                            video\.ellijaymakerspace\.org|
                             video\.emergeheart\.info|
                             video\.eradicatinglove\.xyz|
-                            video\.ethantheenigma\.me|
-                            video\.exodus-privacy\.eu\.org|
-                            video\.fbxl\.net|
+                            video\.everythingbagel\.me|
+                            video\.extremelycorporate\.ca|
+                            video\.fabiomanganiello\.com|
+                            video\.fedi\.bzh|
                             video\.fhtagn\.org|
-                            video\.greenmycity\.eu|
-                            video\.guerredeclasse\.fr|
+                            video\.firehawk-systems\.com|
+                            video\.fox-romka\.ru|
+                            video\.fuss\.bz\.it|
+                            video\.glassbeadcollective\.org|
+                            video\.graine-pdl\.org|
                             video\.gyt\.is|
-                            video\.hackers\.town|
+                            video\.hainry\.fr|
                             video\.hardlimit\.com|
-                            video\.hooli\.co|
+                            video\.hostux\.net|
                             video\.igem\.org|
+                            video\.infojournal\.fr|
                             video\.internet-czas-dzialac\.pl|
+                            video\.interru\.io|
+                            video\.ipng\.ch|
+                            video\.ironsysadmin\.com|
                             video\.islameye\.com|
-                            video\.kicik\.fr|
+                            video\.jacen\.moe|
+                            video\.jadin\.me|
+                            video\.jeffmcbride\.net|
+                            video\.jigmedatse\.com|
                             video\.kuba-orlik\.name|
-                            video\.kyushojitsu\.ca|
+                            video\.lacalligramme\.fr|
+                            video\.lanceurs-alerte\.fr|
+                            video\.laotra\.red|
+                            video\.lapineige\.fr|
+                            video\.laraffinerie\.re|
                             video\.lavolte\.net|
-                            video\.lespoesiesdheloise\.fr|
                             video\.liberta\.vip|
-                            video\.liege\.bike|
+                            video\.libreti\.net|
+                            video\.licentia\.net|
                             video\.linc\.systems|
                             video\.linux\.it|
                             video\.linuxtrent\.it|
-                            video\.lokal\.social|
+                            video\.liveitlive\.show|
                             video\.lono\.space|
-                            video\.lunasqu\.ee|
+                            video\.lrose\.de|
+                            video\.lunago\.net|
                             video\.lundi\.am|
+                            video\.lycee-experimental\.org|
+                            video\.maechler\.cloud|
                             video\.marcorennmaus\.de|
                             video\.mass-trespass\.uk|
+                            video\.matomocamp\.org|
+                            video\.medienzentrum-harburg\.de|
+                            video\.mentality\.rip|
+                            video\.metaversum\.wtf|
+                            video\.midreality\.com|
+                            video\.mttv\.it|
                             video\.mugoreve\.fr|
-                            video\.mundodesconocido\.com|
+                            video\.mxtthxw\.art|
                             video\.mycrowd\.ca|
+                            video\.niboe\.info|
                             video\.nogafam\.es|
-                            video\.odayacres\.farm|
+                            video\.nstr\.no|
+                            video\.occm\.cc|
+                            video\.off-investigation\.fr|
+                            video\.olos311\.org|
+                            video\.ordinobsolete\.fr|
+                            video\.osvoj\.ru|
+                            video\.ourcommon\.cloud|
                             video\.ozgurkon\.org|
-                            video\.p1ng0ut\.social|
-                            video\.p3x\.de|
                             video\.pcf\.fr|
-                            video\.pony\.gallery|
-                            video\.potate\.space|
-                            video\.pourpenser\.pro|
-                            video\.progressiv\.dev|
+                            video\.pcgaldo\.com|
+                            video\.phyrone\.de|
+                            video\.poul\.org|
+                            video\.publicspaces\.net|
+                            video\.pullopen\.xyz|
+                            video\.r3s\.nrw|
+                            video\.rainevixen\.com|
                             video\.resolutions\.it|
-                            video\.rw501\.de|
-                            video\.screamer\.wiki|
-                            video\.sdm-tools\.net|
+                            video\.retroedge\.tech|
+                            video\.rhizome\.org|
+                            video\.rlp-media\.de|
+                            video\.rs-einrich\.de|
+                            video\.rubdos\.be|
+                            video\.sadmin\.io|
                             video\.sftblw\.moe|
                             video\.shitposter\.club|
-                            video\.skyn3t\.in|
+                            video\.simplex-software\.ru|
+                            video\.slipfox\.xyz|
+                            video\.snug\.moe|
+                            video\.software-fuer-engagierte\.de|
                             video\.soi\.ch|
-                            video\.stuartbrand\.co\.uk|
+                            video\.sonet\.ws|
+                            video\.surazal\.net|
+                            video\.taskcards\.eu|
+                            video\.team-lcbs\.eu|
+                            video\.techforgood\.social|
+                            video\.telemillevaches\.net|
+                            video\.thepolarbear\.co\.uk|
                             video\.thinkof\.name|
-                            video\.toot\.pt|
+                            video\.tii\.space|
+                            video\.tkz\.es|
+                            video\.trankil\.info|
                             video\.triplea\.fr|
+                            video\.tum\.social|
                             video\.turbo\.chat|
+                            video\.uriopss-pdl\.fr|
+                            video\.ustim\.ru|
+                            video\.ut0pia\.org|
                             video\.vaku\.org\.ua|
+                            video\.vegafjord\.me|
                             video\.veloma\.org|
                             video\.violoncello\.ch|
-                            video\.wilkie\.how|
-                            video\.wsf2021\.info|
-                            videorelay\.co|
+                            video\.voidconspiracy\.band|
+                            video\.wakkeren\.nl|
+                            video\.windfluechter\.org|
+                            video\.ziez\.eu|
                             videos-passages\.huma-num\.fr|
-                            videos\.3d-wolf\.com|
+                            videos\.aadtp\.be|
                             videos\.ahp-numerique\.fr|
-                            videos\.alexandrebadalo\.pt|
+                            videos\.alamaisondulibre\.org|
                             videos\.archigny\.net|
+                            videos\.aroaduntraveled\.com|
+                            videos\.b4tech\.org|
                             videos\.benjaminbrady\.ie|
-                            videos\.buceoluegoexisto\.com|
-                            videos\.capas\.se|
-                            videos\.casually\.cat|
+                            videos\.bik\.opencloud\.lu|
                             videos\.cloudron\.io|
+                            videos\.codingotaku\.com|
                             videos\.coletivos\.org|
+                            videos\.collate\.social|
                             videos\.danksquad\.org|
-                            videos\.denshi\.live|
-                            videos\.fromouter\.space|
+                            videos\.digitaldragons\.eu|
+                            videos\.dromeadhere\.fr|
+                            videos\.explain-it\.org|
+                            videos\.factsonthegroundshow\.com|
+                            videos\.foilen\.com|
                             videos\.fsci\.in|
+                            videos\.gamercast\.net|
+                            videos\.gianmarco\.gg|
                             videos\.globenet\.org|
+                            videos\.grafo\.zone|
                             videos\.hauspie\.fr|
                             videos\.hush\.is|
+                            videos\.hyphalfusion\.network|
+                            videos\.icum\.to|
+                            videos\.im\.allmendenetz\.de|
+                            videos\.jacksonchen666\.com|
                             videos\.john-livingston\.fr|
-                            videos\.jordanwarne\.xyz|
-                            videos\.lavoixdessansvoix\.org|
+                            videos\.knazarov\.com|
+                            videos\.kuoushi\.com|
+                            videos\.laliguepaysdelaloire\.org|
+                            videos\.lemouvementassociatif-pdl\.org|
                             videos\.leslionsfloorball\.fr|
-                            videos\.lucero\.top|
-                            videos\.martyn\.berlin|
+                            videos\.librescrum\.org|
                             videos\.mastodont\.cat|
-                            videos\.monstro1\.com|
-                            videos\.npo\.city|
-                            videos\.optoutpod\.com|
-                            videos\.petch\.rocks|
-                            videos\.pzelawski\.xyz|
+                            videos\.metus\.ca|
+                            videos\.miolo\.org|
+                            videos\.offroad\.town|
+                            videos\.openmandriva\.org|
+                            videos\.parleur\.net|
+                            videos\.pcorp\.us|
+                            videos\.pop\.eu\.com|
                             videos\.rampin\.org|
+                            videos\.rauten\.co\.za|
+                            videos\.ritimo\.org|
+                            videos\.sarcasmstardust\.com|
                             videos\.scanlines\.xyz|
                             videos\.shmalls\.pw|
-                            videos\.sibear\.fr|
                             videos\.stadtfabrikanten\.org|
-                            videos\.tankernn\.eu|
+                            videos\.supertuxkart\.net|
                             videos\.testimonia\.org|
-                            videos\.thisishowidontdisappear\.com|
-                            videos\.traumaheilung\.net|
+                            videos\.thinkerview\.com|
+                            videos\.torrenezzi10\.xyz|
                             videos\.trom\.tf|
-                            videos\.wakkerewereld\.nu|
-                            videos\.weblib\.re|
+                            videos\.utsukta\.org|
+                            videos\.viorsan\.com|
+                            videos\.wherelinux\.xyz|
+                            videos\.wikilibriste\.fr|
                             videos\.yesil\.club|
+                            videos\.yeswiki\.net|
+                            videotube\.duckdns\.org|
+                            vids\.capypara\.de|
                             vids\.roshless\.me|
+                            vids\.stary\.pc\.pl|
                             vids\.tekdmn\.me|
-                            vidz\.dou\.bet|
-                            vod\.lumikko\.dev|
-                            vs\.uniter\.network|
+                            vidz\.julien\.ovh|
+                            views\.southfox\.me|
+                            virtual-girls-are\.definitely-for\.me|
+                            viste\.pt|
+                            vnchich\.com|
+                            vnop\.org|
+                            vod\.newellijay\.tv|
+                            voluntarytube\.com|
+                            vtr\.chikichiki\.tube|
                             vulgarisation-informatique\.fr|
-                            watch\.breadtube\.tv|
-                            watch\.deranalyst\.ch|
+                            watch\.easya\.solutions|
+                            watch\.goodluckgabe\.life|
                             watch\.ignorance\.eu|
-                            watch\.krazy\.party|
+                            watch\.jimmydore\.com|
                             watch\.libertaria\.space|
-                            watch\.rt4mn\.org|
-                            watch\.softinio\.com|
+                            watch\.nuked\.social|
+                            watch\.ocaml\.org|
+                            watch\.thelema\.social|
                             watch\.tubelab\.video|
                             web-fellow\.de|
                             webtv\.vandoeuvre\.net|
-                            wechill\.space|
+                            wetubevid\.online|
                             wikileaks\.video|
                             wiwi\.video|
-                            worldofvids\.com|
-                            wwtube\.net|
-                            www4\.mir\.inter21\.net|
-                            www\.birkeundnymphe\.de|
-                            www\.captain-german\.com|
-                            www\.wiki-tube\.de|
+                            wow\.such\.disappointment\.fail|
+                            www\.jvideos\.net|
+                            www\.kotikoff\.net|
+                            www\.makertube\.net|
+                            www\.mypeer\.tube|
+                            www\.nadajemy\.com|
+                            www\.neptube\.io|
+                            www\.rocaguinarda\.tv|
+                            www\.vnshow\.net|
                             xxivproduction\.video|
-                            xxx\.noho\.st|
+                            yt\.orokoro\.ru|
+                            ytube\.retronerd\.at|
+                            zumvideo\.de|
 
                             # from youtube-dl
                             peertube\.rainbowswingers\.net|
@@ -1305,24 +1571,6 @@ class PeerTubePlaylistIE(InfoExtractor):
                     (?P<id>[^/]+)
                     ''' % (PeerTubeIE._INSTANCES_RE, '|'.join(_TYPES.keys()))
     _TESTS = [{
-        'url': 'https://peertube.tux.ovh/w/p/3af94cba-95e8-4b74-b37a-807ab6d82526',
-        'info_dict': {
-            'id': '3af94cba-95e8-4b74-b37a-807ab6d82526',
-            'description': 'playlist',
-            'timestamp': 1611171863,
-            'title': 'playlist',
-        },
-        'playlist_mincount': 6,
-    }, {
-        'url': 'https://peertube.tux.ovh/w/p/wkyqcQBnsvFxtUB2pkYc1e',
-        'info_dict': {
-            'id': 'wkyqcQBnsvFxtUB2pkYc1e',
-            'description': 'Cette liste de vidéos contient uniquement les jeux qui peuvent être terminés en une seule vidéo.',
-            'title': 'Let\'s Play',
-            'timestamp': 1604147331,
-        },
-        'playlist_mincount': 6,
-    }, {
         'url': 'https://peertube.debian.social/w/p/hFdJoTuyhNJVa1cDWd1d12',
         'info_dict': {
             'id': 'hFdJoTuyhNJVa1cDWd1d12',

From 05420227aaab60a39c0f9ade069c5862be36b1fa Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Mon, 5 Feb 2024 20:39:07 +0000
Subject: [PATCH 848/871] [ie/nytimes] Extract timestamp (#9142)

Authored by: SirElderling
---
 yt_dlp/extractor/nytimes.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/nytimes.py b/yt_dlp/extractor/nytimes.py
index 354eb02c3..3019202a2 100644
--- a/yt_dlp/extractor/nytimes.py
+++ b/yt_dlp/extractor/nytimes.py
@@ -32,6 +32,7 @@ class NYTimesBaseIE(InfoExtractor):
         renderedRepresentation
       }
       duration
+      firstPublished
       promotionalHeadline
       promotionalMedia {
         ... on Image {
@@ -124,6 +125,7 @@ def _extract_video(self, media_id):
             'id': media_id,
             'title': data.get('promotionalHeadline'),
             'description': data.get('summary'),
+            'timestamp': parse_iso8601(data.get('firstPublished')),
             'duration': float_or_none(data.get('duration'), scale=1000),
             'creator': ', '.join(traverse_obj(data, (  # TODO: change to 'creators'
                 'bylines', ..., 'renderedRepresentation', {lambda x: remove_start(x, 'By ')}))),
@@ -145,8 +147,8 @@ class NYTimesIE(NYTimesBaseIE):
             'ext': 'mp4',
             'title': 'Verbatim: What Is a Photocopier?',
             'description': 'md5:93603dada88ddbda9395632fdc5da260',
-            'timestamp': 1398631707,  # FIXME
-            'upload_date': '20140427',  # FIXME
+            'timestamp': 1398646132,
+            'upload_date': '20140428',
             'creator': 'Brett Weiner',
             'thumbnail': r're:https?://\w+\.nyt.com/images/.+\.jpg',
             'duration': 419,
@@ -310,6 +312,8 @@ class NYTimesCookingIE(NYTimesBaseIE):
             'ext': 'mp4',
             'title': 'How to Make Mac and Cheese',
             'description': 'md5:b8f2f33ec1fb7523b21367147c9594f1',
+            'timestamp': 1522950315,
+            'upload_date': '20180405',
             'duration': 9.51,
             'creator': 'Alison Roman',
             'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',

From 540b68298192874c75ad5ee4589bed64d02a7d55 Mon Sep 17 00:00:00 2001
From: Dmitry Meyer <me@undef.im>
Date: Fri, 9 Feb 2024 18:34:56 +0300
Subject: [PATCH 849/871] [ie/Boosty] Add extractor (#9144)

Closes #5900, Closes #8704
Authored by: un-def
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/boosty.py      | 209 ++++++++++++++++++++++++++++++++
 2 files changed, 210 insertions(+)
 create mode 100644 yt_dlp/extractor/boosty.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index e7dd34c77..5d1dd6038 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -257,6 +257,7 @@
 from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bongacams import BongaCamsIE
+from .boosty import BoostyIE
 from .bostonglobe import BostonGlobeIE
 from .box import BoxIE
 from .boxcast import BoxCastVideoIE
diff --git a/yt_dlp/extractor/boosty.py b/yt_dlp/extractor/boosty.py
new file mode 100644
index 000000000..fb14ca146
--- /dev/null
+++ b/yt_dlp/extractor/boosty.py
@@ -0,0 +1,209 @@
+from .common import InfoExtractor
+from .youtube import YoutubeIE
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    qualities,
+    str_or_none,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class BoostyIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?boosty\.to/(?P<user>[^/#?]+)/posts/(?P<post_id>[^/#?]+)'
+    _TESTS = [{
+        # single ok_video
+        'url': 'https://boosty.to/kuplinov/posts/e55d050c-e3bb-4873-a7db-ac7a49b40c38',
+        'info_dict': {
+            'id': 'd7473824-352e-48e2-ae53-d4aa39459968',
+            'title': 'phasma_3',
+            'channel': 'Kuplinov',
+            'channel_id': '7958701',
+            'timestamp': 1655031975,
+            'upload_date': '20220612',
+            'release_timestamp': 1655049000,
+            'release_date': '20220612',
+            'modified_timestamp': 1668680993,
+            'modified_date': '20221117',
+            'tags': ['куплинов', 'phasmophobia'],
+            'like_count': int,
+            'ext': 'mp4',
+            'duration': 105,
+            'view_count': int,
+            'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+        },
+    }, {
+        # multiple ok_video
+        'url': 'https://boosty.to/maddyson/posts/0c652798-3b35-471f-8b48-a76a0b28736f',
+        'info_dict': {
+            'id': '0c652798-3b35-471f-8b48-a76a0b28736f',
+            'title': 'то что не пропустил юта6',
+            'channel': 'Илья Давыдов',
+            'channel_id': '6808257',
+            'timestamp': 1694017040,
+            'upload_date': '20230906',
+            'release_timestamp': 1694017040,
+            'release_date': '20230906',
+            'modified_timestamp': 1694071178,
+            'modified_date': '20230907',
+            'like_count': int,
+        },
+        'playlist_count': 3,
+        'playlist': [{
+            'info_dict': {
+                'id': 'cc325a9f-a563-41c6-bf47-516c1b506c9a',
+                'title': 'то что не пропустил юта6',
+                'channel': 'Илья Давыдов',
+                'channel_id': '6808257',
+                'timestamp': 1694017040,
+                'upload_date': '20230906',
+                'release_timestamp': 1694017040,
+                'release_date': '20230906',
+                'modified_timestamp': 1694071178,
+                'modified_date': '20230907',
+                'like_count': int,
+                'ext': 'mp4',
+                'duration': 31204,
+                'view_count': int,
+                'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+            },
+        }, {
+            'info_dict': {
+                'id': 'd07b0a72-9493-4512-b54e-55ce468fd4b7',
+                'title': 'то что не пропустил юта6',
+                'channel': 'Илья Давыдов',
+                'channel_id': '6808257',
+                'timestamp': 1694017040,
+                'upload_date': '20230906',
+                'release_timestamp': 1694017040,
+                'release_date': '20230906',
+                'modified_timestamp': 1694071178,
+                'modified_date': '20230907',
+                'like_count': int,
+                'ext': 'mp4',
+                'duration': 25704,
+                'view_count': int,
+                'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+            },
+        }, {
+            'info_dict': {
+                'id': '4a3bba32-78c8-422a-9432-2791aff60b42',
+                'title': 'то что не пропустил юта6',
+                'channel': 'Илья Давыдов',
+                'channel_id': '6808257',
+                'timestamp': 1694017040,
+                'upload_date': '20230906',
+                'release_timestamp': 1694017040,
+                'release_date': '20230906',
+                'modified_timestamp': 1694071178,
+                'modified_date': '20230907',
+                'like_count': int,
+                'ext': 'mp4',
+                'duration': 31867,
+                'view_count': int,
+                'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+            },
+        }],
+    }, {
+        # single external video (youtube)
+        'url': 'https://boosty.to/denischuzhoy/posts/6094a487-bcec-4cf8-a453-43313b463c38',
+        'info_dict': {
+            'id': 'EXelTnve5lY',
+            'title': 'Послание Президента Федеральному Собранию | Класс народа',
+            'upload_date': '20210425',
+            'channel': 'Денис Чужой',
+            'tags': 'count:10',
+            'like_count': int,
+            'ext': 'mp4',
+            'duration': 816,
+            'view_count': int,
+            'thumbnail': r're:^https://i\.ytimg\.com/',
+            'age_limit': 0,
+            'availability': 'public',
+            'categories': list,
+            'channel_follower_count': int,
+            'channel_id': 'UCCzVNbWZfYpBfyofCCUD_0w',
+            'channel_is_verified': bool,
+            'channel_url': r're:^https://www\.youtube\.com/',
+            'comment_count': int,
+            'description': str,
+            'heatmap': 'count:100',
+            'live_status': str,
+            'playable_in_embed': bool,
+            'uploader': str,
+            'uploader_id': str,
+            'uploader_url': r're:^https://www\.youtube\.com/',
+        },
+    }]
+
+    _MP4_TYPES = ('tiny', 'lowest', 'low', 'medium', 'high', 'full_hd', 'quad_hd', 'ultra_hd')
+
+    def _extract_formats(self, player_urls, video_id):
+        formats = []
+        quality = qualities(self._MP4_TYPES)
+        for player_url in traverse_obj(player_urls, lambda _, v: url_or_none(v['url'])):
+            url = player_url['url']
+            format_type = player_url.get('type')
+            if format_type in ('hls', 'hls_live', 'live_ondemand_hls', 'live_playback_hls'):
+                formats.extend(self._extract_m3u8_formats(url, video_id, m3u8_id='hls', fatal=False))
+            elif format_type in ('dash', 'dash_live', 'live_playback_dash'):
+                formats.extend(self._extract_mpd_formats(url, video_id, mpd_id='dash', fatal=False))
+            elif format_type in self._MP4_TYPES:
+                formats.append({
+                    'url': url,
+                    'ext': 'mp4',
+                    'format_id': format_type,
+                    'quality': quality(format_type),
+                })
+            else:
+                self.report_warning(f'Unknown format type: {format_type!r}')
+        return formats
+
+    def _real_extract(self, url):
+        user, post_id = self._match_valid_url(url).group('user', 'post_id')
+        post = self._download_json(
+            f'https://api.boosty.to/v1/blog/{user}/post/{post_id}', post_id,
+            note='Downloading post data', errnote='Unable to download post data')
+
+        post_title = post.get('title')
+        if not post_title:
+            self.report_warning('Unable to extract post title. Falling back to parsing html page')
+            webpage = self._download_webpage(url, video_id=post_id)
+            post_title = self._og_search_title(webpage, default=None) or self._html_extract_title(webpage)
+
+        common_metadata = {
+            'title': post_title,
+            **traverse_obj(post, {
+                'channel': ('user', 'name', {str}),
+                'channel_id': ('user', 'id', {str_or_none}),
+                'timestamp': ('createdAt', {int_or_none}),
+                'release_timestamp': ('publishTime', {int_or_none}),
+                'modified_timestamp': ('updatedAt', {int_or_none}),
+                'tags': ('tags', ..., 'title', {str}),
+                'like_count': ('count', 'likes', {int_or_none}),
+            }),
+        }
+        entries = []
+        for item in traverse_obj(post, ('data', ..., {dict})):
+            item_type = item.get('type')
+            if item_type == 'video' and url_or_none(item.get('url')):
+                entries.append(self.url_result(item['url'], YoutubeIE))
+            elif item_type == 'ok_video':
+                video_id = item.get('id') or post_id
+                entries.append({
+                    'id': video_id,
+                    'formats': self._extract_formats(item.get('playerUrls'), video_id),
+                    **common_metadata,
+                    **traverse_obj(item, {
+                        'title': ('title', {str}),
+                        'duration': ('duration', {int_or_none}),
+                        'view_count': ('viewsCounter', {int_or_none}),
+                        'thumbnail': (('previewUrl', 'defaultPreview'), {url_or_none}),
+                    }, get_all=False)})
+
+        if not entries:
+            raise ExtractorError('No videos found', expected=True)
+        if len(entries) == 1:
+            return entries[0]
+        return self.playlist_result(entries, post_id, post_title, **common_metadata)

From 882e3b753c79c7799ce135c3a5edb72494b576af Mon Sep 17 00:00:00 2001
From: "lauren n. liberda" <lauren@selfisekai.rocks>
Date: Sat, 10 Feb 2024 00:11:34 +0100
Subject: [PATCH 850/871] [ie/tvp] Support livestreams (#8860)

Closes #8824
Authored by: selfisekai
---
 yt_dlp/extractor/tvp.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/tvp.py b/yt_dlp/extractor/tvp.py
index 2aa0dd870..a8d00e243 100644
--- a/yt_dlp/extractor/tvp.py
+++ b/yt_dlp/extractor/tvp.py
@@ -21,7 +21,7 @@
 class TVPIE(InfoExtractor):
     IE_NAME = 'tvp'
     IE_DESC = 'Telewizja Polska'
-    _VALID_URL = r'https?://(?:[^/]+\.)?(?:tvp(?:parlament)?\.(?:pl|info)|tvpworld\.com|swipeto\.pl)/(?:(?!\d+/)[^/]+/)*(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?(?:tvp(?:parlament)?\.(?:pl|info)|tvpworld\.com|swipeto\.pl)/(?:(?!\d+/)[^/]+/)*(?P<id>\d+)(?:[/?#]|$)'
 
     _TESTS = [{
         # TVPlayer 2 in js wrapper
@@ -514,7 +514,7 @@ def _parse_video(self, video, with_url=True):
 
 class TVPVODVideoIE(TVPVODBaseIE):
     IE_NAME = 'tvp:vod'
-    _VALID_URL = r'https?://vod\.tvp\.pl/[a-z\d-]+,\d+/[a-z\d-]+(?<!-odcinki)(?:-odcinki,\d+/odcinek-\d+,S\d+E\d+)?,(?P<id>\d+)(?:\?[^#]+)?(?:#.+)?$'
+    _VALID_URL = r'https?://vod\.tvp\.pl/(?P<category>[a-z\d-]+,\d+)/[a-z\d-]+(?<!-odcinki)(?:-odcinki,\d+/odcinek-\d+,S\d+E\d+)?,(?P<id>\d+)/?(?:[?#]|$)'
 
     _TESTS = [{
         'url': 'https://vod.tvp.pl/dla-dzieci,24/laboratorium-alchemika-odcinki,309338/odcinek-24,S01E24,311357',
@@ -560,12 +560,23 @@ class TVPVODVideoIE(TVPVODBaseIE):
             'thumbnail': 're:https?://.+',
         },
         'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://vod.tvp.pl/live,1/tvp-world,399731',
+        'info_dict': {
+            'id': '399731',
+            'ext': 'mp4',
+            'title': r're:TVP WORLD \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
+            'live_status': 'is_live',
+            'thumbnail': 're:https?://.+',
+        },
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        category, video_id = self._match_valid_url(url).group('category', 'id')
 
-        info_dict = self._parse_video(self._call_api(f'vods/{video_id}', video_id), with_url=False)
+        is_live = category == 'live,1'
+        entity = 'lives' if is_live else 'vods'
+        info_dict = self._parse_video(self._call_api(f'{entity}/{video_id}', video_id), with_url=False)
 
         playlist = self._call_api(f'{video_id}/videos/playlist', video_id, query={'videoType': 'MOVIE'})
 
@@ -582,6 +593,8 @@ def _real_extract(self, url):
                 'ext': 'ttml',
             })
 
+        info_dict['is_live'] = is_live
+
         return info_dict
 
 

From a1b778428991b1779203bac243ef4e9b6baea90c Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sun, 11 Feb 2024 14:58:18 +0100
Subject: [PATCH 851/871] [build] Move bundle scripts into `bundle` submodule

Authored by: bashonly
---
 .github/workflows/build.yml           | 20 ++++-----
 .github/workflows/release-master.yml  |  2 +-
 .github/workflows/release-nightly.yml |  2 +-
 README.md                             | 24 ++++++-----
 bundle/__init__.py                    |  1 +
 bundle/py2exe.py                      | 59 +++++++++++++++++++++++++++
 pyinst.py => bundle/pyinstaller.py    |  2 +-
 pyproject.toml                        |  3 ++
 setup.py                              | 56 +------------------------
 9 files changed, 91 insertions(+), 78 deletions(-)
 create mode 100644 bundle/__init__.py
 create mode 100755 bundle/py2exe.py
 rename pyinst.py => bundle/pyinstaller.py (98%)
 mode change 100644 => 100755

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 036ce4348..4b05e7cf9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -144,9 +144,9 @@ jobs:
         run: |
           unset LD_LIBRARY_PATH  # Harmful; set by setup-python
           conda activate build
-          python pyinst.py --onedir
+          python -m bundle.pyinstaller --onedir
           (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
-          python pyinst.py
+          python -m bundle.pyinstaller
           mv ./dist/yt-dlp_linux ./yt-dlp_linux
           mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip
 
@@ -211,7 +211,7 @@ jobs:
             python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt  # Cached version may be out of date
             python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
             python3.8 devscripts/make_lazy_extractors.py
-            python3.8 pyinst.py
+            python3.8 -m bundle.pyinstaller
 
             if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then
               arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}"
@@ -250,9 +250,9 @@ jobs:
           python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
-          python3 pyinst.py --target-architecture universal2 --onedir
+          python3 -m bundle.pyinstaller --target-architecture universal2 --onedir
           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
-          python3 pyinst.py --target-architecture universal2
+          python3 -m bundle.pyinstaller --target-architecture universal2
 
       - name: Verify --update-to
         if: vars.UPDATE_TO_VERIFICATION
@@ -302,7 +302,7 @@ jobs:
           python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
-          python3 pyinst.py
+          python3 -m bundle.pyinstaller
           mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
 
       - name: Verify --update-to
@@ -342,10 +342,10 @@ jobs:
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
-          python setup.py py2exe
+          python -m bundle.py2exe
           Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
-          python pyinst.py
-          python pyinst.py --onedir
+          python -m bundle.pyinstaller
+          python -m bundle.pyinstaller --onedir
           Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
 
       - name: Verify --update-to
@@ -391,7 +391,7 @@ jobs:
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
-          python pyinst.py
+          python -m bundle.pyinstaller
 
       - name: Verify --update-to
         if: vars.UPDATE_TO_VERIFICATION
diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 0664137a9..af14b053e 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -7,7 +7,7 @@ on:
       - "yt_dlp/**.py"
       - "!yt_dlp/version.py"
       - "setup.py"
-      - "pyinst.py"
+      - "bundle/*.py"
 concurrency:
   group: release-master
 permissions:
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index 2e623a67c..3f1418936 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Check for new commits
         id: check_for_new_commits
         run: |
-          relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "setup.py" "pyinst.py")
+          relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "setup.py" "bundle/*.py")
           echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT"
 
   release:
diff --git a/README.md b/README.md
index 7dc3bb2f6..c74777d2f 100644
--- a/README.md
+++ b/README.md
@@ -321,19 +321,21 @@ ### Deprecated
 ## COMPILE
 
 ### Standalone PyInstaller Builds
-To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). Once you have all the necessary dependencies installed, simply run `pyinst.py`. The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used.
+To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used. You can run the following commands:
 
-    python3 -m pip install -U pyinstaller -r requirements.txt
-    python3 devscripts/make_lazy_extractors.py
-    python3 pyinst.py
+```
+python3 -m pip install -U pyinstaller -r requirements.txt
+python3 devscripts/make_lazy_extractors.py
+python3 -m bundle.pyinstaller
+```
 
 On some systems, you may need to use `py` or `python` instead of `python3`.
 
-`pyinst.py` accepts any arguments that can be passed to `pyinstaller`, such as `--onefile/-F` or `--onedir/-D`, which is further [documented here](https://pyinstaller.org/en/stable/usage.html#what-to-generate).
+`bundle/pyinstaller.py` accepts any arguments that can be passed to `pyinstaller`, such as `--onefile/-F` or `--onedir/-D`, which is further [documented here](https://pyinstaller.org/en/stable/usage.html#what-to-generate).
 
 **Note**: Pyinstaller versions below 4.4 [do not support](https://github.com/pyinstaller/pyinstaller#requirements-and-tested-platforms) Python installed from the Windows store without using a virtual environment.
 
-**Important**: Running `pyinstaller` directly **without** using `pyinst.py` is **not** officially supported. This may or may not work correctly.
+**Important**: Running `pyinstaller` directly **without** using `bundle/pyinstaller.py` is **not** officially supported. This may or may not work correctly.
 
 ### Platform-independent Binary (UNIX)
 You will need the build tools `python` (3.8+), `zip`, `make` (GNU), `pandoc`\* and `pytest`\*.
@@ -346,11 +348,13 @@ ### Standalone Py2Exe Builds (Windows)
 
 While we provide the option to build with [py2exe](https://www.py2exe.org), it is recommended to build [using PyInstaller](#standalone-pyinstaller-builds) instead since the py2exe builds **cannot contain `pycryptodomex`/`certifi` and needs VC++14** on the target computer to run.
 
-If you wish to build it anyway, install Python and py2exe, and then simply run `setup.py py2exe`
+If you wish to build it anyway, install Python (if it is not already installed) and you can run the following commands:
 
-    py -m pip install -U py2exe -r requirements.txt
-    py devscripts/make_lazy_extractors.py
-    py setup.py py2exe
+```
+py -m pip install -U py2exe -r requirements.txt
+py devscripts/make_lazy_extractors.py
+py -m bundle.py2exe
+```
 
 ### Related scripts
 
diff --git a/bundle/__init__.py b/bundle/__init__.py
new file mode 100644
index 000000000..932b79829
--- /dev/null
+++ b/bundle/__init__.py
@@ -0,0 +1 @@
+# Empty file
diff --git a/bundle/py2exe.py b/bundle/py2exe.py
new file mode 100755
index 000000000..a7e4113f1
--- /dev/null
+++ b/bundle/py2exe.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+# Allow execution from anywhere
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import warnings
+
+from py2exe import freeze
+
+from devscripts.utils import read_version
+
+VERSION = read_version()
+
+
+def main():
+    warnings.warn(
+        'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
+        'It is recommended to run "pyinst.py" to build using pyinstaller instead')
+
+    return freeze(
+        console=[{
+            'script': './yt_dlp/__main__.py',
+            'dest_base': 'yt-dlp',
+            'icon_resources': [(1, 'devscripts/logo.ico')],
+        }],
+        version_info={
+            'version': VERSION,
+            'description': 'A youtube-dl fork with additional features and patches',
+            'comments': 'Official repository: <https://github.com/yt-dlp/yt-dlp>',
+            'product_name': 'yt-dlp',
+            'product_version': VERSION,
+        },
+        options={
+            'bundle_files': 0,
+            'compressed': 1,
+            'optimize': 2,
+            'dist_dir': './dist',
+            'excludes': [
+                # py2exe cannot import Crypto
+                'Crypto',
+                'Cryptodome',
+                # py2exe appears to confuse this with our socks library.
+                # We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
+                'urllib3.contrib.socks'
+            ],
+            'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
+            # Modules that are only imported dynamically must be added here
+            'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
+                         'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
+        },
+        zipfile=None,
+    )
+
+
+if __name__ == '__main__':
+    main()
diff --git a/pyinst.py b/bundle/pyinstaller.py
old mode 100644
new mode 100755
similarity index 98%
rename from pyinst.py
rename to bundle/pyinstaller.py
index c36f6acd4..db9dbfde5
--- a/pyinst.py
+++ b/bundle/pyinstaller.py
@@ -4,7 +4,7 @@
 import os
 import sys
 
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import platform
 
diff --git a/pyproject.toml b/pyproject.toml
index 97718ec43..626d9aa13 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,3 +3,6 @@ build-backend = 'setuptools.build_meta'
 # https://github.com/yt-dlp/yt-dlp/issues/5941
 # https://github.com/pypa/distutils/issues/17
 requires = ['setuptools > 50']
+
+[project.entry-points.pyinstaller40]
+hook-dirs = "yt_dlp.__pyinstaller:get_hook_dirs"
diff --git a/setup.py b/setup.py
index 3d9a69d10..fc5b50468 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,6 @@
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
 import subprocess
-import warnings
 
 try:
     from setuptools import Command, find_packages, setup
@@ -39,46 +38,6 @@ def packages():
     ]
 
 
-def py2exe_params():
-    warnings.warn(
-        'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
-        'It is recommended to run "pyinst.py" to build using pyinstaller instead')
-
-    return {
-        'console': [{
-            'script': './yt_dlp/__main__.py',
-            'dest_base': 'yt-dlp',
-            'icon_resources': [(1, 'devscripts/logo.ico')],
-        }],
-        'version_info': {
-            'version': VERSION,
-            'description': DESCRIPTION,
-            'comments': LONG_DESCRIPTION.split('\n')[0],
-            'product_name': 'yt-dlp',
-            'product_version': VERSION,
-        },
-        'options': {
-            'bundle_files': 0,
-            'compressed': 1,
-            'optimize': 2,
-            'dist_dir': './dist',
-            'excludes': [
-                # py2exe cannot import Crypto
-                'Crypto',
-                'Cryptodome',
-                # py2exe appears to confuse this with our socks library.
-                # We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
-                'urllib3.contrib.socks'
-            ],
-            'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
-            # Modules that are only imported dynamically must be added here
-            'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
-                         'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
-        },
-        'zipfile': None,
-    }
-
-
 def build_params():
     files_spec = [
         ('share/bash-completion/completions', ['completions/bash/yt-dlp']),
@@ -127,20 +86,7 @@ def run(self):
 
 
 def main():
-    if sys.argv[1:2] == ['py2exe']:
-        params = py2exe_params()
-        try:
-            from py2exe import freeze
-        except ImportError:
-            import py2exe  # noqa: F401
-            warnings.warn('You are using an outdated version of py2exe. Support for this version will be removed in the future')
-            params['console'][0].update(params.pop('version_info'))
-            params['options'] = {'py2exe': params.pop('options')}
-        else:
-            return freeze(**params)
-    else:
-        params = build_params()
-
+    params = build_params()
     setup(
         name='yt-dlp',  # package name (do not change/remove comment)
         version=VERSION,

From 868d2f60a7cb59b410c8cbfb452cbdb072687b81 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sun, 11 Feb 2024 15:07:45 +0100
Subject: [PATCH 852/871] [build:Makefile] Add automated `CODE_FOLDERS` and
 `CODE_FILES`

Authored by: bashonly
---
 Makefile | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index c85b24c13..296fc3260 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,7 @@ clean-test:
 	*.mp4 *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.swf *.swp *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
 clean-dist:
 	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
-	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
+	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS
 clean-cache:
 	find . \( \
 		-type d -name .pytest_cache -o -type d -name __pycache__ -o -name "*.pyc" -o -name "*.class" \
@@ -73,24 +73,24 @@ test:
 offlinetest: codetest
 	$(PYTHON) -m pytest -k "not download"
 
-# XXX: This is hard to maintain
-CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/compat/urllib yt_dlp/utils yt_dlp/dependencies yt_dlp/networking
-yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
+CODE_FOLDERS := $(shell find yt_dlp -type d -not -name '__*' -exec sh -c 'test -e "$$1"/__init__.py' sh {} \; -print)
+CODE_FILES := $(shell for f in $(CODE_FOLDERS); do echo "$$f" | awk '{gsub(/\/[^\/]+/,"/*"); print $$1"/*.py"}'; done | sort -u)
+yt-dlp: $(CODE_FILES)
 	mkdir -p zip
 	for d in $(CODE_FOLDERS) ; do \
 	  mkdir -p zip/$$d ;\
 	  cp -pPR $$d/*.py zip/$$d/ ;\
 	done
-	touch -t 200001010101 zip/yt_dlp/*.py zip/yt_dlp/*/*.py
+	cd zip ; touch -t 200001010101 $(CODE_FILES)
 	mv zip/yt_dlp/__main__.py zip/
-	cd zip ; zip -q ../yt-dlp yt_dlp/*.py yt_dlp/*/*.py __main__.py
+	cd zip ; zip -q ../yt-dlp $(CODE_FILES) __main__.py
 	rm -rf zip
 	echo '#!$(PYTHON)' > yt-dlp
 	cat yt-dlp.zip >> yt-dlp
 	rm yt-dlp.zip
 	chmod a+x yt-dlp
 
-README.md: yt_dlp/*.py yt_dlp/*/*.py devscripts/make_readme.py
+README.md: $(CODE_FILES) devscripts/make_readme.py
 	COLUMNS=80 $(PYTHON) yt_dlp/__main__.py --ignore-config --help | $(PYTHON) devscripts/make_readme.py
 
 CONTRIBUTING.md: README.md devscripts/make_contributing.py
@@ -115,15 +115,15 @@ yt-dlp.1: README.md devscripts/prepare_manpage.py
 	pandoc -s -f $(MARKDOWN) -t man yt-dlp.1.temp.md -o yt-dlp.1
 	rm -f yt-dlp.1.temp.md
 
-completions/bash/yt-dlp: yt_dlp/*.py yt_dlp/*/*.py devscripts/bash-completion.in
+completions/bash/yt-dlp: $(CODE_FILES) devscripts/bash-completion.in
 	mkdir -p completions/bash
 	$(PYTHON) devscripts/bash-completion.py
 
-completions/zsh/_yt-dlp: yt_dlp/*.py yt_dlp/*/*.py devscripts/zsh-completion.in
+completions/zsh/_yt-dlp: $(CODE_FILES) devscripts/zsh-completion.in
 	mkdir -p completions/zsh
 	$(PYTHON) devscripts/zsh-completion.py
 
-completions/fish/yt-dlp.fish: yt_dlp/*.py yt_dlp/*/*.py devscripts/fish-completion.in
+completions/fish/yt-dlp.fish: $(CODE_FILES) devscripts/fish-completion.in
 	mkdir -p completions/fish
 	$(PYTHON) devscripts/fish-completion.py
 
@@ -148,8 +148,5 @@ yt-dlp.tar.gz: all
 		setup.py setup.cfg yt-dlp yt_dlp requirements.txt \
 		devscripts test
 
-AUTHORS: .mailmap
-	git shortlog -s -n | cut -f2 | sort > AUTHORS
-
-.mailmap:
-	git shortlog -s -e -n | awk '!(out[$$NF]++) { $$1="";sub(/^[ \t]+/,""); print}' > .mailmap
+AUTHORS:
+	git shortlog -s -n HEAD | cut -f2 | sort > AUTHORS

From 775cde82dc5b1dc64ab0539a92dd8c7ba6c0ad33 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sun, 11 Feb 2024 15:13:03 +0100
Subject: [PATCH 853/871] [build] Migrate to `pyproject.toml` and `hatchling`

Authored by: bashonly
---
 .github/workflows/release-master.yml  |   2 +-
 .github/workflows/release-nightly.yml |   2 +-
 .github/workflows/release.yml         |   9 +-
 MANIFEST.in                           |  10 --
 Makefile                              |  11 +--
 pyproject.toml                        | 120 +++++++++++++++++++++++-
 setup.cfg                             |   4 -
 setup.py                              | 129 --------------------------
 8 files changed, 130 insertions(+), 157 deletions(-)
 delete mode 100644 MANIFEST.in
 delete mode 100644 setup.py

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index af14b053e..2430dc5f8 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -6,8 +6,8 @@ on:
     paths:
       - "yt_dlp/**.py"
       - "!yt_dlp/version.py"
-      - "setup.py"
       - "bundle/*.py"
+      - "pyproject.toml"
 concurrency:
   group: release-master
 permissions:
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index 3f1418936..16d583846 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Check for new commits
         id: check_for_new_commits
         run: |
-          relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "setup.py" "bundle/*.py")
+          relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "bundle/*.py" "pyproject.toml")
           echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT"
 
   release:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 69b5e3152..d1508e5e6 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -266,14 +266,19 @@ jobs:
         run: |
           python devscripts/update-version.py -c "${{ env.channel }}" -r "${{ env.target_repo }}" -s "${{ env.suffix }}" "${{ env.version }}"
           python devscripts/make_lazy_extractors.py
-          sed -i -E "s/(name=')[^']+(',  # package name)/\1${{ env.pypi_project }}\2/" setup.py
+          sed -i -E '0,/(name = ")[^"]+(")/s//\1${{ env.pypi_project }}\2/' pyproject.toml
 
       - name: Build
         run: |
           rm -rf dist/*
           make pypi-files
+          printf '%s\n\n' \
+            'Official repository: <https://github.com/yt-dlp/yt-dlp>' \
+            '**PS**: Some links in this document will not work since this is a copy of the README.md from Github' > ./README.md.new
+          cat ./README.md >> ./README.md.new && mv -f ./README.md.new ./README.md
           python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
-          python setup.py sdist bdist_wheel
+          make clean-cache
+          python -m build --no-isolation .
 
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index bc2f056c0..000000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1,10 +0,0 @@
-include AUTHORS
-include Changelog.md
-include LICENSE
-include README.md
-include completions/*/*
-include supportedsites.md
-include yt-dlp.1
-include requirements.txt
-recursive-include devscripts *
-recursive-include test *
diff --git a/Makefile b/Makefile
index 296fc3260..2f36c0cd1 100644
--- a/Makefile
+++ b/Makefile
@@ -6,11 +6,11 @@ doc: README.md CONTRIBUTING.md issuetemplates supportedsites
 ot: offlinetest
 tar: yt-dlp.tar.gz
 
-# Keep this list in sync with MANIFEST.in
+# Keep this list in sync with pyproject.toml includes/artifacts
 # intended use: when building a source distribution,
-# make pypi-files && python setup.py sdist
+# make pypi-files && python3 -m build -sn .
 pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \
-	        completions yt-dlp.1 requirements.txt setup.cfg devscripts/* test/*
+	        completions yt-dlp.1 pyproject.toml setup.cfg devscripts/* test/*
 
 .PHONY: all clean install test tar pypi-files completions ot offlinetest codetest supportedsites
 
@@ -144,9 +144,8 @@ yt-dlp.tar.gz: all
 		-- \
 		README.md supportedsites.md Changelog.md LICENSE \
 		CONTRIBUTING.md Collaborators.md CONTRIBUTORS AUTHORS \
-		Makefile MANIFEST.in yt-dlp.1 README.txt completions \
-		setup.py setup.cfg yt-dlp yt_dlp requirements.txt \
-		devscripts test
+		Makefile yt-dlp.1 README.txt completions .gitignore \
+		setup.cfg yt-dlp yt_dlp pyproject.toml devscripts test
 
 AUTHORS:
 	git shortlog -s -n HEAD | cut -f2 | sort > AUTHORS
diff --git a/pyproject.toml b/pyproject.toml
index 626d9aa13..5ef013279 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,8 +1,120 @@
 [build-system]
-build-backend = 'setuptools.build_meta'
-# https://github.com/yt-dlp/yt-dlp/issues/5941
-# https://github.com/pypa/distutils/issues/17
-requires = ['setuptools > 50']
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "yt-dlp"
+maintainers = [
+    {name = "pukkandan", email = "pukkandan.ytdlp@gmail.com"},
+    {name = "Grub4K", email = "contact@grub4k.xyz"},
+    {name = "bashonly", email = "bashonly@protonmail.com"},
+]
+description = "A youtube-dl fork with additional features and patches"
+readme = "README.md"
+requires-python = ">=3.8"
+keywords = [
+    "youtube-dl",
+    "video-downloader",
+    "youtube-downloader",
+    "sponsorblock",
+    "youtube-dlc",
+    "yt-dlp",
+]
+license = {file = "LICENSE"}
+classifiers = [
+    "Topic :: Multimedia :: Video",
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: Implementation",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "License :: OSI Approved :: The Unlicense (Unlicense)",
+    "Operating System :: OS Independent",
+]
+dynamic = ["version"]
+dependencies = [
+    "brotli; implementation_name=='cpython'",
+    "brotlicffi; implementation_name!='cpython'",
+    "certifi",
+    "mutagen",
+    "pycryptodomex",
+    "requests>=2.31.0,<3",
+    "urllib3>=1.26.17,<3",
+    "websockets>=12.0",
+]
+
+[project.optional-dependencies]
+secretstorage = [
+    "cffi",
+    "secretstorage",
+]
+build = [
+    "build",
+    "hatchling",
+    "pip",
+    "wheel",
+]
+dev = [
+    "flake8",
+    "isort",
+    "pytest",
+]
+pyinstaller = ["pyinstaller>=6.3"]
+py2exe = ["py2exe>=0.12"]
+
+[project.urls]
+Documentation = "https://github.com/yt-dlp/yt-dlp#readme"
+Repository = "https://github.com/yt-dlp/yt-dlp"
+Tracker = "https://github.com/yt-dlp/yt-dlp/issues"
+Funding = "https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators"
+
+[project.scripts]
+yt-dlp = "yt_dlp:main"
 
 [project.entry-points.pyinstaller40]
 hook-dirs = "yt_dlp.__pyinstaller:get_hook_dirs"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/yt_dlp",
+    "/devscripts",
+    "/test",
+    "/.gitignore",  # included by default, needed for auto-excludes
+    "/Changelog.md",
+    "/LICENSE",  # included as license
+    "/pyproject.toml",  # included by default
+    "/README.md",  # included as readme
+    "/setup.cfg",
+    "/supportedsites.md",
+]
+exclude = ["/yt_dlp/__pyinstaller"]
+artifacts = [
+    "/yt_dlp/extractor/lazy_extractors.py",
+    "/completions",
+    "/AUTHORS",  # included by default
+    "/README.txt",
+    "/yt-dlp.1",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["yt_dlp"]
+exclude = ["/yt_dlp/__pyinstaller"]
+artifacts = ["/yt_dlp/extractor/lazy_extractors.py"]
+
+[tool.hatch.build.targets.wheel.shared-data]
+"completions/bash/yt-dlp" = "share/bash-completion/completions/yt-dlp"
+"completions/zsh/_yt-dlp" = "share/zsh/site-functions/_yt-dlp"
+"completions/fish/yt-dlp.fish" = "share/fish/vendor_completions.d/yt-dlp.fish"
+"README.txt" = "share/doc/yt_dlp/README.txt"
+"yt-dlp.1" = "share/man/man1/yt-dlp.1"
+
+[tool.hatch.version]
+path = "yt_dlp/version.py"
+pattern = "_pkg_version = '(?P<version>[^']+)'"
diff --git a/setup.cfg b/setup.cfg
index a799f7293..aeb4cee58 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,7 +1,3 @@
-[wheel]
-universal = true
-
-
 [flake8]
 exclude = build,venv,.tox,.git,.pytest_cache
 ignore = E402,E501,E731,E741,W503
diff --git a/setup.py b/setup.py
deleted file mode 100644
index fc5b50468..000000000
--- a/setup.py
+++ /dev/null
@@ -1,129 +0,0 @@
-#!/usr/bin/env python3
-
-# Allow execution from anywhere
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-import subprocess
-
-try:
-    from setuptools import Command, find_packages, setup
-    setuptools_available = True
-except ImportError:
-    from distutils.core import Command, setup
-    setuptools_available = False
-
-from devscripts.utils import read_file, read_version
-
-VERSION = read_version(varname='_pkg_version')
-
-DESCRIPTION = 'A youtube-dl fork with additional features and patches'
-
-LONG_DESCRIPTION = '\n\n'.join((
-    'Official repository: <https://github.com/yt-dlp/yt-dlp>',
-    '**PS**: Some links in this document will not work since this is a copy of the README.md from Github',
-    read_file('README.md')))
-
-REQUIREMENTS = read_file('requirements.txt').splitlines()
-
-
-def packages():
-    if setuptools_available:
-        return find_packages(exclude=('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts'))
-
-    return [
-        'yt_dlp', 'yt_dlp.extractor', 'yt_dlp.downloader', 'yt_dlp.postprocessor', 'yt_dlp.compat',
-    ]
-
-
-def build_params():
-    files_spec = [
-        ('share/bash-completion/completions', ['completions/bash/yt-dlp']),
-        ('share/zsh/site-functions', ['completions/zsh/_yt-dlp']),
-        ('share/fish/vendor_completions.d', ['completions/fish/yt-dlp.fish']),
-        ('share/doc/yt_dlp', ['README.txt']),
-        ('share/man/man1', ['yt-dlp.1'])
-    ]
-    data_files = []
-    for dirname, files in files_spec:
-        resfiles = []
-        for fn in files:
-            if not os.path.exists(fn):
-                warnings.warn(f'Skipping file {fn} since it is not present. Try running " make pypi-files " first')
-            else:
-                resfiles.append(fn)
-        data_files.append((dirname, resfiles))
-
-    params = {'data_files': data_files}
-
-    if setuptools_available:
-        params['entry_points'] = {
-            'console_scripts': ['yt-dlp = yt_dlp:main'],
-            'pyinstaller40': ['hook-dirs = yt_dlp.__pyinstaller:get_hook_dirs'],
-        }
-    else:
-        params['scripts'] = ['yt-dlp']
-    return params
-
-
-class build_lazy_extractors(Command):
-    description = 'Build the extractor lazy loading module'
-    user_options = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        if self.dry_run:
-            print('Skipping build of lazy extractors in dry run mode')
-            return
-        subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py'])
-
-
-def main():
-    params = build_params()
-    setup(
-        name='yt-dlp',  # package name (do not change/remove comment)
-        version=VERSION,
-        maintainer='pukkandan',
-        maintainer_email='pukkandan.ytdlp@gmail.com',
-        description=DESCRIPTION,
-        long_description=LONG_DESCRIPTION,
-        long_description_content_type='text/markdown',
-        url='https://github.com/yt-dlp/yt-dlp',
-        packages=packages(),
-        install_requires=REQUIREMENTS,
-        python_requires='>=3.8',
-        project_urls={
-            'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
-            'Source': 'https://github.com/yt-dlp/yt-dlp',
-            'Tracker': 'https://github.com/yt-dlp/yt-dlp/issues',
-            'Funding': 'https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators',
-        },
-        classifiers=[
-            'Topic :: Multimedia :: Video',
-            'Development Status :: 5 - Production/Stable',
-            'Environment :: Console',
-            'Programming Language :: Python',
-            'Programming Language :: Python :: 3.8',
-            'Programming Language :: Python :: 3.9',
-            'Programming Language :: Python :: 3.10',
-            'Programming Language :: Python :: 3.11',
-            'Programming Language :: Python :: 3.12',
-            'Programming Language :: Python :: Implementation',
-            'Programming Language :: Python :: Implementation :: CPython',
-            'Programming Language :: Python :: Implementation :: PyPy',
-            'License :: Public Domain',
-            'Operating System :: OS Independent',
-        ],
-        cmdclass={'build_lazy_extractors': build_lazy_extractors},
-        **params
-    )
-
-
-main()

From fd647775e27e030ab17387c249e2ebeba68f8ff0 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 11 Feb 2024 15:14:42 +0100
Subject: [PATCH 854/871] [devscripts] `tomlparse`: Add makeshift toml parser

Authored by: Grub4K
---
 devscripts/tomlparse.py | 189 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 189 insertions(+)
 create mode 100755 devscripts/tomlparse.py

diff --git a/devscripts/tomlparse.py b/devscripts/tomlparse.py
new file mode 100755
index 000000000..85ac4eef7
--- /dev/null
+++ b/devscripts/tomlparse.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+
+"""
+Simple parser for spec compliant toml files
+
+A simple toml parser for files that comply with the spec.
+Should only be used to parse `pyproject.toml` for `install_deps.py`.
+
+IMPORTANT: INVALID FILES OR MULTILINE STRINGS ARE NOT SUPPORTED!
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+import re
+
+WS = r'(?:[\ \t]*)'
+STRING_RE = re.compile(r'"(?:\\.|[^\\"\n])*"|\'[^\'\n]*\'')
+SINGLE_KEY_RE = re.compile(rf'{STRING_RE.pattern}|[A-Za-z0-9_-]+')
+KEY_RE = re.compile(rf'{WS}(?:{SINGLE_KEY_RE.pattern}){WS}(?:\.{WS}(?:{SINGLE_KEY_RE.pattern}){WS})*')
+EQUALS_RE = re.compile(rf'={WS}')
+WS_RE = re.compile(WS)
+
+_SUBTABLE = rf'(?P<subtable>^\[(?P<is_list>\[)?(?P<path>{KEY_RE.pattern})\]\]?)'
+EXPRESSION_RE = re.compile(rf'^(?:{_SUBTABLE}|{KEY_RE.pattern}=)', re.MULTILINE)
+
+LIST_WS_RE = re.compile(rf'{WS}((#[^\n]*)?\n{WS})*')
+LEFTOVER_VALUE_RE = re.compile(r'[^,}\]\t\n#]+')
+
+
+def parse_key(value: str):
+    for match in SINGLE_KEY_RE.finditer(value):
+        if match[0][0] == '"':
+            yield json.loads(match[0])
+        elif match[0][0] == '\'':
+            yield match[0][1:-1]
+        else:
+            yield match[0]
+
+
+def get_target(root: dict, paths: list[str], is_list=False):
+    target = root
+
+    for index, key in enumerate(paths, 1):
+        use_list = is_list and index == len(paths)
+        result = target.get(key)
+        if result is None:
+            result = [] if use_list else {}
+            target[key] = result
+
+        if isinstance(result, dict):
+            target = result
+        elif use_list:
+            target = {}
+            result.append(target)
+        else:
+            target = result[-1]
+
+    assert isinstance(target, dict)
+    return target
+
+
+def parse_enclosed(data: str, index: int, end: str, ws_re: re.Pattern):
+    index += 1
+
+    if match := ws_re.match(data, index):
+        index = match.end()
+
+    while data[index] != end:
+        index = yield True, index
+
+        if match := ws_re.match(data, index):
+            index = match.end()
+
+        if data[index] == ',':
+            index += 1
+
+        if match := ws_re.match(data, index):
+            index = match.end()
+
+    assert data[index] == end
+    yield False, index + 1
+
+
+def parse_value(data: str, index: int):
+    if data[index] == '[':
+        result = []
+
+        indices = parse_enclosed(data, index, ']', LIST_WS_RE)
+        valid, index = next(indices)
+        while valid:
+            index, value = parse_value(data, index)
+            result.append(value)
+            valid, index = indices.send(index)
+
+        return index, result
+
+    if data[index] == '{':
+        result = {}
+
+        indices = parse_enclosed(data, index, '}', WS_RE)
+        valid, index = next(indices)
+        while valid:
+            valid, index = indices.send(parse_kv_pair(data, index, result))
+
+        return index, result
+
+    if match := STRING_RE.match(data, index):
+        return match.end(), json.loads(match[0]) if match[0][0] == '"' else match[0][1:-1]
+
+    match = LEFTOVER_VALUE_RE.match(data, index)
+    assert match
+    value = match[0].strip()
+    for func in [
+        int,
+        float,
+        datetime.time.fromisoformat,
+        datetime.date.fromisoformat,
+        datetime.datetime.fromisoformat,
+        {'true': True, 'false': False}.get,
+    ]:
+        try:
+            value = func(value)
+            break
+        except Exception:
+            pass
+
+    return match.end(), value
+
+
+def parse_kv_pair(data: str, index: int, target: dict):
+    match = KEY_RE.match(data, index)
+    if not match:
+        return None
+
+    *keys, key = parse_key(match[0])
+
+    match = EQUALS_RE.match(data, match.end())
+    assert match
+    index = match.end()
+
+    index, value = parse_value(data, index)
+    get_target(target, keys)[key] = value
+    return index
+
+
+def parse_toml(data: str):
+    root = {}
+    target = root
+
+    index = 0
+    while True:
+        match = EXPRESSION_RE.search(data, index)
+        if not match:
+            break
+
+        if match.group('subtable'):
+            index = match.end()
+            path, is_list = match.group('path', 'is_list')
+            target = get_target(root, list(parse_key(path)), bool(is_list))
+            continue
+
+        index = parse_kv_pair(data, match.start(), target)
+        assert index is not None
+
+    return root
+
+
+def main():
+    import argparse
+    from pathlib import Path
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('infile', type=Path, help='The TOML file to read as input')
+    args = parser.parse_args()
+
+    with args.infile.open('r', encoding='utf-8') as file:
+        data = file.read()
+
+    def default(obj):
+        if isinstance(obj, (datetime.date, datetime.time, datetime.datetime)):
+            return obj.isoformat()
+
+    print(json.dumps(parse_toml(data), default=default))
+
+
+if __name__ == '__main__':
+    main()

From b8a433aaca86b15cb9f1a451b0f69371d2fc22a9 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sun, 11 Feb 2024 15:17:08 +0100
Subject: [PATCH 855/871] [devscripts] `install_deps`: Add script and migrate
 to it

Authored by: bashonly
---
 .github/workflows/build.yml      | 36 +++++++++--------
 .github/workflows/core.yml       |  2 +-
 .github/workflows/download.yml   |  4 +-
 .github/workflows/quick-test.yml |  6 +--
 .github/workflows/release.yml    |  3 +-
 README.md                        |  5 ++-
 devscripts/install_deps.py       | 66 ++++++++++++++++++++++++++++++++
 requirements.txt                 |  8 ----
 8 files changed, 95 insertions(+), 35 deletions(-)
 create mode 100755 devscripts/install_deps.py
 delete mode 100644 requirements.txt

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4b05e7cf9..082164c9e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -121,16 +121,14 @@ jobs:
       - name: Install Requirements
         run: |
           sudo apt -y install zip pandoc man sed
-          reqs=$(mktemp)
-          cat > "$reqs" << EOF
+          cat > ./requirements.txt << EOF
           python=3.10.*
-          pyinstaller
-          cffi
           brotli-python
-          secretstorage
           EOF
-          sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
-          mamba create -n build --file "$reqs"
+          python devscripts/install_deps.py --print \
+            --exclude brotli --exclude brotlicffi \
+            --include secretstorage --include pyinstaller >> ./requirements.txt
+          mamba create -n build --file ./requirements.txt
 
       - name: Prepare
         run: |
@@ -203,12 +201,13 @@ jobs:
             apt update
             apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
             python3.8 -m pip install -U pip setuptools wheel
-            # Cannot access requirements.txt from the repo directory at this stage
+            # Cannot access any files from the repo directory at this stage
             python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage
 
           run: |
             cd repo
-            python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt  # Cached version may be out of date
+            python3.8 devscripts/install_deps.py -o --include build
+            python3.8 devscripts/install_deps.py --include pyinstaller --include secretstorage  # Cached version may be out of date
             python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
             python3.8 devscripts/make_lazy_extractors.py
             python3.8 -m bundle.pyinstaller
@@ -240,9 +239,10 @@ jobs:
       - name: Install Requirements
         run: |
           brew install coreutils
-          python3 -m pip install -U --user pip setuptools wheel
+          python3 devscripts/install_deps.py --user -o --include build
+          python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt
           # We need to ignore wheels otherwise we break universal2 builds
-          python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
+          python3 -m pip install -U --user --no-binary :all: -r requirements.txt
 
       - name: Prepare
         run: |
@@ -293,8 +293,8 @@ jobs:
       - name: Install Requirements
         run: |
           brew install coreutils
-          python3 -m pip install -U --user pip setuptools wheel
-          python3 -m pip install -U --user Pyinstaller -r requirements.txt
+          python3 devscripts/install_deps.py --user -o --include build
+          python3 devscripts/install_deps.py --user --include pyinstaller
 
       - name: Prepare
         run: |
@@ -333,8 +333,9 @@ jobs:
           python-version: "3.8"
       - name: Install Requirements
         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
-          python -m pip install -U pip setuptools wheel py2exe
-          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
+          python devscripts/install_deps.py -o --include build
+          python devscripts/install_deps.py --include py2exe
+          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl"
 
       - name: Prepare
         run: |
@@ -382,8 +383,9 @@ jobs:
           architecture: "x86"
       - name: Install Requirements
         run: |
-          python -m pip install -U pip setuptools wheel
-          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
+          python devscripts/install_deps.py -o --include build
+          python devscripts/install_deps.py
+          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl"
 
       - name: Prepare
         run: |
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index eaaf03dee..f694c9bdd 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -53,7 +53,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install test requirements
-      run: pip install pytest -r requirements.txt
+      run: python3 ./devscripts/install_deps.py --include dev
     - name: Run tests
       continue-on-error: False
       run: |
diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml
index 9f47d6718..84339d970 100644
--- a/.github/workflows/download.yml
+++ b/.github/workflows/download.yml
@@ -15,7 +15,7 @@ jobs:
       with:
         python-version: 3.9
     - name: Install test requirements
-      run: pip install pytest -r requirements.txt
+      run: python3 ./devscripts/install_deps.py --include dev
     - name: Run tests
       continue-on-error: true
       run: python3 ./devscripts/run_tests.py download
@@ -42,7 +42,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install test requirements
-      run: pip install pytest -r requirements.txt
+      run: python3 ./devscripts/install_deps.py --include dev
     - name: Run tests
       continue-on-error: true
       run: python3 ./devscripts/run_tests.py download
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index 84fca62d4..4e9616926 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -15,7 +15,7 @@ jobs:
       with:
         python-version: '3.8'
     - name: Install test requirements
-      run: pip install pytest -r requirements.txt
+      run: python3 ./devscripts/install_deps.py --include dev
     - name: Run tests
       run: |
         python3 -m yt_dlp -v || true
@@ -28,8 +28,8 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
     - name: Install flake8
-      run: pip install flake8
+      run: python3 ./devscripts/install_deps.py -o --include dev
     - name: Make lazy extractors
-      run: python devscripts/make_lazy_extractors.py
+      run: python3 ./devscripts/make_lazy_extractors.py
     - name: Run flake8
       run: flake8 .
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d1508e5e6..1653add4f 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -253,8 +253,7 @@ jobs:
       - name: Install Requirements
         run: |
           sudo apt -y install pandoc man
-          python -m pip install -U pip setuptools wheel twine
-          python -m pip install -U -r requirements.txt
+          python devscripts/install_deps.py -o --include build
 
       - name: Prepare
         env:
diff --git a/README.md b/README.md
index c74777d2f..2fcb09917 100644
--- a/README.md
+++ b/README.md
@@ -324,7 +324,7 @@ ### Standalone PyInstaller Builds
 To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used. You can run the following commands:
 
 ```
-python3 -m pip install -U pyinstaller -r requirements.txt
+python3 devscripts/install_deps.py --include pyinstaller
 python3 devscripts/make_lazy_extractors.py
 python3 -m bundle.pyinstaller
 ```
@@ -351,13 +351,14 @@ ### Standalone Py2Exe Builds (Windows)
 If you wish to build it anyway, install Python (if it is not already installed) and you can run the following commands:
 
 ```
-py -m pip install -U py2exe -r requirements.txt
+py devscripts/install_deps.py --include py2exe
 py devscripts/make_lazy_extractors.py
 py -m bundle.py2exe
 ```
 
 ### Related scripts
 
+* **`devscripts/install_deps.py`** - Install dependencies for yt-dlp.
 * **`devscripts/update-version.py`** - Update the version number based on current date.
 * **`devscripts/set-variant.py`** - Set the build variant of the executable.
 * **`devscripts/make_changelog.py`** - Create a markdown changelog using short commit messages and update `CONTRIBUTORS` file.
diff --git a/devscripts/install_deps.py b/devscripts/install_deps.py
new file mode 100755
index 000000000..715e5b044
--- /dev/null
+++ b/devscripts/install_deps.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+# Allow execution from anywhere
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import argparse
+import re
+import subprocess
+
+from devscripts.tomlparse import parse_toml
+from devscripts.utils import read_file
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Install dependencies for yt-dlp')
+    parser.add_argument(
+        'input', nargs='?', metavar='TOMLFILE', default='pyproject.toml', help='Input file (default: %(default)s)')
+    parser.add_argument(
+        '-e', '--exclude', metavar='REQUIREMENT', action='append', help='Exclude a required dependency')
+    parser.add_argument(
+        '-i', '--include', metavar='GROUP', action='append', help='Include an optional dependency group')
+    parser.add_argument(
+        '-o', '--only-optional', action='store_true', help='Only install optional dependencies')
+    parser.add_argument(
+        '-p', '--print', action='store_true', help='Only print a requirements.txt to stdout')
+    parser.add_argument(
+        '-u', '--user', action='store_true', help='Install with pip as --user')
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    toml_data = parse_toml(read_file(args.input))
+    deps = toml_data['project']['dependencies']
+    targets = deps.copy() if not args.only_optional else []
+
+    for exclude in args.exclude or []:
+        for dep in deps:
+            simplified_dep = re.match(r'[\w-]+', dep)[0]
+            if dep in targets and (exclude.lower() == simplified_dep.lower() or exclude == dep):
+                targets.remove(dep)
+
+    optional_deps = toml_data['project']['optional-dependencies']
+    for include in args.include or []:
+        group = optional_deps.get(include)
+        if group:
+            targets.extend(group)
+
+    if args.print:
+        for target in targets:
+            print(target)
+        return
+
+    pip_args = [sys.executable, '-m', 'pip', 'install', '-U']
+    if args.user:
+        pip_args.append('--user')
+    pip_args.extend(targets)
+
+    return subprocess.call(pip_args)
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 06ff82a80..000000000
--- a/requirements.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-mutagen
-pycryptodomex
-brotli; implementation_name=='cpython'
-brotlicffi; implementation_name!='cpython'
-certifi
-requests>=2.31.0,<3
-urllib3>=1.26.17,<3
-websockets>=12.0

From 920397634d1e84e76d2cb897bd6d69ba0c6bd5ca Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sun, 11 Feb 2024 15:24:41 +0100
Subject: [PATCH 856/871] [build] Fix `secretstorage` for ARM builds

Authored by: bashonly
---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 082164c9e..0c2b0f684 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -199,10 +199,10 @@ jobs:
           dockerRunArgs: --volume "${PWD}/repo:/repo"
           install: | # Installing Python 3.10 from the Deadsnakes repo raises errors
             apt update
-            apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
+            apt -y install zlib1g-dev libffi-dev python3.8 python3.8-dev python3.8-distutils python3-pip
             python3.8 -m pip install -U pip setuptools wheel
             # Cannot access any files from the repo directory at this stage
-            python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage
+            python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage cffi
 
           run: |
             cd repo

From 867f637b95b342e1cb9f1dc3c6cf0ffe727187ce Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sun, 11 Feb 2024 17:35:27 +0100
Subject: [PATCH 857/871] [cleanup] Build files cleanup

- Fix `AUTHORS` file by doing an unshallow checkout
- Update triggers for nightly/master release

Authored by: bashonly
---
 .github/workflows/release-master.yml  | 2 ++
 .github/workflows/release-nightly.yml | 9 ++++++++-
 .github/workflows/release.yml         | 2 ++
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 2430dc5f8..a84547580 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -8,6 +8,8 @@ on:
       - "!yt_dlp/version.py"
       - "bundle/*.py"
       - "pyproject.toml"
+      - "Makefile"
+      - ".github/workflows/build.yml"
 concurrency:
   group: release-master
 permissions:
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index 16d583846..f459a3a17 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -18,7 +18,14 @@ jobs:
       - name: Check for new commits
         id: check_for_new_commits
         run: |
-          relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "bundle/*.py" "pyproject.toml")
+          relevant_files=(
+            "yt_dlp/*.py"
+            ':!yt_dlp/version.py'
+            "bundle/*.py"
+            "pyproject.toml"
+            "Makefile"
+            ".github/workflows/build.yml"
+          )
           echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT"
 
   release:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 1653add4f..eded11a13 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -246,6 +246,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

From b14e818b37f62e3224da157b3ad768b3f0815fcd Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sun, 11 Feb 2024 15:47:16 +0100
Subject: [PATCH 858/871] [ci] Bump `actions/setup-python` to v5

Authored by: bashonly
---
 .github/workflows/build.yml      | 6 +++---
 .github/workflows/core.yml       | 2 +-
 .github/workflows/download.yml   | 4 ++--
 .github/workflows/quick-test.yml | 4 ++--
 .github/workflows/release.yml    | 6 +++---
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0c2b0f684..4d8e8bf38 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -107,7 +107,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - uses: conda-incubator/setup-miniconda@v2
@@ -328,7 +328,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with: # 3.8 is used for Win7 support
           python-version: "3.8"
       - name: Install Requirements
@@ -377,7 +377,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.8"
           architecture: "x86"
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index f694c9bdd..ba8630630 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -49,7 +49,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install test requirements
diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml
index 84339d970..7256804d9 100644
--- a/.github/workflows/download.yml
+++ b/.github/workflows/download.yml
@@ -11,7 +11,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Install test requirements
@@ -38,7 +38,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install test requirements
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index 4e9616926..3114e7bdd 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -11,7 +11,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.8'
     - name: Install test requirements
@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
     - name: Install flake8
       run: python3 ./devscripts/install_deps.py -o --include dev
     - name: Make lazy extractors
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index eded11a13..fac096be7 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -71,7 +71,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 
@@ -248,7 +248,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 
@@ -297,7 +297,7 @@ jobs:
         with:
           fetch-depth: 0
       - uses: actions/download-artifact@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 

From b0059f0413a6ba6ab0a3aec1f00188ce083cd8bf Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sun, 11 Feb 2024 15:47:48 +0100
Subject: [PATCH 859/871] [build] Bump `conda-incubator/setup-miniconda` to v3

Authored by: bashonly
---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4d8e8bf38..e8a97e3f4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -110,7 +110,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           miniforge-variant: Mambaforge
           use-mamba: true

From 3876429d72afb35247f4b2531eb9b16cfc7e0968 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sun, 11 Feb 2024 15:48:09 +0100
Subject: [PATCH 860/871] [build] Bump `actions/upload-artifact` to v4 and
 adjust workflows

Authored by: bashonly
---
 .github/workflows/build.yml   | 36 ++++++++++++++++++++++++++---------
 .github/workflows/release.yml |  6 +++++-
 2 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e8a97e3f4..cd7ead796 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -162,13 +162,15 @@ jobs:
           done
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-${{ github.job }}
           path: |
             yt-dlp
             yt-dlp.tar.gz
             yt-dlp_linux
             yt-dlp_linux.zip
+          compression-level: 0
 
   linux_arm:
     needs: process
@@ -223,10 +225,12 @@ jobs:
             fi
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-linux_${{ matrix.architecture }}
           path: | # run-on-arch-action designates armv7l as armv7
             repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
+          compression-level: 0
 
   macos:
     needs: process
@@ -265,11 +269,13 @@ jobs:
           [[ "$version" != "$downgraded_version" ]]
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-${{ github.job }}
           path: |
             dist/yt-dlp_macos
             dist/yt-dlp_macos.zip
+          compression-level: 0
 
   macos_legacy:
     needs: process
@@ -316,10 +322,12 @@ jobs:
           [[ "$version" != "$downgraded_version" ]]
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-${{ github.job }}
           path: |
             dist/yt-dlp_macos_legacy
+          compression-level: 0
 
   windows:
     needs: process
@@ -363,12 +371,14 @@ jobs:
           }
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-${{ github.job }}
           path: |
             dist/yt-dlp.exe
             dist/yt-dlp_min.exe
             dist/yt-dlp_win.zip
+          compression-level: 0
 
   windows32:
     needs: process
@@ -409,10 +419,12 @@ jobs:
           }
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-${{ github.job }}
           path: |
             dist/yt-dlp_x86.exe
+          compression-level: 0
 
   meta_files:
     if: inputs.meta_files && always() && !cancelled()
@@ -426,7 +438,11 @@ jobs:
       - windows32
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifact
+          pattern: build-*
+          merge-multiple: true
 
       - name: Make SHA2-SUMS files
         run: |
@@ -461,8 +477,10 @@ jobs:
           done
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-${{ github.job }}
           path: |
-            SHA*SUMS*
             _update_spec
+            SHA*SUMS*
+          compression-level: 0
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index fac096be7..f5c6a793e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -296,7 +296,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifact
+          pattern: build-*
+          merge-multiple: true
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"

From 1ed5ee2f045f717e814f84ba461dadc58e712266 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Tue, 13 Feb 2024 04:11:17 +0100
Subject: [PATCH 861/871] [ie/Ant1NewsGrEmbed] Fix extractor (#9191)

Authored by: seproDev
---
 yt_dlp/extractor/antenna.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/antenna.py b/yt_dlp/extractor/antenna.py
index c78717aa9..17a4b6900 100644
--- a/yt_dlp/extractor/antenna.py
+++ b/yt_dlp/extractor/antenna.py
@@ -78,14 +78,14 @@ class Ant1NewsGrArticleIE(AntennaBaseIE):
 
     _TESTS = [{
         'url': 'https://www.ant1news.gr/afieromata/article/549468/o-tzeims-mpont-sta-meteora-oi-apeiles-kai-o-xesikomos-ton-kalogeron',
-        'md5': '294f18331bb516539d72d85a82887dcc',
+        'md5': '57eb8d12181f0fa2b14b0b138e1de9b6',
         'info_dict': {
             'id': '_xvg/m_cmbatw=',
             'ext': 'mp4',
             'title': 'md5:a93e8ecf2e4073bfdffcb38f59945411',
-            'timestamp': 1603092840,
-            'upload_date': '20201019',
-            'thumbnail': 'https://ant1media.azureedge.net/imgHandler/640/756206d2-d640-40e2-b201-3555abdfc0db.jpg',
+            'timestamp': 1666166520,
+            'upload_date': '20221019',
+            'thumbnail': 'https://ant1media.azureedge.net/imgHandler/1920/756206d2-d640-40e2-b201-3555abdfc0db.jpg',
         },
     }, {
         'url': 'https://ant1news.gr/Society/article/620286/symmoria-anilikon-dikigoros-thymaton-ithelan-na-toys-apoteleiosoyn',
@@ -117,7 +117,7 @@ class Ant1NewsGrEmbedIE(AntennaBaseIE):
     _BASE_PLAYER_URL_RE = r'(?:https?:)?//(?:[a-zA-Z0-9\-]+\.)?(?:antenna|ant1news)\.gr/templates/pages/player'
     _VALID_URL = rf'{_BASE_PLAYER_URL_RE}\?([^#]+&)?cid=(?P<id>[^#&]+)'
     _EMBED_REGEX = [rf'<iframe[^>]+?src=(?P<_q1>["\'])(?P<url>{_BASE_PLAYER_URL_RE}\?(?:(?!(?P=_q1)).)+)(?P=_q1)']
-    _API_PATH = '/news/templates/data/jsonPlayer'
+    _API_PATH = '/templates/data/jsonPlayer'
 
     _TESTS = [{
         'url': 'https://www.antenna.gr/templates/pages/player?cid=3f_li_c_az_jw_y_u=&w=670&h=377',

From 03536126d32bd861e38536371f0cd5f1b71dcb7a Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Tue, 13 Feb 2024 04:11:40 +0100
Subject: [PATCH 862/871] [ie/CrooksAndLiars] Fix extractor (#9192)

Authored by: seproDev
---
 yt_dlp/extractor/crooksandliars.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/crooksandliars.py b/yt_dlp/extractor/crooksandliars.py
index 4de7e3d53..2ee0730c9 100644
--- a/yt_dlp/extractor/crooksandliars.py
+++ b/yt_dlp/extractor/crooksandliars.py
@@ -33,10 +33,7 @@ def _real_extract(self, url):
         webpage = self._download_webpage(
             'http://embed.crooksandliars.com/embed/%s' % video_id, video_id)
 
-        manifest = self._parse_json(
-            self._search_regex(
-                r'var\s+manifest\s*=\s*({.+?})\n', webpage, 'manifest JSON'),
-            video_id)
+        manifest = self._search_json(r'var\s+manifest\s*=', webpage, 'manifest JSON', video_id)
 
         quality = qualities(('webm_low', 'mp4_low', 'webm_high', 'mp4_high'))
 

From cd0443fb14e2ed805abb02792473457553a123d1 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Tue, 13 Feb 2024 04:12:17 +0100
Subject: [PATCH 863/871] [ie/Funk] Fix extractor (#9194)

Authored by: seproDev
---
 yt_dlp/extractor/funk.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/yt_dlp/extractor/funk.py b/yt_dlp/extractor/funk.py
index 539d719c5..8bdea3fce 100644
--- a/yt_dlp/extractor/funk.py
+++ b/yt_dlp/extractor/funk.py
@@ -1,25 +1,29 @@
 from .common import InfoExtractor
 from .nexx import NexxIE
-from ..utils import (
-    int_or_none,
-    str_or_none,
-)
 
 
 class FunkIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.|origin\.)?funk\.net/(?:channel|playlist)/[^/]+/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.funk.net/channel/ba-793/die-lustigsten-instrumente-aus-dem-internet-teil-2-1155821',
-        'md5': '8dd9d9ab59b4aa4173b3197f2ea48e81',
+        'md5': '8610449476156f338761a75391b0017d',
         'info_dict': {
             'id': '1155821',
             'ext': 'mp4',
             'title': 'Die LUSTIGSTEN INSTRUMENTE aus dem Internet - Teil 2',
-            'description': 'md5:a691d0413ef4835588c5b03ded670c1f',
+            'description': 'md5:2a03b67596eda0d1b5125c299f45e953',
             'timestamp': 1514507395,
             'upload_date': '20171229',
+            'duration': 426.0,
+            'cast': ['United Creators PMB GmbH'],
+            'thumbnail': 'https://assets.nexx.cloud/media/75/56/79/3YKUSJN1LACN0CRxL.jpg',
+            'display_id': 'die-lustigsten-instrumente-aus-dem-internet-teil-2',
+            'alt_title': 'Die LUSTIGSTEN INSTRUMENTE aus dem Internet Teil 2',
+            'season_number': 0,
+            'season': 'Season 0',
+            'episode_number': 0,
+            'episode': 'Episode 0',
         },
-
     }, {
         'url': 'https://www.funk.net/playlist/neuesteVideos/kameras-auf-dem-fusion-festival-1618699',
         'only_matching': True,
@@ -27,18 +31,10 @@ class FunkIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id, nexx_id = self._match_valid_url(url).groups()
-        video = self._download_json(
-            'https://www.funk.net/api/v4.0/videos/' + nexx_id, nexx_id)
         return {
             '_type': 'url_transparent',
-            'url': 'nexx:741:' + nexx_id,
+            'url': f'nexx:741:{nexx_id}',
             'ie_key': NexxIE.ie_key(),
             'id': nexx_id,
-            'title': video.get('title'),
-            'description': video.get('description'),
-            'duration': int_or_none(video.get('duration')),
-            'channel_id': str_or_none(video.get('channelId')),
             'display_id': display_id,
-            'tags': video.get('tags'),
-            'thumbnail': video.get('imageUrlLandscape'),
         }

From 9401736fd08767c58af45a1e36ff5929c5fa1ac9 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Tue, 13 Feb 2024 20:52:41 +0100
Subject: [PATCH 864/871] [ie/LeFigaroVideoEmbed] Fix extractor (#9198)

Authored by: seproDev
---
 yt_dlp/extractor/lefigaro.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/yt_dlp/extractor/lefigaro.py b/yt_dlp/extractor/lefigaro.py
index 9465095db..a452d8706 100644
--- a/yt_dlp/extractor/lefigaro.py
+++ b/yt_dlp/extractor/lefigaro.py
@@ -13,7 +13,7 @@ class LeFigaroVideoEmbedIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://video.lefigaro.fr/embed/figaro/video/les-francais-ne-veulent-ils-plus-travailler-suivez-en-direct-le-club-le-figaro-idees/',
-        'md5': 'e94de44cd80818084352fcf8de1ce82c',
+        'md5': 'a0c3069b7e4c4526abf0053a7713f56f',
         'info_dict': {
             'id': 'g9j7Eovo',
             'title': 'Les Français ne veulent-ils plus travailler ? Retrouvez Le Club Le Figaro Idées',
@@ -26,7 +26,7 @@ class LeFigaroVideoEmbedIE(InfoExtractor):
         },
     }, {
         'url': 'https://video.lefigaro.fr/embed/figaro/video/intelligence-artificielle-faut-il-sen-mefier/',
-        'md5': '0b3f10332b812034b3a3eda1ef877c5f',
+        'md5': '319c662943dd777bab835cae1e2d73a5',
         'info_dict': {
             'id': 'LeAgybyc',
             'title': 'Intelligence artificielle : faut-il s’en méfier ?',
@@ -41,7 +41,7 @@ class LeFigaroVideoEmbedIE(InfoExtractor):
 
     _WEBPAGE_TESTS = [{
         'url': 'https://video.lefigaro.fr/figaro/video/suivez-en-direct-le-club-le-figaro-international-avec-philippe-gelie-9/',
-        'md5': '3972ddf2d5f8b98699f191687258e2f9',
+        'md5': '6289f9489efb969e38245f31721596fe',
         'info_dict': {
             'id': 'QChnbPYA',
             'title': 'Où en est le couple franco-allemand ? Retrouvez Le Club Le Figaro International',
@@ -55,7 +55,7 @@ class LeFigaroVideoEmbedIE(InfoExtractor):
         },
     }, {
         'url': 'https://video.lefigaro.fr/figaro/video/la-philosophe-nathalie-sarthou-lajus-est-linvitee-du-figaro-live/',
-        'md5': '3ac0a0769546ee6be41ab52caea5d9a9',
+        'md5': 'f6df814cae53e85937621599d2967520',
         'info_dict': {
             'id': 'QJzqoNbf',
             'title': 'La philosophe Nathalie Sarthou-Lajus est l’invitée du Figaro Live',
@@ -73,7 +73,8 @@ def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        player_data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['pageData']['playerData']
+        player_data = self._search_nextjs_data(
+            webpage, display_id)['props']['pageProps']['initialProps']['pageData']['playerData']
 
         return self.url_result(
             f'jwplatform:{player_data["videoId"]}', title=player_data.get('title'),

From 3dc9232e1aa58fe3c2d8cafb50e8162d6f0e891e Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Tue, 13 Feb 2024 20:53:17 +0100
Subject: [PATCH 865/871] [ie/MagellanTV] Support episodes (#9199)

Authored by: seproDev
---
 yt_dlp/extractor/magellantv.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/yt_dlp/extractor/magellantv.py b/yt_dlp/extractor/magellantv.py
index 0947a450a..6f2524ba2 100644
--- a/yt_dlp/extractor/magellantv.py
+++ b/yt_dlp/extractor/magellantv.py
@@ -28,12 +28,24 @@ class MagellanTVIE(InfoExtractor):
             'tags': ['Investigation', 'True Crime', 'Justice', 'Europe'],
         },
         'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.magellantv.com/watch/celebration-nation',
+        'info_dict': {
+            'id': 'celebration-nation',
+            'ext': 'mp4',
+            'tags': ['Art & Culture', 'Human Interest', 'Anthropology', 'China', 'History'],
+            'duration': 2640.0,
+            'title': 'Ancestors',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['reactContext']['video']['detail']
+        data = traverse_obj(self._search_nextjs_data(webpage, video_id), (
+            'props', 'pageProps', 'reactContext',
+            (('video', 'detail'), ('series', 'currentEpisode')), {dict}), get_all=False)
         formats, subtitles = self._extract_m3u8_formats_and_subtitles(data['jwpVideoUrl'], video_id)
 
         return {

From fb44020fa98e47620b3aa1dab94b4c5b7bfb40bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Florian=20Mei=C3=9Fner?=
 <936176+t-nil@users.noreply.github.com>
Date: Wed, 14 Feb 2024 22:12:34 +0100
Subject: [PATCH 866/871] [build:Makefile] Fix man pages generated by
 `pandoc>=3` (#7047)

Closes #7046, Closes #8481
Authored by: t-nil
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2f36c0cd1..5dddaaecc 100644
--- a/Makefile
+++ b/Makefile
@@ -42,7 +42,7 @@ PYTHON ?= /usr/bin/env python3
 SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi)
 
 # set markdown input format to "markdown-smart" for pandoc version 2 and to "markdown" for pandoc prior to version 2
-MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` = "2" ]; then echo markdown-smart; else echo markdown; fi)
+MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` -ge "2" ]; then echo markdown-smart; else echo markdown; fi)
 
 install: lazy-extractors yt-dlp yt-dlp.1 completions
 	mkdir -p $(DESTDIR)$(BINDIR)

From beaa1a44554d04d9fe63a743a5bb4431ca778f28 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Thu, 15 Feb 2024 16:42:43 -0600
Subject: [PATCH 867/871] [build:Makefile] Ensure compatibility with BSD `make`
 (#9210)

Authored by: bashonly
---
 Makefile | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 5dddaaecc..c33984f6f 100644
--- a/Makefile
+++ b/Makefile
@@ -38,11 +38,13 @@ MANDIR ?= $(PREFIX)/man
 SHAREDIR ?= $(PREFIX)/share
 PYTHON ?= /usr/bin/env python3
 
-# set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
-SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi)
+# $(shell) and $(error) are no-ops in BSD Make and the != variable assignment operator is not supported by GNU Make <4.0
+VERSION_CHECK != echo supported
+VERSION_CHECK ?= $(error GNU Make 4+ or BSD Make is required)
+CHECK_VERSION := $(VERSION_CHECK)
 
-# set markdown input format to "markdown-smart" for pandoc version 2 and to "markdown" for pandoc prior to version 2
-MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` -ge "2" ]; then echo markdown-smart; else echo markdown; fi)
+# set markdown input format to "markdown-smart" for pandoc version 2+ and to "markdown" for pandoc prior to version 2
+MARKDOWN != if [ "`pandoc -v | head -n1 | cut -d' ' -f2 | head -c1`" -ge "2" ]; then echo markdown-smart; else echo markdown; fi
 
 install: lazy-extractors yt-dlp yt-dlp.1 completions
 	mkdir -p $(DESTDIR)$(BINDIR)
@@ -73,17 +75,17 @@ test:
 offlinetest: codetest
 	$(PYTHON) -m pytest -k "not download"
 
-CODE_FOLDERS := $(shell find yt_dlp -type d -not -name '__*' -exec sh -c 'test -e "$$1"/__init__.py' sh {} \; -print)
-CODE_FILES := $(shell for f in $(CODE_FOLDERS); do echo "$$f" | awk '{gsub(/\/[^\/]+/,"/*"); print $$1"/*.py"}'; done | sort -u)
+CODE_FOLDERS != find yt_dlp -type f -name '__init__.py' -exec dirname {} \+ | grep -v '/__' | sort
+CODE_FILES != for f in $(CODE_FOLDERS) ; do echo "$$f" | sed 's,$$,/*.py,' ; done
 yt-dlp: $(CODE_FILES)
 	mkdir -p zip
 	for d in $(CODE_FOLDERS) ; do \
 	  mkdir -p zip/$$d ;\
 	  cp -pPR $$d/*.py zip/$$d/ ;\
 	done
-	cd zip ; touch -t 200001010101 $(CODE_FILES)
+	(cd zip && touch -t 200001010101 $(CODE_FILES))
 	mv zip/yt_dlp/__main__.py zip/
-	cd zip ; zip -q ../yt-dlp $(CODE_FILES) __main__.py
+	(cd zip && zip -q ../yt-dlp $(CODE_FILES) __main__.py)
 	rm -rf zip
 	echo '#!$(PYTHON)' > yt-dlp
 	cat yt-dlp.zip >> yt-dlp
@@ -127,7 +129,7 @@ completions/fish/yt-dlp.fish: $(CODE_FILES) devscripts/fish-completion.in
 	mkdir -p completions/fish
 	$(PYTHON) devscripts/fish-completion.py
 
-_EXTRACTOR_FILES = $(shell find yt_dlp/extractor -name '*.py' -and -not -name 'lazy_extractors.py')
+_EXTRACTOR_FILES != find yt_dlp/extractor -name '*.py' -and -not -name 'lazy_extractors.py'
 yt_dlp/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES)
 	$(PYTHON) devscripts/make_lazy_extractors.py $@
 
@@ -141,6 +143,7 @@ yt-dlp.tar.gz: all
 		--exclude '__pycache__' \
 		--exclude '.pytest_cache' \
 		--exclude '.git' \
+		--exclude '__pyinstaller' \
 		-- \
 		README.md supportedsites.md Changelog.md LICENSE \
 		CONTRIBUTING.md Collaborators.md CONTRIBUTORS AUTHORS \

From 2e30b5567b5c6113d46b39163db5b044aea8667e Mon Sep 17 00:00:00 2001
From: ringus1 <ringus1@users.noreply.github.com>
Date: Thu, 15 Feb 2024 13:46:57 -0600
Subject: [PATCH 868/871] [ie/facebook] Improve extraction

Partially addresses #4311

Authored by: jingtra, ringus1

Co-authored-by: Jing Kjeldsen <jingtra@gmail.com>
---
 yt_dlp/extractor/facebook.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 830bbcc3c..834b1df18 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -500,6 +500,7 @@ def extract_metadata(webpage):
                 webpage, 'description', default=None)
             uploader_data = (
                 get_first(media, ('owner', {dict}))
+                or get_first(post, ('video', 'creation_story', 'attachments', ..., 'media', lambda k, v: k == 'owner' and v['name']))
                 or get_first(post, (..., 'video', lambda k, v: k == 'owner' and v['name']))
                 or get_first(post, ('node', 'actors', ..., {dict}))
                 or get_first(post, ('event', 'event_creator', {dict})) or {})
@@ -583,8 +584,8 @@ def extract_relay_data(_filter):
         def extract_relay_prefetched_data(_filter):
             return traverse_obj(extract_relay_data(_filter), (
                 'require', (None, (..., ..., ..., '__bbox', 'require')),
-                lambda _, v: 'RelayPrefetchedStreamCache' in v, ..., ...,
-                '__bbox', 'result', 'data', {dict}), get_all=False) or {}
+                lambda _, v: any(key.startswith('RelayPrefetchedStreamCache') for key in v),
+                ..., ..., '__bbox', 'result', 'data', {dict}), get_all=False) or {}
 
         if not video_data:
             server_js_data = self._parse_json(self._search_regex([

From 017adb28e7fe7b8c8fc472332d86740f31141519 Mon Sep 17 00:00:00 2001
From: barsnick <barsnick@users.noreply.github.com>
Date: Fri, 16 Feb 2024 01:19:00 +0100
Subject: [PATCH 869/871] [ie/LinkedIn] Fix metadata and extract subtitles
 (#9056)

Closes #9003
Authored by: barsnick
---
 yt_dlp/extractor/linkedin.py | 53 ++++++++++++++++++++++++------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/yt_dlp/extractor/linkedin.py b/yt_dlp/extractor/linkedin.py
index 2bf2e9a11..ad41c0e20 100644
--- a/yt_dlp/extractor/linkedin.py
+++ b/yt_dlp/extractor/linkedin.py
@@ -3,16 +3,15 @@
 
 from .common import InfoExtractor
 from ..utils import (
-    clean_html,
-    extract_attributes,
     ExtractorError,
+    extract_attributes,
     float_or_none,
-    get_element_by_class,
     int_or_none,
     srt_subtitles_timecode,
-    strip_or_none,
     mimetype2ext,
+    traverse_obj,
     try_get,
+    url_or_none,
     urlencode_postdata,
     urljoin,
 )
@@ -83,15 +82,29 @@ def _get_video_id(self, video_data, course_slug, video_slug):
 
 
 class LinkedInIE(LinkedInBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?linkedin\.com/posts/.+?(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?linkedin\.com/posts/[^/?#]+-(?P<id>\d+)-\w{4}/?(?:[?#]|$)'
     _TESTS = [{
         'url': 'https://www.linkedin.com/posts/mishalkhawaja_sendinblueviews-toronto-digitalmarketing-ugcPost-6850898786781339649-mM20',
         'info_dict': {
             'id': '6850898786781339649',
             'ext': 'mp4',
-            'title': 'Mishal K. on LinkedIn: #sendinblueviews #toronto #digitalmarketing',
-            'description': 'md5:be125430bab1c574f16aeb186a4d5b19',
-            'creator': 'Mishal K.'
+            'title': 'Mishal K. on LinkedIn: #sendinblueviews #toronto #digitalmarketing #nowhiring #sendinblue…',
+            'description': 'md5:2998a31f6f479376dd62831f53a80f71',
+            'uploader': 'Mishal K.',
+            'thumbnail': 're:^https?://media.licdn.com/dms/image/.*$',
+            'like_count': int
+        },
+    }, {
+        'url': 'https://www.linkedin.com/posts/the-mathworks_2_what-is-mathworks-cloud-center-activity-7151241570371948544-4Gu7',
+        'info_dict': {
+            'id': '7151241570371948544',
+            'ext': 'mp4',
+            'title': 'MathWorks on LinkedIn: What Is MathWorks Cloud Center?',
+            'description': 'md5:95f9d4eeb6337882fb47eefe13d7a40c',
+            'uploader': 'MathWorks',
+            'thumbnail': 're:^https?://media.licdn.com/dms/image/.*$',
+            'like_count': int,
+            'subtitles': 'mincount:1'
         },
     }]
 
@@ -99,26 +112,30 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_extract_title(webpage)
-        description = clean_html(get_element_by_class('share-update-card__update-text', webpage))
-        like_count = int_or_none(get_element_by_class('social-counts-reactions__social-counts-numRections', webpage))
-        creator = strip_or_none(clean_html(get_element_by_class('comment__actor-name', webpage)))
-
-        sources = self._parse_json(extract_attributes(self._search_regex(r'(<video[^>]+>)', webpage, 'video'))['data-sources'], video_id)
+        video_attrs = extract_attributes(self._search_regex(r'(<video[^>]+>)', webpage, 'video'))
+        sources = self._parse_json(video_attrs['data-sources'], video_id)
         formats = [{
             'url': source['src'],
             'ext': mimetype2ext(source.get('type')),
             'tbr': float_or_none(source.get('data-bitrate'), scale=1000),
         } for source in sources]
+        subtitles = {'en': [{
+            'url': video_attrs['data-captions-url'],
+            'ext': 'vtt',
+        }]} if url_or_none(video_attrs.get('data-captions-url')) else {}
 
         return {
             'id': video_id,
             'formats': formats,
-            'title': title,
-            'like_count': like_count,
-            'creator': creator,
+            'title': self._og_search_title(webpage, default=None) or self._html_extract_title(webpage),
+            'like_count': int_or_none(self._search_regex(
+                r'\bdata-num-reactions="(\d+)"', webpage, 'reactions', default=None)),
+            'uploader': traverse_obj(
+                self._yield_json_ld(webpage, video_id),
+                (lambda _, v: v['@type'] == 'SocialMediaPosting', 'author', 'name', {str}), get_all=False),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': description,
+            'description': self._og_search_description(webpage, default=None),
+            'subtitles': subtitles,
         }
 
 

From f78814923748277e7067b796f25870686fb46205 Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Fri, 16 Feb 2024 01:20:58 +0100
Subject: [PATCH 870/871] [ie/rai] Filter unavailable formats (#9189)

Closes #9154
Authored by: nixxo
---
 yt_dlp/extractor/rai.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/rai.py b/yt_dlp/extractor/rai.py
index df4102a40..f6219c2db 100644
--- a/yt_dlp/extractor/rai.py
+++ b/yt_dlp/extractor/rai.py
@@ -1,6 +1,7 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     clean_html,
     determine_ext,
@@ -91,7 +92,7 @@ def fix_cdata(s):
             self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
 
         if not audio_only and not is_live:
-            formats.extend(self._create_http_urls(media_url, relinker_url, formats))
+            formats.extend(self._create_http_urls(media_url, relinker_url, formats, video_id))
 
         return filter_dict({
             'is_live': is_live,
@@ -99,7 +100,7 @@ def fix_cdata(s):
             'formats': formats,
         })
 
-    def _create_http_urls(self, manifest_url, relinker_url, fmts):
+    def _create_http_urls(self, manifest_url, relinker_url, fmts, video_id):
         _MANIFEST_REG = r'/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4)?(?:\.csmil)?/playlist\.m3u8'
         _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
         _QUALITY = {
@@ -166,6 +167,14 @@ def get_format_info(tbr):
                 'fps': 25,
             }
 
+        # Check if MP4 download is available
+        try:
+            self._request_webpage(
+                HEADRequest(_MP4_TMPL % (relinker_url, '*')), video_id, 'Checking MP4 availability')
+        except ExtractorError as e:
+            self.to_screen(f'{video_id}: MP4 direct download is not available: {e.cause}')
+            return []
+
         # filter out single-stream formats
         fmts = [f for f in fmts
                 if not f.get('vcodec') == 'none' and not f.get('acodec') == 'none']

From ddd4b5e10a653bee78e656107710021c1b82934c Mon Sep 17 00:00:00 2001
From: diman8 <dbuyakov@gmail.com>
Date: Fri, 16 Feb 2024 17:59:25 +0100
Subject: [PATCH 871/871] [ie/SVTPage] Fix extractor (#8938)

Closes #8930
Authored by: diman8
---
 yt_dlp/extractor/svt.py | 81 ++++++++++++++++++++++++++++++-----------
 1 file changed, 59 insertions(+), 22 deletions(-)

diff --git a/yt_dlp/extractor/svt.py b/yt_dlp/extractor/svt.py
index 18da87534..573147a45 100644
--- a/yt_dlp/extractor/svt.py
+++ b/yt_dlp/extractor/svt.py
@@ -7,8 +7,6 @@
     determine_ext,
     dict_get,
     int_or_none,
-    str_or_none,
-    strip_or_none,
     traverse_obj,
     try_get,
     unified_timestamp,
@@ -388,15 +386,55 @@ def _real_extract(self, url):
             dict_get(series, ('longDescription', 'shortDescription')))
 
 
-class SVTPageIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?svt\.se/(?P<path>(?:[^/]+/)*(?P<id>[^/?&#]+))'
+class SVTPageIE(SVTBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?svt\.se/(?:[^/?#]+/)*(?P<id>[^/?&#]+)'
     _TESTS = [{
+        'url': 'https://www.svt.se/nyheter/lokalt/skane/viktor-18-forlorade-armar-och-ben-i-sepsis-vill-ateruppta-karaten-och-bli-svetsare',
+        'info_dict': {
+            'title': 'Viktor, 18, förlorade armar och ben i sepsis – vill återuppta karaten och bli svetsare',
+            'id': 'viktor-18-forlorade-armar-och-ben-i-sepsis-vill-ateruppta-karaten-och-bli-svetsare',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'https://www.svt.se/nyheter/lokalt/skane/forsvarsmakten-om-trafikkaoset-pa-e22-kunde-inte-varit-dar-snabbare',
+        'info_dict': {
+            'id': 'jXvk42E',
+            'title': 'Försvarsmakten om trafikkaoset på E22: Kunde inte varit där snabbare',
+            'ext': 'mp4',
+            "duration": 80,
+            'age_limit': 0,
+            'timestamp': 1704370009,
+            'episode': 'Försvarsmakten om trafikkaoset på E22: Kunde inte varit där snabbare',
+            'series': 'Lokala Nyheter Skåne',
+            'upload_date': '20240104'
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://www.svt.se/nyheter/svtforum/2023-tungt-ar-for-svensk-media',
+        'info_dict': {
+            'title': '2023 tungt år för svensk media',
+            'id': 'ewqAZv4',
+            'ext': 'mp4',
+            "duration": 3074,
+            'age_limit': 0,
+            'series': '',
+            'timestamp': 1702980479,
+            'upload_date': '20231219',
+            'episode': 'Mediestudier'
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
         'url': 'https://www.svt.se/sport/ishockey/bakom-masken-lehners-kamp-mot-mental-ohalsa',
         'info_dict': {
             'id': '25298267',
             'title': 'Bakom masken – Lehners kamp mot mental ohälsa',
         },
         'playlist_count': 4,
+        'skip': 'Video is gone'
     }, {
         'url': 'https://www.svt.se/nyheter/utrikes/svenska-andrea-ar-en-mil-fran-branderna-i-kalifornien',
         'info_dict': {
@@ -404,6 +442,7 @@ class SVTPageIE(InfoExtractor):
             'title': 'Svenska Andrea redo att fly sitt hem i Kalifornien',
         },
         'playlist_count': 2,
+        'skip': 'Video is gone'
     }, {
         # only programTitle
         'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
@@ -414,6 +453,7 @@ class SVTPageIE(InfoExtractor):
             'duration': 27,
             'age_limit': 0,
         },
+        'skip': 'Video is gone'
     }, {
         'url': 'https://www.svt.se/nyheter/lokalt/vast/svt-testar-tar-nagon-upp-skrapet-1',
         'only_matching': True,
@@ -427,26 +467,23 @@ def suitable(cls, url):
         return False if SVTIE.suitable(url) or SVTPlayIE.suitable(url) else super(SVTPageIE, cls).suitable(url)
 
     def _real_extract(self, url):
-        path, display_id = self._match_valid_url(url).groups()
+        display_id = self._match_id(url)
 
-        article = self._download_json(
-            'https://api.svt.se/nss-api/page/' + path, display_id,
-            query={'q': 'articles'})['articles']['content'][0]
+        webpage = self._download_webpage(url, display_id)
+        title = self._og_search_title(webpage)
 
-        entries = []
+        urql_state = self._search_json(
+            r'window\.svt\.nyh\.urqlState\s*=', webpage, 'json data', display_id)
 
-        def _process_content(content):
-            if content.get('_type') in ('VIDEOCLIP', 'VIDEOEPISODE'):
-                video_id = compat_str(content['image']['svtId'])
-                entries.append(self.url_result(
-                    'svt:' + video_id, SVTPlayIE.ie_key(), video_id))
+        data = traverse_obj(urql_state, (..., 'data', {str}, {json.loads}), get_all=False) or {}
 
-        for media in article.get('media', []):
-            _process_content(media)
+        def entries():
+            for video_id in set(traverse_obj(data, (
+                'page', (('topMedia', 'svtId'), ('body', ..., 'video', 'svtId')), {str}
+            ))):
+                info = self._extract_video(
+                    self._download_json(f'https://api.svt.se/video/{video_id}', video_id), video_id)
+                info['title'] = title
+                yield info
 
-        for obj in article.get('structuredBody', []):
-            _process_content(obj.get('content') or {})
-
-        return self.playlist_result(
-            entries, str_or_none(article.get('id')),
-            strip_or_none(article.get('title')))
+        return self.playlist_result(entries(), display_id, title)